From 2c6e765d569a2d83b57782b33f7271d0a83a572d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Jul 2020 23:50:45 +0200 Subject: [PATCH 0001/1284] Add an updateSchema() method in the Query class --- src/sqlancer/GlobalState.java | 5 ++++- src/sqlancer/clickhouse/ClickHouseProvider.java | 5 +++++ src/sqlancer/cockroachdb/CockroachDBProvider.java | 5 +++++ src/sqlancer/duckdb/DuckDBProvider.java | 5 +++++ src/sqlancer/mariadb/MariaDBProvider.java | 5 +++++ src/sqlancer/mysql/MySQLGlobalState.java | 7 +++++++ src/sqlancer/postgres/PostgresGlobalState.java | 5 +++++ src/sqlancer/sqlite3/SQLite3Provider.java | 5 +++++ src/sqlancer/tidb/TiDBProvider.java | 5 +++++ 9 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 16c4e6233..cc8c334a8 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,6 +1,7 @@ package sqlancer; import java.sql.Connection; +import java.sql.SQLException; import sqlancer.Main.QueryManager; import sqlancer.Main.StateLogger; @@ -11,7 +12,7 @@ * @param * the option parameter. */ -public class GlobalState { +public abstract class GlobalState { private Connection con; private Randomly r; @@ -87,4 +88,6 @@ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } + protected abstract void updateSchema() throws SQLException; + } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 7e519dca3..82fd1749c 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -87,6 +87,11 @@ public String getOracleName() { public String getDatabaseName() { return super.getDatabaseName() + this.getOracleName(); } + + @Override + protected void updateSchema() throws SQLException { + setSchema(ClickHouseSchema.fromConnection(getConnection(), getDatabaseName())); + } } @Override diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 6e7f7352d..afa02f379 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -127,6 +127,11 @@ public CockroachDBSchema getSchema() { return schema; } + @Override + protected void updateSchema() throws SQLException { + setSchema(CockroachDBSchema.fromConnection(getConnection(), getDatabaseName())); + } + } @Override diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index ab6399b7e..cdb62ada0 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -104,6 +104,11 @@ public DuckDBSchema getSchema() { return schema; } + @Override + protected void updateSchema() throws SQLException { + setSchema(DuckDBSchema.fromConnection(getConnection(), getDatabaseName())); + } + } @Override diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 985b1375a..e76205abf 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -204,6 +204,11 @@ public MariaDBSchema getSchema() { return schema; } + @Override + protected void updateSchema() throws SQLException { + setSchema(MariaDBSchema.fromConnection(getConnection(), getDatabaseName())); + } + } @Override diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 173993a4e..f8eff8171 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -1,6 +1,8 @@ package sqlancer.mysql; +import java.sql.SQLException; + import sqlancer.GlobalState; public class MySQLGlobalState extends GlobalState { @@ -15,4 +17,9 @@ public MySQLSchema getSchema() { return schema; } + @Override + protected void updateSchema() throws SQLException { + setSchema(MySQLSchema.fromConnection(getConnection(), getDatabaseName())); + } + } diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 29ee305fe..332c58d44 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -98,4 +98,9 @@ public String getRandomOpclass() { return Randomly.fromList(opClasses); } + @Override + protected void updateSchema() throws SQLException { + setSchema(PostgresSchema.fromConnection(getConnection(), getDatabaseName())); + } + } diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 2249c0d1e..60033a2b7 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -181,6 +181,11 @@ public SQLite3Options getSqliteOptions() { return sqliteOptions; } + @Override + protected void updateSchema() throws SQLException { + setSchema(SQLite3Schema.fromConnection(this)); + } + } private enum TableType { diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 77010cea3..db1df95f6 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -87,6 +87,11 @@ public TiDBSchema getSchema() { return schema; } + @Override + protected void updateSchema() throws SQLException { + setSchema(TiDBSchema.fromConnection(getConnection(), getDatabaseName())); + } + } private static int mapActions(TiDBGlobalState globalState, Action a) { From b9e84694b22fb4785c41be8ad52cd015db3dd9ae Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jul 2020 10:53:37 +0200 Subject: [PATCH 0002/1284] Automatically update the schema and log statements --- src/sqlancer/GlobalState.java | 10 +++ src/sqlancer/Main.java | 9 +++ src/sqlancer/QueryAdapter.java | 8 +++ src/sqlancer/StatementExecutor.java | 6 +- .../clickhouse/ClickHouseProvider.java | 36 +--------- .../gen/ClickHouseTableGenerator.java | 2 +- .../cockroachdb/CockroachDBProvider.java | 56 ++++------------ src/sqlancer/duckdb/DuckDBProvider.java | 42 +++--------- src/sqlancer/mariadb/MariaDBProvider.java | 64 ++++++------------ .../mariadb/gen/MariaDBTableGenerator.java | 2 +- src/sqlancer/mysql/MySQLGlobalState.java | 13 ++-- src/sqlancer/mysql/MySQLProvider.java | 19 +----- .../mysql/gen/MySQLTableGenerator.java | 4 +- .../postgres/PostgresGlobalState.java | 13 ++-- src/sqlancer/postgres/PostgresProvider.java | 37 ++-------- src/sqlancer/sqlite3/SQLite3Provider.java | 67 +++++++------------ ...Lite3CreateVirtualRtreeTabelGenerator.java | 2 +- ...SQLite3CreateVirtualFTSTableGenerator.java | 3 +- .../gen/ddl/SQLite3TableGenerator.java | 2 +- src/sqlancer/tidb/TiDBProvider.java | 65 ++++++------------ 20 files changed, 147 insertions(+), 313 deletions(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index cc8c334a8..38b6561bf 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -90,4 +90,14 @@ public void setDatabaseName(String databaseName) { protected abstract void updateSchema() throws SQLException; + public boolean executeStatement(Query q) throws SQLException { + if (getOptions().logEachSelect()) { + getLogger().writeCurrent(q.getQueryString()); + } + boolean success = manager.execute(q); + if (q.couldAffectSchema()) { + updateSchema(); + } + return success; + } } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index ae8f874b7..b5b06e359 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -320,7 +320,16 @@ public void run() throws SQLException { state.setConnection(con); state.setStateLogger(logger); state.setManager(manager); + if (options.logEachSelect()) { + logger.writeCurrent(state.getState()); + } provider.generateAndTestDatabase(state); + try { + logger.getCurrentFileWriter().close(); + logger.currentFileWriter = null; + } catch (IOException e) { + throw new AssertionError(e); + } } } diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index 0f8cb9ef2..6a8ad505e 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -24,12 +24,20 @@ public QueryAdapter(String query, Collection expectedErrors) { this.query = query; this.expectedErrors = expectedErrors; this.couldAffectSchema = false; + checkQueryString(); } public QueryAdapter(String query, Collection expectedErrors, boolean couldAffectSchema) { this.query = query; this.expectedErrors = expectedErrors; this.couldAffectSchema = couldAffectSchema; + checkQueryString(); + } + + private void checkQueryString() { + if (query.contains("CREATE TABLE") && !couldAffectSchema) { + throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); + } } @Override diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index caf7c5201..3aa9defa5 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -64,15 +64,13 @@ public void executeStatements() throws SQLException { int nrTries = 0; do { query = nextAction.getQuery(globalState); - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(query.getQueryString()); - } - success = globalState.getManager().execute(query); + success = globalState.executeStatement(query); } while (!success && nrTries++ < globalState.getOptions().getNrStatementRetryCount()); } catch (IgnoreMeException e) { } if (query != null && query.couldAffectSchema()) { + globalState.updateSchema(); queryConsumer.notify(query); } total--; diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 82fd1749c..d46307a02 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -1,6 +1,5 @@ package sqlancer.clickhouse; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -13,7 +12,6 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -62,10 +60,6 @@ public static class ClickHouseGlobalState extends GlobalState private ClickHouseSchema schema; private ClickHouseOptions clickHouseOptions; - public void setSchema(ClickHouseSchema schema) { - this.schema = schema; - } - public ClickHouseSchema getSchema() { return schema; } @@ -90,40 +84,23 @@ public String getDatabaseName() { @Override protected void updateSchema() throws SQLException { - setSchema(ClickHouseSchema.fromConnection(getConnection(), getDatabaseName())); + this.schema = ClickHouseSchema.fromConnection(getConnection(), getDatabaseName()); } } @Override public void generateAndTestDatabase(ClickHouseGlobalState globalState) throws SQLException { - StateLogger logger = globalState.getLogger(); QueryManager manager = globalState.getManager(); - globalState - .setSchema(ClickHouseSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName())); for (int i = 0; i < Randomly.fromOptions(1); i++) { boolean success = false; do { Query qt = new ClickHouseTableGenerator().getQuery(globalState); - success = manager.execute(qt); - logger.writeCurrent(globalState.getState()); - globalState.setSchema( - ClickHouseSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName())); - try { - logger.getCurrentFileWriter().close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - logger.currentFileWriter = null; + success = globalState.executeStatement(qt); } while (!success); } StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), ClickHouseProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - globalState.setSchema(ClickHouseSchema.fromConnection(globalState.getConnection(), - globalState.getDatabaseName())); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } @@ -149,15 +126,6 @@ public void generateAndTestDatabase(ClickHouseGlobalState globalState) throws SQ manager.incrementSelectQueryCount(); } - try { - if (globalState.getOptions().logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } } @Override diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index a2f355174..d597c7e47 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -46,7 +46,7 @@ public Query getQuery(ClickHouseGlobalState globalState) { sb.append(" ORDER BY tuple()"); } sb.append(";"); - return new QueryAdapter(sb.toString(), errors); + return new QueryAdapter(sb.toString(), errors, true); } private void potentiallyAppendCodec() { diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index afa02f379..d5bc6600d 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -1,6 +1,5 @@ package sqlancer.cockroachdb; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -15,14 +14,12 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.StateToReproduce; import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; @@ -119,17 +116,20 @@ public static class CockroachDBGlobalState extends GlobalState standardSettings = new ArrayList<>(); standardSettings.add("--Don't send automatic bug reports\n" + "SET CLUSTER SETTING debug.panic_on_failed_assertions = true;"); @@ -172,22 +166,12 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S do { try { Query q = CockroachDBTableGenerator.generate(globalState); - success = manager.execute(q); - logger.writeCurrent(state); - try { - logger.getCurrentFileWriter().close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - logger.currentFileWriter = null; + success = globalState.executeStatement(q); } catch (IgnoreMeException e) { // continue trying } } while (!success); - globalState.setSchema(CockroachDBSchema.fromConnection(con, databaseName)); } - logger.writeCurrent(state); int[] nrRemaining = new int[Action.values().length]; List actions = new ArrayList<>(); @@ -263,19 +247,13 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S int nrTries = 0; do { query = nextAction.getQuery(globalState); - if (options.logEachSelect()) { - logger.writeCurrent(query.getQueryString()); - } - success = manager.execute(query); + success = globalState.executeStatement(query); } while (!success && nrTries++ < 1000); } catch (IgnoreMeException e) { } - if (query != null && query.couldAffectSchema()) { - globalState.setSchema(CockroachDBSchema.fromConnection(con, databaseName)); - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } + if (query != null && query.couldAffectSchema() && globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); } total--; } @@ -292,16 +270,6 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S } } - try { - if (options.logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } @Override diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index cdb62ada0..f1ec580d0 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -1,6 +1,5 @@ package sqlancer.duckdb; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -13,7 +12,6 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -96,50 +94,39 @@ public static class DuckDBGlobalState extends GlobalState { private DuckDBSchema schema; - public void setSchema(DuckDBSchema schema) { - this.schema = schema; - } - public DuckDBSchema getSchema() { + if (schema == null) { + try { + updateSchema(); + } catch (SQLException e) { + throw new AssertionError(e); + } + } return schema; } @Override protected void updateSchema() throws SQLException { - setSchema(DuckDBSchema.fromConnection(getConnection(), getDatabaseName())); + this.schema = DuckDBSchema.fromConnection(getConnection(), getDatabaseName()); } } @Override public void generateAndTestDatabase(DuckDBGlobalState globalState) throws SQLException { - StateLogger logger = globalState.getLogger(); QueryManager manager = globalState.getManager(); - globalState.setSchema(DuckDBSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName())); - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(globalState.getState()); - } for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { boolean success = false; do { Query qt = new DuckDBTableGenerator().getQuery(globalState); - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(qt.getQueryString()); - } - success = manager.execute(qt); - globalState.setSchema( - DuckDBSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName())); + success = globalState.executeStatement(qt); } while (!success); } - if (globalState.getSchema().getDatabaseTables().size() == 0) { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); // TODO } StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), DuckDBProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - globalState.setSchema(DuckDBSchema.fromConnection(globalState.getConnection(), - globalState.getDatabaseName())); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } @@ -163,15 +150,6 @@ public void generateAndTestDatabase(DuckDBGlobalState globalState) throws SQLExc } } - try { - if (globalState.getOptions().logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } globalState.getConnection().close(); } diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index e76205abf..21cadb795 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -10,13 +10,11 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.StateToReproduce; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.gen.MariaDBIndexGenerator; import sqlancer.mariadb.gen.MariaDBInsertGenerator; @@ -55,25 +53,12 @@ enum Action { public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLException { this.databaseName = globalState.getDatabaseName(); MainOptions options = globalState.getOptions(); - Connection con = globalState.getConnection(); - StateLogger logger = globalState.getLogger(); - StateToReproduce state = globalState.getState(); QueryManager manager = globalState.getManager(); - MariaDBSchema newSchema = MariaDBSchema.fromConnection(con, databaseName); - globalState.setSchema(newSchema); - if (options.logEachSelect()) { - logger.writeCurrent(state); - } - while (newSchema.getDatabaseTables().size() < Randomly.smallNumber() + 1) { - String tableName = SQLite3Common.createTableName(newSchema.getDatabaseTables().size()); - Query createTable = MariaDBTableGenerator.generate(tableName, r, newSchema); - if (options.logEachSelect()) { - logger.writeCurrent(createTable.getQueryString()); - } - manager.execute(createTable); - newSchema = MariaDBSchema.fromConnection(con, databaseName); - globalState.setSchema(newSchema); + while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); + Query createTable = MariaDBTableGenerator.generate(tableName, r, globalState.getSchema()); + globalState.executeStatement(createTable); } int[] nrRemaining = new int[Action.values().length]; @@ -127,31 +112,31 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx try { switch (nextAction) { case CHECKSUM: - query = MariaDBTableAdminCommandGenerator.checksumTable(newSchema); + query = MariaDBTableAdminCommandGenerator.checksumTable(globalState.getSchema()); break; case CHECK_TABLE: - query = MariaDBTableAdminCommandGenerator.checkTable(newSchema); + query = MariaDBTableAdminCommandGenerator.checkTable(globalState.getSchema()); break; case TRUNCATE: - query = MariaDBTruncateGenerator.truncate(newSchema); + query = MariaDBTruncateGenerator.truncate(globalState.getSchema()); break; case REPAIR_TABLE: - query = MariaDBTableAdminCommandGenerator.repairTable(newSchema); + query = MariaDBTableAdminCommandGenerator.repairTable(globalState.getSchema()); break; case INSERT: - query = MariaDBInsertGenerator.insert(newSchema, r); + query = MariaDBInsertGenerator.insert(globalState.getSchema(), r); break; case OPTIMIZE: - query = MariaDBTableAdminCommandGenerator.optimizeTable(newSchema); + query = MariaDBTableAdminCommandGenerator.optimizeTable(globalState.getSchema()); break; case ANALYZE_TABLE: - query = MariaDBTableAdminCommandGenerator.analyzeTable(newSchema); + query = MariaDBTableAdminCommandGenerator.analyzeTable(globalState.getSchema()); break; case UPDATE: - query = MariaDBUpdateGenerator.update(newSchema, r); + query = MariaDBUpdateGenerator.update(globalState.getSchema(), r); break; case CREATE_INDEX: - query = MariaDBIndexGenerator.generate(newSchema); + query = MariaDBIndexGenerator.generate(globalState.getSchema()); break; case SET: query = MariaDBSetGenerator.set(r, options); @@ -164,21 +149,13 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx continue; } try { - if (options.logEachSelect()) { - logger.writeCurrent(query.getQueryString()); - } - manager.execute(query); - if (query.couldAffectSchema()) { - newSchema = MariaDBSchema.fromConnection(con, databaseName); - globalState.setSchema(newSchema); - } + globalState.executeStatement(query); } catch (Throwable t) { System.err.println(query.getQueryString()); throw t; } total--; } - newSchema = MariaDBSchema.fromConnection(con, databaseName); // MariaDBNoRECOracle queryGenerator = new MariaDBNoRECOracle(globalState); for (int i = 0; i < options.getNrQueries(); i++) { @@ -196,17 +173,20 @@ public static class MariaDBGlobalState extends GlobalState { private MariaDBSchema schema; - public void setSchema(MariaDBSchema schema) { - this.schema = schema; - } - public MariaDBSchema getSchema() { + if (schema == null) { + try { + updateSchema(); + } catch (SQLException e) { + throw new AssertionError(e); + } + } return schema; } @Override protected void updateSchema() throws SQLException { - setSchema(MariaDBSchema.fromConnection(getConnection(), getDatabaseName())); + this.schema = MariaDBSchema.fromConnection(getConnection(), getDatabaseName()); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java index 9c6690074..88ca4893b 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java @@ -40,7 +40,7 @@ private Query gen() { } else { likeOtherTable(); } - return new QueryAdapter(sb.toString(), errors); + return new QueryAdapter(sb.toString(), errors, true); } private enum PrimaryKeyState { diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index f8eff8171..9bfd38c5c 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -9,17 +9,20 @@ public class MySQLGlobalState extends GlobalState { private MySQLSchema schema; - public void setSchema(MySQLSchema schema) { - this.schema = schema; - } - public MySQLSchema getSchema() { + if (schema == null) { + try { + updateSchema(); + } catch (SQLException e) { + throw new AssertionError(); + } + } return schema; } @Override protected void updateSchema() throws SQLException { - setSchema(MySQLSchema.fromConnection(getConnection(), getDatabaseName())); + this.schema = MySQLSchema.fromConnection(getConnection(), getDatabaseName()); } } diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index ea097bf67..7604a8528 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -13,7 +13,6 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; @@ -151,31 +150,17 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { public void generateAndTestDatabase(MySQLGlobalState globalState) throws SQLException { this.databaseName = globalState.getDatabaseName(); this.manager = globalState.getManager(); - Connection con = globalState.getConnection(); MainOptions options = globalState.getOptions(); - StateLogger logger = globalState.getLogger(); - StateToReproduce state = globalState.getState(); Randomly r = globalState.getRandomly(); - globalState.setSchema(MySQLSchema.fromConnection(con, databaseName)); - if (options.logEachSelect()) { - logger.writeCurrent(state); - } while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); Query createTable = MySQLTableGenerator.generate(tableName, r, globalState.getSchema()); - if (options.logEachSelect()) { - logger.writeCurrent(createTable.getQueryString()); - } - manager.execute(createTable); - globalState.setSchema(MySQLSchema.fromConnection(con, databaseName)); + globalState.executeStatement(createTable); } StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), MySQLProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - globalState.setSchema(MySQLSchema.fromConnection(con, databaseName)); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } @@ -189,8 +174,6 @@ public void generateAndTestDatabase(MySQLGlobalState globalState) throws SQLExce // } // } - globalState.setSchema(MySQLSchema.fromConnection(con, databaseName)); - TestOracle oracle = new MySQLTLPWhereOracle(globalState); for (int i = 0; i < options.getNrQueries(); i++) { try { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index c67a805ad..33ec48fa3 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -54,7 +54,7 @@ private Query create() { if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { sb.append(" LIKE "); sb.append(schema.getRandomTable().getName()); - return new QueryAdapter(sb.toString()); + return new QueryAdapter(sb.toString(), true); } else { sb.append("("); for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { @@ -77,7 +77,7 @@ private Query create() { errors.add("Table handler doesn't support NULL in given index"); errors.add("Got error -1 - 'Unknown error -1' from storage engine"); addCommonErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new QueryAdapter(sb.toString(), errors, true); } addCommonErrors(errors); return new QueryAdapter(sb.toString(), errors, true); diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 332c58d44..117acca38 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -29,11 +29,14 @@ public void setConnection(Connection con) { } } - public void setSchema(PostgresSchema schema) { - this.schema = schema; - } - public PostgresSchema getSchema() { + if (schema == null) { + try { + updateSchema(); + } catch (SQLException e) { + throw new AssertionError(); + } + } return schema; } @@ -100,7 +103,7 @@ public String getRandomOpclass() { @Override protected void updateSchema() throws SQLException { - setSchema(PostgresSchema.fromConnection(getConnection(), getDatabaseName())); + this.schema = PostgresSchema.fromConnection(getConnection(), getDatabaseName()); } } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 192aa4cb3..be4730cc9 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -15,7 +15,6 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; @@ -72,21 +71,11 @@ public enum Action implements AbstractAction { COMMIT(g -> { Query query; if (Randomly.getBoolean()) { - query = new QueryAdapter("COMMIT") { - @Override - public boolean couldAffectSchema() { - return true; - } - }; + query = new QueryAdapter("COMMIT", true); } else if (Randomly.getBoolean()) { query = PostgresTransactionGenerator.executeBegin(); } else { - query = new QueryAdapter("ROLLBACK") { - @Override - public boolean couldAffectSchema() { - return true; - } - }; + query = new QueryAdapter("ROLLBACK", true); } return query; }), // @@ -197,25 +186,13 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { @Override public void generateAndTestDatabase(PostgresGlobalState globalState) throws SQLException { MainOptions options = globalState.getOptions(); - StateLogger logger = globalState.getLogger(); - StateToReproduce state = globalState.getState(); - String databaseName = globalState.getDatabaseName(); - Connection con = globalState.getConnection(); QueryManager manager = globalState.getManager(); - if (options.logEachSelect()) { - logger.writeCurrent(state); - } - globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); - while (globalState.getSchema().getDatabaseTables().size() < 1) { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { try { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); Query createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), generateOnlyKnown, globalState); - if (options.logEachSelect()) { - logger.writeCurrent(createTable.getQueryString()); - } - manager.execute(createTable); - globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); + globalState.executeStatement(createTable); } catch (IgnoreMeException e) { } @@ -223,9 +200,6 @@ public void generateAndTestDatabase(PostgresGlobalState globalState) throws SQLE StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), PostgresProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } @@ -242,8 +216,7 @@ public void generateAndTestDatabase(PostgresGlobalState globalState) throws SQLE // } se.executeStatements(); manager.incrementCreateDatabase(); - manager.execute(new QueryAdapter("COMMIT")); - globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); + manager.execute(new QueryAdapter("COMMIT", true)); manager.execute(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 60033a2b7..6246cb685 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -2,7 +2,6 @@ import java.io.File; import java.io.FileWriter; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -15,7 +14,6 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; @@ -166,13 +164,16 @@ public static class SQLite3GlobalState extends GlobalState { private SQLite3Options sqliteOptions; public SQLite3Schema getSchema() { + if (schema == null) { + try { + updateSchema(); + } catch (SQLException e) { + throw new AssertionError(e); + } + } return schema; } - public void setSchema(SQLite3Schema schema) { - this.schema = schema; - } - public void setSqliteOptions(SQLite3Options sqliteOptions) { this.sqliteOptions = sqliteOptions; } @@ -183,7 +184,7 @@ public SQLite3Options getSqliteOptions() { @Override protected void updateSchema() throws SQLException { - setSchema(SQLite3Schema.fromConnection(this)); + this.schema = SQLite3Schema.fromConnection(this); } } @@ -202,7 +203,6 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx Randomly r = new Randomly(SQLite3SpecialStringGenerator::generate); globalState.setSqliteOptions(sqliteOptions); globalState.setRandomly(r); - StateLogger logger = globalState.getLogger(); this.state = (SQLite3StateToReproduce) globalState.getState(); globalState.setState(state); if (globalState.getDmbsSpecificOptions().generateDatabase) { @@ -217,19 +217,18 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx } int i = 0; - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); do { Query tableQuery = getTableQuery(r, i++); - executeStatement(globalState, manager, tableQuery); - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); - } while (globalState.getSchema().getDatabaseTables().size() != nrTablesToCreate); + globalState.executeStatement(tableQuery); + logStatement(globalState, tableQuery); + } while (globalState.getSchema().getDatabaseTables().size() < nrTablesToCreate); assert globalState.getSchema().getTables().getTables().size() == nrTablesToCreate; checkTablesForGeneratedColumnLoops(globalState); if (globalState.getDmbsSpecificOptions().testDBStats && Randomly.getBooleanWithSmallProbability()) { QueryAdapter tableQuery = new QueryAdapter( "CREATE VIRTUAL TABLE IF NOT EXISTS stat USING dbstat(main)"); - executeStatement(globalState, manager, tableQuery); - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); + globalState.executeStatement(tableQuery); + logStatement(globalState, tableQuery); } int[] nrRemaining = new int[Action.values().length]; List actions = new ArrayList<>(); @@ -290,10 +289,6 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx total += nrPerformed; } - if (options.logEachSelect()) { - logger.writeCurrent(state); - } - while (total != 0) { Action nextAction = null; int selection = r.getInteger(0, total); @@ -312,31 +307,26 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx Query query = null; try { query = nextAction.getQuery(globalState); - if (options.logEachSelect()) { - logger.writeCurrent(query.getQueryString()); - } - executeStatement(globalState, manager, query); + globalState.executeStatement(query); + logStatement(globalState, query); } catch (IgnoreMeException e) { } - if (query != null && query.couldAffectSchema()) { - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } + if (query != null && query.couldAffectSchema() + && globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); } total--; } Query query = SQLite3TransactionGenerator.generateCommit(globalState); - executeStatement(globalState, manager, query); + globalState.executeStatement(query); + logStatement(globalState, query); // also do an abort for DEFERRABLE INITIALLY DEFERRED query = SQLite3TransactionGenerator.generateRollbackTransaction(globalState); - executeStatement(globalState, manager, query); - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); + globalState.executeStatement(query); + logStatement(globalState, query); manager.incrementCreateDatabase(); - } else { - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); } TestOracle oracle = globalState.getSqliteOptions().oracle.create(globalState); if (oracle.onlyWorksForNonEmptyTables()) { @@ -355,23 +345,12 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx } } - try { - if (options.logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } if (globalState.getDmbsSpecificOptions().exitAfterFirstDatabase) { System.exit(0); } } - private void executeStatement(SQLite3GlobalState globalState, QueryManager manager, Query tableQuery) - throws SQLException { - manager.execute(tableQuery); + private void logStatement(SQLite3GlobalState globalState, Query tableQuery) { if (globalState.getDmbsSpecificOptions().printStatements) { String s = tableQuery.getQueryString(); if (!s.endsWith(";")) { diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index c782eff41..31619e986 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -48,7 +48,7 @@ public static Query createTableStatement(String rTreeTableName, SQLite3GlobalSta errors.add("Wrong number of columns for an rtree table"); errors.add("Too many columns for an rtree table"); - return new QueryAdapter(sb.toString(), errors); + return new QueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java index 5afcbc415..b1d031918 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java @@ -45,7 +45,8 @@ public Query create() { } else { createFts5Table(); } - return new QueryAdapter(sb.toString(), Arrays.asList("unrecognized parameter", "unknown tokenizer: ascii")); + return new QueryAdapter(sb.toString(), Arrays.asList("unrecognized parameter", "unknown tokenizer: ascii"), + true); } private void createFts4Table() { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index 1bafef41f..72e8fe9e8 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -55,7 +55,7 @@ public static Query createTableStatement(String tableName, SQLite3GlobalState gl errors.add("parser stack overflow"); errors.add("malformed JSON"); errors.add("JSON cannot hold BLOB values"); - return new QueryAdapter(sqLite3TableGenerator.sb.toString(), errors); + return new QueryAdapter(sqLite3TableGenerator.sb.toString(), errors, true); } public void start() { diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index db1df95f6..395e08dff 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -1,6 +1,5 @@ package sqlancer.tidb; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -15,13 +14,11 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.StateToReproduce; import sqlancer.StatementExecutor; import sqlancer.TestOracle; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -79,17 +76,20 @@ public static class TiDBGlobalState extends GlobalState { private TiDBSchema schema; - public void setSchema(TiDBSchema schema) { - this.schema = schema; - } - public TiDBSchema getSchema() { + if (schema == null) { + try { + updateSchema(); + } catch (SQLException e) { + throw new AssertionError(); + } + } return schema; } @Override protected void updateSchema() throws SQLException { - setSchema(TiDBSchema.fromConnection(getConnection(), getDatabaseName())); + this.schema = TiDBSchema.fromConnection(getConnection(), getDatabaseName()); } } @@ -124,48 +124,30 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { @Override public void generateAndTestDatabase(TiDBGlobalState globalState) throws SQLException { QueryManager manager = globalState.getManager(); - Connection con = globalState.getConnection(); - String databaseName = globalState.getDatabaseName(); - globalState.setSchema(TiDBSchema.fromConnection(con, databaseName)); - StateLogger logger = globalState.getLogger(); - StateToReproduce state = globalState.getState(); for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { boolean success = false; do { Query qt = new TiDBTableGenerator().getQuery(globalState); - success = manager.execute(qt); - logger.writeCurrent(state); - globalState.setSchema(TiDBSchema.fromConnection(con, databaseName)); - try { - logger.getCurrentFileWriter().close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - logger.currentFileWriter = null; + success = globalState.executeStatement(qt); } while (!success); } - globalState.setSchema(TiDBSchema.fromConnection(con, databaseName)); StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), TiDBProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - try { - globalState.setSchema(TiDBSchema.fromConnection(con, databaseName)); - } catch (SQLException e) { - if (q.getQueryString().contains("CREATE VIEW") || e.getMessage().contains( - "references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them")) { - throw new IgnoreMeException(); // TODO: drop view instead - } else { - throw new AssertionError(e); - } - } - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } }); - se.executeStatements(); + try { + se.executeStatements(); + } catch (SQLException e) { + if (e.getMessage().contains( + "references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them")) { + throw new IgnoreMeException(); // TODO: drop view instead + } else { + throw new AssertionError(e); + } + } manager.incrementCreateDatabase(); List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { try { @@ -184,15 +166,6 @@ public void generateAndTestDatabase(TiDBGlobalState globalState) throws SQLExcep } } - try { - if (globalState.getOptions().logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } } From 916cb198df7c01e65c0f8620fcbd37c89df0de1b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jul 2020 11:17:32 +0200 Subject: [PATCH 0003/1284] Make the schema part of the global state base class --- src/sqlancer/ComparatorHelper.java | 8 +++---- src/sqlancer/DatabaseProvider.java | 2 +- src/sqlancer/GlobalState.java | 23 ++++++++++++++++--- src/sqlancer/Main.java | 8 +++---- src/sqlancer/ProviderAdapter.java | 2 +- src/sqlancer/Query.java | 10 ++++---- src/sqlancer/QueryAdapter.java | 4 ++-- src/sqlancer/QueryResultCheckAdapter.java | 2 +- src/sqlancer/StatementExecutor.java | 2 +- .../clickhouse/ClickHouseProvider.java | 9 ++------ .../cockroachdb/CockroachDBProvider.java | 17 ++------------ .../oracle/CockroachDBNoRECOracle.java | 2 +- src/sqlancer/duckdb/DuckDBProvider.java | 17 ++------------ src/sqlancer/mariadb/MariaDBProvider.java | 17 ++------------ src/sqlancer/mysql/MySQLGlobalState.java | 17 ++------------ .../postgres/PostgresGlobalState.java | 16 ++----------- src/sqlancer/postgres/PostgresProvider.java | 3 +-- .../gen/PostgresQueryCatalogGenerator.java | 2 +- src/sqlancer/sqlite3/SQLite3Provider.java | 16 ++----------- src/sqlancer/tidb/TiDBProvider.java | 17 ++------------ 20 files changed, 58 insertions(+), 136 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 45d3b5d74..d850858ab 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -33,7 +33,7 @@ static boolean equals(double a, double b) { } public static List getResultSetFirstColumnAsString(String queryString, Set errors, - GlobalState state) throws SQLException { + GlobalState state) throws SQLException { if (state.getOptions().logEachSelect()) { // TODO: refactor me state.getLogger().writeCurrent(queryString); @@ -83,7 +83,7 @@ public static List getResultSetFirstColumnAsString(String queryString, S } public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - String originalQueryString, List combinedString, GlobalState state) { + String originalQueryString, List combinedString, GlobalState state) { if (resultSet.size() != secondResultSet.size()) { String queryFormatString = "%s; -- cardinality: %d"; String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); @@ -117,7 +117,7 @@ public static void assumeResultSetsAreEqual(List resultSet, List } public static List getCombinedResultSet(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, + String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, Set errors) throws SQLException { List secondResultSet; if (asUnion) { @@ -138,7 +138,7 @@ public static List getCombinedResultSet(String firstQueryString, String } public static List getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, + String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, Set errors) throws SQLException { String unionString; if (asUnion) { diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index ff323f8bf..9d45b30d2 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -4,7 +4,7 @@ import java.sql.Connection; import java.sql.SQLException; -public interface DatabaseProvider, O> { +public interface DatabaseProvider, O> { /** * Gets the the {@link GlobalState} class. diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 38b6561bf..9503f50d7 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -12,12 +12,13 @@ * @param * the option parameter. */ -public abstract class GlobalState { +public abstract class GlobalState { private Connection con; private Randomly r; private MainOptions options; private O dmbsSpecificOptions; + private S schema; private StateLogger logger; private StateToReproduce state; private QueryManager manager; @@ -88,8 +89,6 @@ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } - protected abstract void updateSchema() throws SQLException; - public boolean executeStatement(Query q) throws SQLException { if (getOptions().logEachSelect()) { getLogger().writeCurrent(q.getQueryString()); @@ -100,4 +99,22 @@ public boolean executeStatement(Query q) throws SQLException { } return success; } + + public S getSchema() { + if (schema == null) { + try { + updateSchema(); + } catch (SQLException e) { + throw new AssertionError(); + } + } + return schema; + } + + protected void setSchema(S schema) { + this.schema = schema; + } + + protected abstract void updateSchema() throws SQLException; + } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index b5b06e359..5229cb447 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -234,9 +234,9 @@ private void printState(FileWriter writer, StateToReproduce state) { public static class QueryManager { - private final GlobalState globalState; + private final GlobalState globalState; - QueryManager(GlobalState globalState) { + QueryManager(GlobalState globalState) { this.globalState = globalState; } @@ -267,7 +267,7 @@ public static void main(String[] args) { System.exit(executeMain(args)); } - public static class DBMSExecutor, O> { + public static class DBMSExecutor, O> { private final DatabaseProvider provider; private final MainOptions options; @@ -342,7 +342,7 @@ public StateToReproduce getStateToReproduce() { } } - public static class DBMSExecutorFactory, O> { + public static class DBMSExecutorFactory, O> { private final DatabaseProvider provider; private final MainOptions options; diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index fcc076a23..d23cc1721 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -2,7 +2,7 @@ import java.io.FileWriter; -public abstract class ProviderAdapter, O> implements DatabaseProvider { +public abstract class ProviderAdapter, O> implements DatabaseProvider { private final Class globalClass; private final Class optionClass; diff --git a/src/sqlancer/Query.java b/src/sqlancer/Query.java index 622f48035..abb3a5718 100644 --- a/src/sqlancer/Query.java +++ b/src/sqlancer/Query.java @@ -23,7 +23,7 @@ public abstract class Query { * * @throws SQLException */ - public abstract boolean execute(GlobalState globalState) throws SQLException; + public abstract boolean execute(GlobalState globalState) throws SQLException; public abstract Collection getExpectedErrors(); @@ -32,21 +32,21 @@ public String toString() { return getQueryString(); } - public ResultSet executeAndGet(GlobalState globalState) throws SQLException { + public ResultSet executeAndGet(GlobalState globalState) throws SQLException { throw new AssertionError(); } - public boolean executeLogged(GlobalState globalState) throws SQLException { + public boolean executeLogged(GlobalState globalState) throws SQLException { logQueryString(globalState); return execute(globalState); } - public ResultSet executeAndGetLogged(GlobalState globalState) throws SQLException { + public ResultSet executeAndGetLogged(GlobalState globalState) throws SQLException { logQueryString(globalState); return executeAndGet(globalState); } - private void logQueryString(GlobalState globalState) { + private void logQueryString(GlobalState globalState) { if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(getQueryString()); } diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index 6a8ad505e..8acb244b8 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -46,7 +46,7 @@ public String getQueryString() { } @Override - public boolean execute(GlobalState globalState) throws SQLException { + public boolean execute(GlobalState globalState) throws SQLException { try (Statement s = globalState.getConnection().createStatement()) { s.execute(query); Main.nrSuccessfulActions.addAndGet(1); @@ -72,7 +72,7 @@ public void checkException(Exception e) throws AssertionError { } @Override - public ResultSet executeAndGet(GlobalState globalState) throws SQLException { + public ResultSet executeAndGet(GlobalState globalState) throws SQLException { Statement s = globalState.getConnection().createStatement(); ResultSet result = null; try { diff --git a/src/sqlancer/QueryResultCheckAdapter.java b/src/sqlancer/QueryResultCheckAdapter.java index c8f294a02..e0c131ed8 100644 --- a/src/sqlancer/QueryResultCheckAdapter.java +++ b/src/sqlancer/QueryResultCheckAdapter.java @@ -15,7 +15,7 @@ public QueryResultCheckAdapter(String query, Consumer rsChecker) { } @Override - public boolean execute(GlobalState globalState) throws SQLException { + public boolean execute(GlobalState globalState) throws SQLException { try (Statement s = globalState.getConnection().createStatement()) { ResultSet rs = s.executeQuery(getQueryString()); rsChecker.accept(rs); diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index 3aa9defa5..a344a9286 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -4,7 +4,7 @@ import java.util.ArrayList; import java.util.List; -public class StatementExecutor, A extends AbstractAction> { +public class StatementExecutor, A extends AbstractAction> { private final G globalState; private final A[] actions; diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index d46307a02..678515538 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -55,15 +55,10 @@ private static int mapActions(ClickHouseGlobalState globalState, Action a) { } } - public static class ClickHouseGlobalState extends GlobalState { + public static class ClickHouseGlobalState extends GlobalState { - private ClickHouseSchema schema; private ClickHouseOptions clickHouseOptions; - public ClickHouseSchema getSchema() { - return schema; - } - public void setClickHouseOptions(ClickHouseOptions clickHouseOptions) { this.clickHouseOptions = clickHouseOptions; } @@ -84,7 +79,7 @@ public String getDatabaseName() { @Override protected void updateSchema() throws SQLException { - this.schema = ClickHouseSchema.fromConnection(getConnection(), getDatabaseName()); + setSchema(ClickHouseSchema.fromConnection(getConnection(), getDatabaseName())); } } diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index d5bc6600d..34f1c7cb6 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -112,24 +112,11 @@ public Query getQuery(CockroachDBGlobalState state) throws SQLException { } } - public static class CockroachDBGlobalState extends GlobalState { - - private CockroachDBSchema schema; - - public CockroachDBSchema getSchema() { - if (schema == null) { - try { - updateSchema(); - } catch (SQLException e) { - throw new AssertionError(); - } - } - return schema; - } + public static class CockroachDBGlobalState extends GlobalState { @Override protected void updateSchema() throws SQLException { - this.schema = CockroachDBSchema.fromConnection(getConnection(), getDatabaseName()); + setSchema(CockroachDBSchema.fromConnection(getConnection(), getDatabaseName())); } } diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 3babf7ee6..1a331232e 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -140,7 +140,7 @@ private int getNonOptimizedResult(CockroachDBExpression whereCondition, List globalState, Query q) throws AssertionError { + private int getCount(GlobalState globalState, Query q) throws AssertionError { int count = 0; try (ResultSet rs = q.executeAndGet(globalState)) { if (rs == null) { diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index f1ec580d0..eb89a79c8 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -90,24 +90,11 @@ private static int mapActions(DuckDBGlobalState globalState, Action a) { } } - public static class DuckDBGlobalState extends GlobalState { - - private DuckDBSchema schema; - - public DuckDBSchema getSchema() { - if (schema == null) { - try { - updateSchema(); - } catch (SQLException e) { - throw new AssertionError(e); - } - } - return schema; - } + public static class DuckDBGlobalState extends GlobalState { @Override protected void updateSchema() throws SQLException { - this.schema = DuckDBSchema.fromConnection(getConnection(), getDatabaseName()); + setSchema(DuckDBSchema.fromConnection(getConnection(), getDatabaseName())); } } diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 21cadb795..3abcfc9b6 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -169,24 +169,11 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx } - public static class MariaDBGlobalState extends GlobalState { - - private MariaDBSchema schema; - - public MariaDBSchema getSchema() { - if (schema == null) { - try { - updateSchema(); - } catch (SQLException e) { - throw new AssertionError(e); - } - } - return schema; - } + public static class MariaDBGlobalState extends GlobalState { @Override protected void updateSchema() throws SQLException { - this.schema = MariaDBSchema.fromConnection(getConnection(), getDatabaseName()); + setSchema(MariaDBSchema.fromConnection(getConnection(), getDatabaseName())); } } diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 9bfd38c5c..c464fb4de 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -5,24 +5,11 @@ import sqlancer.GlobalState; -public class MySQLGlobalState extends GlobalState { - - private MySQLSchema schema; - - public MySQLSchema getSchema() { - if (schema == null) { - try { - updateSchema(); - } catch (SQLException e) { - throw new AssertionError(); - } - } - return schema; - } +public class MySQLGlobalState extends GlobalState { @Override protected void updateSchema() throws SQLException { - this.schema = MySQLSchema.fromConnection(getConnection(), getDatabaseName()); + setSchema(MySQLSchema.fromConnection(getConnection(), getDatabaseName())); } } diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 117acca38..58e5be0a4 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -10,12 +10,11 @@ import sqlancer.GlobalState; import sqlancer.Randomly; -public class PostgresGlobalState extends GlobalState { +public class PostgresGlobalState extends GlobalState { private List operators; private List collates; private List opClasses; - private PostgresSchema schema; @Override public void setConnection(Connection con) { @@ -29,17 +28,6 @@ public void setConnection(Connection con) { } } - public PostgresSchema getSchema() { - if (schema == null) { - try { - updateSchema(); - } catch (SQLException e) { - throw new AssertionError(); - } - } - return schema; - } - private List getCollnames(Connection con) throws SQLException { List opClasses = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -103,7 +91,7 @@ public String getRandomOpclass() { @Override protected void updateSchema() throws SQLException { - this.schema = PostgresSchema.fromConnection(getConnection(), getDatabaseName()); + setSchema(PostgresSchema.fromConnection(getConnection(), getDatabaseName())); } } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index be4730cc9..03bca1cdd 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -12,7 +12,6 @@ import sqlancer.AbstractAction; import sqlancer.CompositeTestOracle; -import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; import sqlancer.MainOptions; @@ -263,7 +262,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce return con; } - private String getCreateDatabaseCommand(String databaseName, Connection con, GlobalState state) { + private String getCreateDatabaseCommand(String databaseName, Connection con, PostgresGlobalState state) { StringBuilder sb = new StringBuilder(); sb.append("CREATE DATABASE " + databaseName + " "); if (Randomly.getBoolean() && ((PostgresOptions) state.getDmbsSpecificOptions()).testCollations) { diff --git a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java index 2746cc814..6c2a60646 100644 --- a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java @@ -33,7 +33,7 @@ public static Query query() { "pg_ts_template", "pg_type", "pg_user_mapping")); return new QueryAdapter(sb.toString()) { @Override - public boolean execute(GlobalState globalState) throws SQLException { + public boolean execute(GlobalState globalState) throws SQLException { try (Statement s = globalState.getConnection().createStatement()) { try (ResultSet rs = s.executeQuery(getQueryString())) { // CHECKSTYLE:OFF diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 6246cb685..7aff5ebbc 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -158,22 +158,10 @@ public Query getQuery(SQLite3GlobalState state) throws SQLException { } } - public static class SQLite3GlobalState extends GlobalState { + public static class SQLite3GlobalState extends GlobalState { - private SQLite3Schema schema; private SQLite3Options sqliteOptions; - public SQLite3Schema getSchema() { - if (schema == null) { - try { - updateSchema(); - } catch (SQLException e) { - throw new AssertionError(e); - } - } - return schema; - } - public void setSqliteOptions(SQLite3Options sqliteOptions) { this.sqliteOptions = sqliteOptions; } @@ -184,7 +172,7 @@ public SQLite3Options getSqliteOptions() { @Override protected void updateSchema() throws SQLException { - this.schema = SQLite3Schema.fromConnection(this); + setSchema(SQLite3Schema.fromConnection(this)); } } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 395e08dff..7b91541ae 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -72,24 +72,11 @@ public Query getQuery(TiDBGlobalState state) throws SQLException { } } - public static class TiDBGlobalState extends GlobalState { - - private TiDBSchema schema; - - public TiDBSchema getSchema() { - if (schema == null) { - try { - updateSchema(); - } catch (SQLException e) { - throw new AssertionError(); - } - } - return schema; - } + public static class TiDBGlobalState extends GlobalState { @Override protected void updateSchema() throws SQLException { - this.schema = TiDBSchema.fromConnection(getConnection(), getDatabaseName()); + setSchema(TiDBSchema.fromConnection(getConnection(), getDatabaseName())); } } From d540175f809f976ce744104259d1332314ec6681 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jul 2020 14:08:04 +0200 Subject: [PATCH 0004/1284] Log the execution time of statements --- src/sqlancer/ExecutionTimer.java | 23 +++++++++++++++++++++++ src/sqlancer/GlobalState.java | 14 +++++++++++++- src/sqlancer/Main.java | 19 ++++++++++++++++--- src/sqlancer/MainOptions.java | 10 ++++++++++ 4 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 src/sqlancer/ExecutionTimer.java diff --git a/src/sqlancer/ExecutionTimer.java b/src/sqlancer/ExecutionTimer.java new file mode 100644 index 000000000..3d88697b3 --- /dev/null +++ b/src/sqlancer/ExecutionTimer.java @@ -0,0 +1,23 @@ +package sqlancer; + +public final class ExecutionTimer { + + private long startTime; + private long endTime; + + public ExecutionTimer start() { + startTime = System.currentTimeMillis(); + return this; + } + + public ExecutionTimer end() { + endTime = System.currentTimeMillis(); + return this; + } + + public String asString() { + long timeMillis = endTime - startTime; + return timeMillis + "ms"; + } + +} diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 9503f50d7..a110c0258 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -90,10 +90,22 @@ public void setDatabaseName(String databaseName) { } public boolean executeStatement(Query q) throws SQLException { + boolean logExecutionTime = getOptions().logExecutionTime(); + ExecutionTimer timer = null; + if (logExecutionTime) { + timer = new ExecutionTimer().start(); + } if (getOptions().logEachSelect()) { - getLogger().writeCurrent(q.getQueryString()); + if (logExecutionTime) { + getLogger().writeCurrentNoLineBreak(q.getQueryString()); + } else { + getLogger().writeCurrent(q.getQueryString()); + } } boolean success = manager.execute(q); + if (logExecutionTime) { + getLogger().writeCurrent(" -- " + timer.end().asString()); + } if (q.couldAffectSchema()) { updateSchema(); } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 5229cb447..12fe24c6a 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -157,18 +157,31 @@ public void writeCurrent(StateToReproduce state) { } public void writeCurrent(String queryString) { + write(queryString, "\n"); + } + + private void write(String queryString, String suffix) { if (!logEachSelect) { throw new UnsupportedOperationException(); } try { - getCurrentFileWriter().write(queryString + ";\n"); + getCurrentFileWriter().write(queryString); + if (!queryString.endsWith(";")) { + getCurrentFileWriter().write(';'); + } + if (suffix != null && suffix.length() != 0) { + getCurrentFileWriter().write(suffix); + } currentFileWriter.flush(); } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + throw new AssertionError(); } } + public void writeCurrentNoLineBreak(String queryString) { + write(queryString, ""); + } + public void logRowNotFound(StateToReproduce state) { printState(getLogFileWriter(), state); try { diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 6eeac6440..607e6c004 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -36,6 +36,9 @@ public class MainOptions { @Parameter(names = "--log-each-select", description = "Logs every statement issued", arity = 1) private boolean logEachSelect = true; // NOPMD + @Parameter(names = "--log-execution-time", description = "Logs the execution time of each statement (requires --log-each-select to be enabled)", arity = 1) + private boolean logExecutionTime = true; // NOPMD + @Parameter(names = "--username", description = "The user name used to log into the DBMS") private String userName = "sqlancer"; // NOPMD @@ -67,6 +70,13 @@ public boolean logEachSelect() { return logEachSelect; } + public boolean logExecutionTime() { + if (!logEachSelect) { + throw new AssertionError(); + } + return logExecutionTime; + } + public int getNrQueries() { return nrQueries; } From 1ff7497acc8b24d922b4b6ee8c1a2e596a9f7893 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jul 2020 14:54:16 +0200 Subject: [PATCH 0005/1284] Make the print statement options accessible to all DBMS --- src/sqlancer/GlobalState.java | 6 +++++ src/sqlancer/MainOptions.java | 20 +++++++++++++++++ src/sqlancer/QueryAdapter.java | 15 +++++++++++-- src/sqlancer/sqlite3/SQLite3Options.java | 4 ---- src/sqlancer/sqlite3/SQLite3Provider.java | 22 +------------------ .../sqlite3/oracle/SQLite3Fuzzer.java | 13 +---------- 6 files changed, 41 insertions(+), 39 deletions(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index a110c0258..0f93ca3a2 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -95,6 +95,9 @@ public boolean executeStatement(Query q) throws SQLException { if (logExecutionTime) { timer = new ExecutionTimer().start(); } + if (getOptions().printAllStatements()) { + System.out.println(q.getQueryString()); + } if (getOptions().logEachSelect()) { if (logExecutionTime) { getLogger().writeCurrentNoLineBreak(q.getQueryString()); @@ -103,6 +106,9 @@ public boolean executeStatement(Query q) throws SQLException { } } boolean success = manager.execute(q); + if (success && getOptions().printSucceedingStatements()) { + System.out.println(q.getQueryString()); + } if (logExecutionTime) { getLogger().writeCurrent(" -- " + timer.end().asString()); } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 607e6c004..2786e1add 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -54,6 +54,12 @@ public class MainOptions { @Parameter(names = "--exit-code-error", description = "The exit code that should be returned when an error is encountered (or a bug is found)") private int errorExitCode = -1; // NOPMD + @Parameter(names = "--print-statements", description = "Print all statements to stdout, before they are sent to the DBMS (not yet implemented for all oracles)", arity = 1) + private boolean printStatements; // NOPMD + + @Parameter(names = "--print-succeeding-statements", description = "Print statements that are successfully processed by the DBMS to stdout (not yet implemented for all oracles)", arity = 1) + private boolean printSucceedingStatements; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -70,6 +76,20 @@ public boolean logEachSelect() { return logEachSelect; } + public boolean printAllStatements() { + if (printSucceedingStatements && printStatements) { + throw new AssertionError(); + } + return printStatements; + } + + public boolean printSucceedingStatements() { + if (printStatements && printSucceedingStatements) { + throw new AssertionError(); + } + return printSucceedingStatements; + } + public boolean logExecutionTime() { if (!logEachSelect) { throw new AssertionError(); diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index 8acb244b8..0853979aa 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -21,19 +21,30 @@ public QueryAdapter(String query, boolean couldAffectSchema) { } public QueryAdapter(String query, Collection expectedErrors) { - this.query = query; + this.query = canonicalizeString(query); this.expectedErrors = expectedErrors; this.couldAffectSchema = false; checkQueryString(); } public QueryAdapter(String query, Collection expectedErrors, boolean couldAffectSchema) { - this.query = query; + this.query = canonicalizeString(query); this.expectedErrors = expectedErrors; this.couldAffectSchema = couldAffectSchema; checkQueryString(); } + private String canonicalizeString(String s) { + if (s.endsWith(";")) { + return s; + } else if (!s.contains("--")) { + return s + ";"; + } else { + // query contains a comment + return s; + } + } + private void checkQueryString() { if (query.contains("CREATE TABLE") && !couldAffectSchema) { throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index f570da048..b42ff89fd 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -81,10 +81,6 @@ public class SQLite3Options { "--generate-new-database" }, description = "Specifies whether new databases should be generated", arity = 1) public boolean generateDatabase = true; - @Parameter(names = { - "--print-statements" }, description = "Specifies whether to print SQL statements to stdout", arity = 1) - public boolean printStatements; - @Parameter(names = { "--execute-queries" }, description = "Specifies whether the query in the fuzzer should be executed", arity = 1) public boolean executeQuery = true; diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 7aff5ebbc..9c2438094 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -208,7 +208,6 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx do { Query tableQuery = getTableQuery(r, i++); globalState.executeStatement(tableQuery); - logStatement(globalState, tableQuery); } while (globalState.getSchema().getDatabaseTables().size() < nrTablesToCreate); assert globalState.getSchema().getTables().getTables().size() == nrTablesToCreate; checkTablesForGeneratedColumnLoops(globalState); @@ -216,7 +215,6 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx QueryAdapter tableQuery = new QueryAdapter( "CREATE VIRTUAL TABLE IF NOT EXISTS stat USING dbstat(main)"); globalState.executeStatement(tableQuery); - logStatement(globalState, tableQuery); } int[] nrRemaining = new int[Action.values().length]; List actions = new ArrayList<>(); @@ -296,7 +294,6 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx try { query = nextAction.getQuery(globalState); globalState.executeStatement(query); - logStatement(globalState, query); } catch (IgnoreMeException e) { } @@ -308,12 +305,10 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx } Query query = SQLite3TransactionGenerator.generateCommit(globalState); globalState.executeStatement(query); - logStatement(globalState, query); // also do an abort for DEFERRABLE INITIALLY DEFERRED query = SQLite3TransactionGenerator.generateRollbackTransaction(globalState); globalState.executeStatement(query); - logStatement(globalState, query); manager.incrementCreateDatabase(); } TestOracle oracle = globalState.getSqliteOptions().oracle.create(globalState); @@ -338,16 +333,6 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx } } - private void logStatement(SQLite3GlobalState globalState, Query tableQuery) { - if (globalState.getDmbsSpecificOptions().printStatements) { - String s = tableQuery.getQueryString(); - if (!s.endsWith(";")) { - s = s + ";"; - } - System.out.println(s); - } - } - private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws SQLException { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { Query q = new QueryAdapter("SELECT * FROM " + table.getName(), @@ -401,12 +386,7 @@ private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws SQ Randomly.fromOptions("UTF-8", "UTF-16", "UTF-16le", "UTF-16be"))); } for (String s : pragmasToExecute) { - if (globalState.getDmbsSpecificOptions().printStatements) { - System.out.println(s); - } - Query q = new QueryAdapter(s); - state.statements.add(q); - q.execute(globalState); + globalState.executeStatement(new QueryAdapter(s)); } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index 57190ce08..a47fa4aed 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -2,7 +2,6 @@ import java.sql.SQLException; -import sqlancer.MainOptions; import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.TestOracle; @@ -22,19 +21,9 @@ public SQLite3Fuzzer(SQLite3GlobalState globalState) { public void check() throws SQLException { String s = SQLite3Visitor .asString(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + ";"; - MainOptions options = globalState.getOptions(); try { - if (options.logEachSelect()) { - globalState.getLogger().writeCurrent(s); - } - if (globalState.getDmbsSpecificOptions().printStatements) { - System.out.println(s); - } if (globalState.getDmbsSpecificOptions().executeQuery) { - globalState.getManager().execute(new QueryAdapter(s)); - if (globalState.getDmbsSpecificOptions().executeStatementsAndPrintSuccessfulOnes) { - System.out.println(s); - } + globalState.executeStatement(new QueryAdapter(s)); globalState.getManager().incrementSelectQueryCount(); } } catch (Error e) { From 3306002154ebbe4ed69c73be374cb13192855c49 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jul 2020 15:18:13 +0200 Subject: [PATCH 0006/1284] Make the SQLite3 option to exit after a number of created databases accessible to all DBMS --- src/sqlancer/Main.java | 63 ++++++++++++++--------- src/sqlancer/MainOptions.java | 7 +++ src/sqlancer/sqlite3/SQLite3Options.java | 8 --- src/sqlancer/sqlite3/SQLite3Provider.java | 3 -- 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 12fe24c6a..1295ac990 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -437,33 +437,48 @@ public void run() { } private void runThread(final String databaseName) { - while (true) { - DBMSExecutor executor = executorFactory.getDBMSExecutor(databaseName, seed); + try { + if (options.getMaxGeneratedDatabases() == -1) { + // run without a limit + while (true) { + run(options, execService, executorFactory, seed, databaseName); + } + } else { + for (int i = 0; i < options.getMaxGeneratedDatabases(); i++) { + run(options, execService, executorFactory, seed, databaseName); + } + } + } finally { + threadsShutdown++; + if (threadsShutdown == options.getTotalNumberTries()) { + execService.shutdown(); + } + } + } + + private void run(MainOptions options, ExecutorService execService, + DBMSExecutorFactory executorFactory, final long seed, final String databaseName) { + DBMSExecutor executor = executorFactory.getDBMSExecutor(databaseName, seed); + try { + executor.run(); + } catch (IgnoreMeException e) { + return; + } catch (Throwable reduce) { + reduce.printStackTrace(); + executor.getStateToReproduce().exception = reduce.getMessage(); + executor.getLogger().logFileWriter = null; + executor.getLogger().logException(reduce, executor.getStateToReproduce()); + return; + } finally { try { - executor.run(); - } catch (IgnoreMeException e) { - continue; - } catch (Throwable reduce) { - reduce.printStackTrace(); - executor.getStateToReproduce().exception = reduce.getMessage(); - executor.getLogger().logFileWriter = null; - executor.getLogger().logException(reduce, executor.getStateToReproduce()); - threadsShutdown++; - break; - } finally { - try { - if (options.logEachSelect()) { - if (executor.getLogger().currentFileWriter != null) { - executor.getLogger().currentFileWriter.close(); - } - executor.getLogger().currentFileWriter = null; + if (options.logEachSelect()) { + if (executor.getLogger().currentFileWriter != null) { + executor.getLogger().currentFileWriter.close(); } - } catch (IOException e) { - e.printStackTrace(); - } - if (threadsShutdown == options.getTotalNumberTries()) { - execService.shutdown(); + executor.getLogger().currentFileWriter = null; } + } catch (IOException e) { + e.printStackTrace(); } } } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 2786e1add..3e3c52441 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -51,6 +51,9 @@ public class MainOptions { @Parameter(names = "--timeout-seconds", description = "The timeout in seconds") private int timeoutSeconds = -1; // NOPMD + @Parameter(names = "--max-generated-databases", description = "The maximum number of databases that are generated by each thread") + private int maxGeneratedDatabases = -1; // NOPMD + @Parameter(names = "--exit-code-error", description = "The exit code that should be returned when an error is encountered (or a bug is found)") private int errorExitCode = -1; // NOPMD @@ -136,6 +139,10 @@ public int getTimeoutSeconds() { return timeoutSeconds; } + public int getMaxGeneratedDatabases() { + return maxGeneratedDatabases; + } + public int getErrorExitCode() { return errorExitCode; } diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index b42ff89fd..b066b371a 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -85,14 +85,6 @@ public class SQLite3Options { "--execute-queries" }, description = "Specifies whether the query in the fuzzer should be executed", arity = 1) public boolean executeQuery = true; - @Parameter(names = { - "--print-successful-statements" }, description = "Specifies whether to print SQL statements to stdout", arity = 1) - public boolean executeStatementsAndPrintSuccessfulOnes; - - @Parameter(names = { - "--exit-after-first-database" }, description = "Specifies whether to stop SQLancer after creating and executing the test oracle on the first database", arity = 1) - public boolean exitAfterFirstDatabase; - public enum SQLite3Oracle { PQS { @Override diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 9c2438094..9e11cdb94 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -328,9 +328,6 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx } } - if (globalState.getDmbsSpecificOptions().exitAfterFirstDatabase) { - System.exit(0); - } } private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws SQLException { From c9a626f21a4968e4b0789db0a7bca45e770b9e6f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jul 2020 16:16:22 +0200 Subject: [PATCH 0007/1284] Move common functionality from the individual testing implementations to the ProviderAdapter class --- src/sqlancer/ProviderAdapter.java | 25 ++++++++ .../clickhouse/ClickHouseProvider.java | 20 ++---- .../cockroachdb/CockroachDBProvider.java | 32 ++++------ src/sqlancer/duckdb/DuckDBProvider.java | 20 ++---- src/sqlancer/mariadb/MariaDBProvider.java | 42 ++++-------- .../mariadb/oracle/MariaDBNoRECOracle.java | 6 +- src/sqlancer/mysql/MySQLProvider.java | 64 ++----------------- src/sqlancer/postgres/PostgresProvider.java | 37 ++--------- src/sqlancer/sqlite3/SQLite3Provider.java | 40 +++--------- src/sqlancer/tidb/TiDBProvider.java | 21 ++---- 10 files changed, 94 insertions(+), 213 deletions(-) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index d23cc1721..0f4c5bdf5 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -1,6 +1,7 @@ package sqlancer; import java.io.FileWriter; +import java.sql.SQLException; public abstract class ProviderAdapter, O> implements DatabaseProvider { @@ -32,4 +33,28 @@ public Class getOptionClass() { return optionClass; } + @Override + public void generateAndTestDatabase(G globalState) throws SQLException { + try { + generateDatabase(globalState); + globalState.getManager().incrementCreateDatabase(); + + TestOracle oracle = getTestOracle(globalState); + for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { + try { + oracle.check(); + globalState.getManager().incrementSelectQueryCount(); + } catch (IgnoreMeException e) { + + } + } + } finally { + globalState.getConnection().close(); + } + } + + protected abstract TestOracle getTestOracle(G globalState) throws SQLException; + + public abstract void generateDatabase(G globalState) throws SQLException; + } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 678515538..12ecb2a48 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -11,7 +11,6 @@ import sqlancer.CompositeTestOracle; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -84,8 +83,7 @@ protected void updateSchema() throws SQLException { } @Override - public void generateAndTestDatabase(ClickHouseGlobalState globalState) throws SQLException { - QueryManager manager = globalState.getManager(); + public void generateDatabase(ClickHouseGlobalState globalState) throws SQLException { for (int i = 0; i < Randomly.fromOptions(1); i++) { boolean success = false; do { @@ -101,8 +99,10 @@ public void generateAndTestDatabase(ClickHouseGlobalState globalState) throws SQ } }); se.executeStatements(); - manager.incrementCreateDatabase(); + } + @Override + protected TestOracle getTestOracle(ClickHouseGlobalState globalState) throws SQLException { List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { try { return o.create(globalState); @@ -110,17 +110,7 @@ public void generateAndTestDatabase(ClickHouseGlobalState globalState) throws SQ throw new AssertionError(e1); } }).collect(Collectors.toList()); - CompositeTestOracle oracle = new CompositeTestOracle(oracles); - - for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { - try { - oracle.check(); - } catch (IgnoreMeException e) { - continue; - } - manager.incrementSelectQueryCount(); - } - + return new CompositeTestOracle(oracles); } @Override diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 34f1c7cb6..47ccd25fd 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -122,8 +122,7 @@ protected void updateSchema() throws SQLException { } @Override - public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws SQLException { - Randomly r = new Randomly(); + public void generateDatabase(CockroachDBGlobalState globalState) throws SQLException { QueryManager manager = globalState.getManager(); MainOptions options = globalState.getOptions(); List standardSettings = new ArrayList<>(); @@ -168,30 +167,30 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S int nrPerformed = 0; switch (action) { case INSERT: - nrPerformed = r.getInteger(0, options.getMaxNumberInserts()); + nrPerformed = globalState.getRandomly().getInteger(0, options.getMaxNumberInserts()); break; case UPDATE: case SPLIT: - nrPerformed = r.getInteger(0, 3); + nrPerformed = globalState.getRandomly().getInteger(0, 3); break; case EXPLAIN: - nrPerformed = r.getInteger(0, 10); + nrPerformed = globalState.getRandomly().getInteger(0, 10); break; case SHOW: case TRUNCATE: case DELETE: case CREATE_STATISTICS: - nrPerformed = r.getInteger(0, 2); + nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case CREATE_VIEW: - nrPerformed = r.getInteger(0, 2); + nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case SET_SESSION: case SET_CLUSTER_SETTING: - nrPerformed = r.getInteger(0, 3); + nrPerformed = globalState.getRandomly().getInteger(0, 3); break; case CREATE_INDEX: - nrPerformed = r.getInteger(0, 10); + nrPerformed = globalState.getRandomly().getInteger(0, 10); break; case COMMENT_ON: case SCRUB: @@ -215,7 +214,7 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S while (total != 0) { Action nextAction = null; - int selection = r.getInteger(0, total); + int selection = globalState.getRandomly().getInteger(0, total); int previousRange = 0; for (int i = 0; i < nrRemaining.length; i++) { if (previousRange <= selection && selection < previousRange + nrRemaining[i]) { @@ -244,19 +243,14 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S } total--; } - manager.incrementCreateDatabase(); if (globalState.getDmbsSpecificOptions().makeVectorizationMoreLikely && Randomly.getBoolean()) { manager.execute(new QueryAdapter("SET vectorize=on;")); } - TestOracle oracle = globalState.getDmbsSpecificOptions().oracle.create(globalState); - for (int i = 0; i < options.getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { + } - } - } + @Override + protected TestOracle getTestOracle(CockroachDBGlobalState globalState) throws SQLException { + return globalState.getDmbsSpecificOptions().oracle.create(globalState); } @Override diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index eb89a79c8..e2383beb1 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -11,7 +11,6 @@ import sqlancer.CompositeTestOracle; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -100,8 +99,7 @@ protected void updateSchema() throws SQLException { } @Override - public void generateAndTestDatabase(DuckDBGlobalState globalState) throws SQLException { - QueryManager manager = globalState.getManager(); + public void generateDatabase(DuckDBGlobalState globalState) throws SQLException { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { boolean success = false; do { @@ -119,25 +117,17 @@ public void generateAndTestDatabase(DuckDBGlobalState globalState) throws SQLExc } }); se.executeStatements(); - manager.incrementCreateDatabase(); + } - TestOracle oracle = new CompositeTestOracle(globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { + @Override + protected TestOracle getTestOracle(DuckDBGlobalState globalState) throws SQLException { + return new CompositeTestOracle(globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { try { return o.create(globalState); } catch (SQLException e1) { throw new AssertionError(e1); } }).collect(Collectors.toList())); - - for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - - } - } - globalState.getConnection().close(); } @Override diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 3abcfc9b6..7193ad633 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -9,12 +9,12 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.TestOracle; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.gen.MariaDBIndexGenerator; import sqlancer.mariadb.gen.MariaDBInsertGenerator; @@ -29,8 +29,6 @@ public class MariaDBProvider extends ProviderAdapter { public static final int MAX_EXPRESSION_DEPTH = 3; - private final Randomly r = new Randomly(); - private String databaseName; public MariaDBProvider() { super(MariaDBGlobalState.class, MariaDBOptions.class); @@ -50,14 +48,13 @@ enum Action { } @Override - public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLException { - this.databaseName = globalState.getDatabaseName(); + public void generateDatabase(MariaDBGlobalState globalState) throws SQLException { MainOptions options = globalState.getOptions(); - QueryManager manager = globalState.getManager(); while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = MariaDBTableGenerator.generate(tableName, r, globalState.getSchema()); + Query createTable = MariaDBTableGenerator.generate(tableName, globalState.getRandomly(), + globalState.getSchema()); globalState.executeStatement(createTable); } @@ -76,13 +73,13 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx case ANALYZE_TABLE: case UPDATE: case CREATE_INDEX: - nrPerformed = r.getInteger(0, 2); + nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case SET: nrPerformed = 20; break; case INSERT: - nrPerformed = r.getInteger(0, options.getMaxNumberInserts()); + nrPerformed = globalState.getRandomly().getInteger(0, options.getMaxNumberInserts()); break; default: throw new AssertionError(action); @@ -95,7 +92,7 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx } while (total != 0) { Action nextAction = null; - int selection = r.getInteger(0, total); + int selection = globalState.getRandomly().getInteger(0, total); int previousRange = 0; for (int i = 0; i < nrRemaining.length; i++) { if (previousRange <= selection && selection < previousRange + nrRemaining[i]) { @@ -124,7 +121,7 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx query = MariaDBTableAdminCommandGenerator.repairTable(globalState.getSchema()); break; case INSERT: - query = MariaDBInsertGenerator.insert(globalState.getSchema(), r); + query = MariaDBInsertGenerator.insert(globalState.getSchema(), globalState.getRandomly()); break; case OPTIMIZE: query = MariaDBTableAdminCommandGenerator.optimizeTable(globalState.getSchema()); @@ -133,13 +130,13 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx query = MariaDBTableAdminCommandGenerator.analyzeTable(globalState.getSchema()); break; case UPDATE: - query = MariaDBUpdateGenerator.update(globalState.getSchema(), r); + query = MariaDBUpdateGenerator.update(globalState.getSchema(), globalState.getRandomly()); break; case CREATE_INDEX: query = MariaDBIndexGenerator.generate(globalState.getSchema()); break; case SET: - query = MariaDBSetGenerator.set(r, options); + query = MariaDBSetGenerator.set(globalState.getRandomly(), options); break; default: throw new AssertionError(nextAction); @@ -156,17 +153,11 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx } total--; } - // - MariaDBNoRECOracle queryGenerator = new MariaDBNoRECOracle(globalState); - for (int i = 0; i < options.getNrQueries(); i++) { - try { - queryGenerator.generateAndCheck(); - } catch (IgnoreMeException e) { - - } - manager.incrementSelectQueryCount(); - } + } + @Override + protected TestOracle getTestOracle(MariaDBGlobalState globalState) throws SQLException { + return new MariaDBNoRECOracle(globalState); } public static class MariaDBGlobalState extends GlobalState { @@ -205,9 +196,4 @@ public String getDBMSName() { return "mariadb"; } - @Override - public String toString() { - return String.format("MariaDBProvider [database: %s]", databaseName); - } - } diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index 8e4a28b78..68a4e8585 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -12,6 +12,7 @@ import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.StateToReproduce; +import sqlancer.TestOracle; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; @@ -29,7 +30,7 @@ import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; -public class MariaDBNoRECOracle { +public class MariaDBNoRECOracle implements TestOracle { private final MariaDBSchema s; private final Randomly r; @@ -61,7 +62,8 @@ public MariaDBNoRECOracle(MariaDBGlobalState globalState) { errors.add("digit expected after"); } - public void generateAndCheck() throws SQLException { + @Override + public void check() throws SQLException { MariaDBTable randomTable = s.getRandomTable(); List columns = randomTable.getColumns(); MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(r).setColumns(columns).setCon(con) diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 7604a8528..7ad3e5967 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -12,8 +12,6 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; -import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -45,9 +43,6 @@ public class MySQLProvider extends ProviderAdapter { - private QueryManager manager; - private String databaseName; - public MySQLProvider() { super(MySQLGlobalState.class, MySQLOptions.class); } @@ -147,12 +142,8 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { } @Override - public void generateAndTestDatabase(MySQLGlobalState globalState) throws SQLException { - this.databaseName = globalState.getDatabaseName(); - this.manager = globalState.getManager(); - MainOptions options = globalState.getOptions(); + public void generateDatabase(MySQLGlobalState globalState) throws SQLException { Randomly r = globalState.getRandomly(); - while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); Query createTable = MySQLTableGenerator.generate(tableName, r, globalState.getSchema()); @@ -166,52 +157,12 @@ public void generateAndTestDatabase(MySQLGlobalState globalState) throws SQLExce } }); se.executeStatements(); - manager.incrementCreateDatabase(); - - // for (MySQLTable t : globalState.getSchema().getDatabaseTables()) { - // if (!ensureTableHasRows(con, t, r)) { - // return; - // } - // } - - TestOracle oracle = new MySQLTLPWhereOracle(globalState); - for (int i = 0; i < options.getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - - } - } - - // MySQLQueryGenerator queryGenerator = new MySQLQueryGenerator(manager, r, con, databaseName); - // for (int i = 0; i < options.getNrQueries(); i++) { - // try { - // queryGenerator.generateAndCheckQuery((MySQLStateToReproduce) state, logger, options); - // } catch (IgnoreMeException e) { - // - // } - // manager.incrementSelectQueryCount(); - // } - } - // private boolean ensureTableHasRows(Connection con, MySQLTable randomTable, Randomly r) throws SQLException { - // int nrRows; - // int counter = 1; - // do { - // try { - // Query q = MySQLRowInserter.insertRow(randomTable, r); - // manager.execute(q); - // } catch (SQLException e) { - // if (!SQLite3PivotedQuerySynthesizer.shouldIgnoreException(e)) { - // throw new AssertionError(e); - // } - // } - // nrRows = getNrRows(con, randomTable); - // } while (nrRows == 0 && counter-- != 0); - // return nrRows != 0; - // } + @Override + protected TestOracle getTestOracle(MySQLGlobalState globalState) throws SQLException { + return new MySQLTLPWhereOracle(globalState); // FIXME: options for the other test oracles + } public static int getNrRows(Connection con, MySQLTable table) throws SQLException { try (Statement s = con.createStatement()) { @@ -248,11 +199,6 @@ public String getDBMSName() { return "mysql"; } - @Override - public String toString() { - return String.format("MySQLProvider [database: %s]", databaseName); - } - @Override public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { StringBuilder sb = new StringBuilder(); diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 03bca1cdd..4cf1dfba8 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -13,8 +13,6 @@ import sqlancer.AbstractAction; import sqlancer.CompositeTestOracle; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; -import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -183,9 +181,7 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { } @Override - public void generateAndTestDatabase(PostgresGlobalState globalState) throws SQLException { - MainOptions options = globalState.getOptions(); - QueryManager manager = globalState.getManager(); + public void generateDatabase(PostgresGlobalState globalState) throws SQLException { while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { try { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); @@ -203,22 +199,13 @@ public void generateAndTestDatabase(PostgresGlobalState globalState) throws SQLE throw new IgnoreMeException(); } }); - // TODO: transactions broke during refactoring - // catch (Throwable t) { - // if (t.getMessage().contains("current transaction is aborted")) { - // manager.execute(new QueryAdapter("ABORT")); - // globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); - // } else { - // System.err.println(query.getQueryString()); - // throw t; - // } - // } se.executeStatements(); - manager.incrementCreateDatabase(); - manager.execute(new QueryAdapter("COMMIT", true)); - - manager.execute(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); + globalState.executeStatement(new QueryAdapter("COMMIT", true)); + globalState.executeStatement(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); + } + @Override + protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { try { return o.create(globalState); @@ -226,17 +213,7 @@ public void generateAndTestDatabase(PostgresGlobalState globalState) throws SQLE throw new AssertionError(e1); } }).collect(Collectors.toList()); - CompositeTestOracle oracle = new CompositeTestOracle(oracles); - - for (int i = 0; i < options.getNrQueries(); i++) { - try { - oracle.check(); - } catch (IgnoreMeException e) { - continue; - } - manager.incrementSelectQueryCount(); - } - + return new CompositeTestOracle(oracles); } @Override diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 9e11cdb94..5712b0b48 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -13,7 +13,6 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; @@ -54,11 +53,6 @@ public class SQLite3Provider extends ProviderAdapter DEFAULT_PRAGMAS = Arrays.asList("PRAGMA cache_size = 50000;", "PRAGMA temp_store=MEMORY;", "PRAGMA synchronous=off;"); @@ -182,17 +176,12 @@ private enum TableType { } @Override - public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLException { - this.globalState = globalState; + public void generateDatabase(SQLite3GlobalState globalState) throws SQLException { SQLite3Options sqliteOptions = globalState.getDmbsSpecificOptions(); - QueryManager manager = globalState.getManager(); MainOptions options = globalState.getOptions(); - this.databaseName = globalState.getDatabaseName(); Randomly r = new Randomly(SQLite3SpecialStringGenerator::generate); globalState.setSqliteOptions(sqliteOptions); globalState.setRandomly(r); - this.state = (SQLite3StateToReproduce) globalState.getState(); - globalState.setState(state); if (globalState.getDmbsSpecificOptions().generateDatabase) { addSensiblePragmaDefaults(globalState); @@ -206,7 +195,7 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx int i = 0; do { - Query tableQuery = getTableQuery(r, i++); + Query tableQuery = getTableQuery(globalState, i++); globalState.executeStatement(tableQuery); } while (globalState.getSchema().getDatabaseTables().size() < nrTablesToCreate); assert globalState.getSchema().getTables().getTables().size() == nrTablesToCreate; @@ -274,7 +263,6 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx nrRemaining[action.ordinal()] = nrPerformed; total += nrPerformed; } - while (total != 0) { Action nextAction = null; int selection = r.getInteger(0, total); @@ -309,8 +297,11 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx // also do an abort for DEFERRABLE INITIALLY DEFERRED query = SQLite3TransactionGenerator.generateRollbackTransaction(globalState); globalState.executeStatement(query); - manager.incrementCreateDatabase(); } + } + + @Override + protected TestOracle getTestOracle(SQLite3GlobalState globalState) throws SQLException { TestOracle oracle = globalState.getSqliteOptions().oracle.create(globalState); if (oracle.onlyWorksForNonEmptyTables()) { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { @@ -320,14 +311,7 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx } } } - for (int i = 0; i < options.getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - - } - } + return oracle; } private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws SQLException { @@ -342,7 +326,7 @@ private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) } } - private Query getTableQuery(Randomly r, int i) throws AssertionError { + private Query getTableQuery(SQLite3GlobalState globalState, int i) throws AssertionError { Query tableQuery; List options = new ArrayList<>(Arrays.asList(TableType.values())); if (!globalState.getSqliteOptions().testFts) { @@ -358,7 +342,8 @@ private Query getTableQuery(Randomly r, int i) throws AssertionError { break; case FTS: String ftsTableName = "v" + SQLite3Common.createTableName(i); - tableQuery = SQLite3CreateVirtualFTSTableGenerator.createTableStatement(ftsTableName, r); + tableQuery = SQLite3CreateVirtualFTSTableGenerator.createTableStatement(ftsTableName, + globalState.getRandomly()); break; case RTREE: String rTreeTableName = "rt" + i; @@ -406,11 +391,6 @@ public String getDBMSName() { return "sqlite3"; } - @Override - public String toString() { - return String.format("SQLite3Provider [database: %s]", databaseName); - } - @Override public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 7b91541ae..225ef1a13 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -13,7 +13,6 @@ import sqlancer.CompositeTestOracle; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -109,8 +108,7 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { } @Override - public void generateAndTestDatabase(TiDBGlobalState globalState) throws SQLException { - QueryManager manager = globalState.getManager(); + public void generateDatabase(TiDBGlobalState globalState) throws SQLException { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { boolean success = false; do { @@ -135,7 +133,10 @@ public void generateAndTestDatabase(TiDBGlobalState globalState) throws SQLExcep throw new AssertionError(e); } } - manager.incrementCreateDatabase(); + } + + @Override + protected TestOracle getTestOracle(TiDBGlobalState globalState) throws SQLException { List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { try { return o.create(globalState); @@ -143,17 +144,7 @@ public void generateAndTestDatabase(TiDBGlobalState globalState) throws SQLExcep throw new AssertionError(e1); } }).collect(Collectors.toList()); - CompositeTestOracle oracle = new CompositeTestOracle(oracles); - - for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - - } - } - + return new CompositeTestOracle(oracles); } @Override From 5cbc248b592601f558f844cfb596493ece3e62e9 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 2 Jul 2020 16:59:45 +0300 Subject: [PATCH 0008/1284] Add Travis CI check for ClickHouse. Support TLPAggregate, TLPDistinct --- .travis.yml | 10 ++ codecov.yml | 2 +- src/sqlancer/clickhouse/ClickHouseErrors.java | 1 + .../clickhouse/ClickHouseOptions.java | 16 ++- .../clickhouse/ClickHouseProvider.java | 6 +- .../clickhouse/ClickHouseToStringVisitor.java | 11 ++ .../clickhouse/ClickHouseVisitor.java | 6 + .../clickhouse/ast/ClickHouseExpression.java | 47 +++++++ .../gen/ClickHouseInsertGenerator.java | 2 +- .../gen/ClickHouseTableGenerator.java | 12 +- .../tlp/ClickHouseTLPAggregateOracle.java | 120 ++++++++++++++++++ .../tlp/ClickHouseTLPDistinctOracle.java | 40 ++++++ ...ickHouseBinaryComparisonOperationTest.java | 2 - test/sqlancer/dbms/TestClickHouse.java | 49 +++++-- 14 files changed, 302 insertions(+), 22 deletions(-) create mode 100644 src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java create mode 100644 src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java diff --git a/.travis.yml b/.travis.yml index 47378bb04..13b03cdb0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,3 +95,13 @@ matrix: - createdb test -U postgres script: - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test + - name: ClickHouse + jdk : oraclejdk11 + services: + - docker + before_install: + - docker pull yandex/clickhouse-server:latest + - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest + - sleep 5 + script: + - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse test diff --git a/codecov.yml b/codecov.yml index ba6bb1342..ed891a44d 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,7 +1,7 @@ codecov: require_ci_to_pass: no notify: - after_n_builds: 10 + after_n_builds: 11 coverage: range: "50...100" diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index db704f315..20665b139 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -33,6 +33,7 @@ public static void addExpectedExpressionErrors(Set errors) { errors.add("Attempt to read after eof: while converting"); errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database errors.add("is not under aggregate function"); + errors.add("Invalid type for filter in"); } public static void addExpressionHavingErrors(Set errors) { diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 51554710c..002ccee77 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -4,9 +4,11 @@ import com.beust.jcommander.Parameters; import sqlancer.MainOptions; import sqlancer.TestOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPHavingOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; import java.sql.SQLException; import java.util.Arrays; @@ -28,12 +30,24 @@ public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) t return new ClickHouseTLPWhereOracle(globalState); } }, + TLPDistinct { + @Override + public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPDistinctOracle(globalState); + } + }, TLPGroupBy { @Override public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPGroupByOracle(globalState); } }, + TLPAggregate { + @Override + public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPAggregateOracle(globalState); + } + }, TLPHaving { @Override public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 12ecb2a48..945785c0f 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -85,7 +85,7 @@ protected void updateSchema() throws SQLException { @Override public void generateDatabase(ClickHouseGlobalState globalState) throws SQLException { for (int i = 0; i < Randomly.fromOptions(1); i++) { - boolean success = false; + boolean success; do { Query qt = new ClickHouseTableGenerator().getQuery(globalState); success = globalState.executeStatement(qt); @@ -117,7 +117,7 @@ protected TestOracle getTestOracle(ClickHouseGlobalState globalState) throws SQL public Connection createDatabase(ClickHouseGlobalState globalState) throws SQLException { ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); - String url = "jdbc:clickhouse://localhost:8123/test"; + String url = "jdbc:clickhouse://localhost:8123/default"; String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); @@ -138,7 +138,7 @@ public Connection createDatabase(ClickHouseGlobalState globalState) throws SQLEx e.printStackTrace(); } con.close(); - con = DriverManager.getConnection("jdbc:clickhouse://localhost:18123/" + databaseName, + con = DriverManager.getConnection("jdbc:clickhouse://localhost:8123/" + databaseName, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); return con; } diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index ab966f7d9..877a6d5fa 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -70,6 +70,17 @@ public void visit(ClickHouseSelect select, boolean inner) { sb.append("("); } sb.append("SELECT "); + switch (select.getFromOptions()) { + case DISTINCT: + sb.append("DISTINCT "); + break; + case ALL: + sb.append(""); + break; + default: + throw new AssertionError(select.getFromOptions()); + } + visit(select.getFetchColumns()); sb.append(" FROM "); visit(select.getFromList()); diff --git a/src/sqlancer/clickhouse/ClickHouseVisitor.java b/src/sqlancer/clickhouse/ClickHouseVisitor.java index 25fa65466..e1473f669 100644 --- a/src/sqlancer/clickhouse/ClickHouseVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseVisitor.java @@ -43,6 +43,10 @@ default void visit(ClickHouseColumnReference columnReference) { }; + default void visit(ClickHouseExpression.ClickHousePostfixText op) { + + } + void visit(ClickHouseTableReference tableReference); void visit(ClickHouseCastOperation cast); @@ -70,6 +74,8 @@ default void visit(ClickHouseExpression expr) { visit((ClickHouseCastOperation) expr); } else if (expr instanceof ClickHouseExpression.ClickHouseJoin) { visit((ClickHouseExpression.ClickHouseJoin) expr); + } else if (expr instanceof ClickHouseExpression.ClickHousePostfixText) { + visit((ClickHouseExpression.ClickHousePostfixText) expr); } else if (expr instanceof ClickHouseAggregate) { visit((ClickHouseAggregate) expr); } else { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index 16356c9cd..c32e26131 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.ast; import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.visitor.UnaryOperation; public abstract class ClickHouseExpression { @@ -96,4 +97,50 @@ public String getQuery() { } } + public static class ClickHousePostfixText extends ClickHouseExpression + implements UnaryOperation { + + private final ClickHouseExpression expr; + private final String text; + private ClickHouseConstant expectedValue; + + public ClickHousePostfixText(ClickHouseExpression expr, String text, ClickHouseConstant expectedValue) { + this.expr = expr; + this.text = text; + this.expectedValue = expectedValue; + } + + public ClickHousePostfixText(String text, ClickHouseConstant expectedValue) { + this(null, text, expectedValue); + } + + public String getText() { + return text; + } + + @Override + public ClickHouseConstant getExpectedValue() { + return expectedValue; + } + + @Override + public ClickHouseExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return getText(); + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + } } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 3cd4a9435..2ecccbef0 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -25,7 +25,7 @@ public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { gen = new ClickHouseExpressionGenerator(globalState); errors.add("Cannot insert NULL value into a column of type 'Int32'"); // TODO errors.add("Cannot insert NULL value into a column of type 'String'"); - + errors.add("Memory limit"); errors.add("Cannot parse string"); errors.add("Cannot parse Int32 from String, because value is too short"); } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index d597c7e47..aac5f6413 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -11,13 +11,19 @@ public class ClickHouseTableGenerator { + StringBuilder sb = new StringBuilder("CREATE TABLE "); + Set errors = new HashSet<>(); + + public ClickHouseTableGenerator() { + errors.add("Memory limit"); + errors.add("Directory for table data"); + errors.add("Directory not empty"); + } + private enum ClickHouseEngine { TinyLog, StripeLog, Log, Memory, MergeTree } - StringBuilder sb = new StringBuilder("CREATE TABLE "); - Set errors = new HashSet<>(); - public Query getQuery(ClickHouseGlobalState globalState) { ClickHouseEngine engine = Randomly.fromOptions(ClickHouseEngine.values()); sb.append(globalState.getSchema().getFreeTableName()); diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java new file mode 100644 index 000000000..001cdc547 --- /dev/null +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -0,0 +1,120 @@ +package sqlancer.clickhouse.oracle.tlp; + +import ru.yandex.clickhouse.domain.ClickHouseDataType; +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.QueryAdapter; +import sqlancer.Randomly; +import sqlancer.TestOracle; +import sqlancer.clickhouse.ClickHouseProvider; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; +import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; +import sqlancer.clickhouse.gen.ClickHouseCommon; +import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +public class ClickHouseTLPAggregateOracle implements TestOracle { + + private final ClickHouseProvider.ClickHouseGlobalState state; + private ClickHouseExpressionGenerator gen; + + public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState state) { + this.state = state; + } + + @Override + public void check() throws SQLException { + ClickHouseSchema s = state.getSchema(); + ClickHouseSchema.ClickHouseTables targetTables = s.getRandomTableNonEmptyTables(); + gen = new ClickHouseExpressionGenerator(state).setColumns(targetTables.getColumns()); + ClickHouseSelect select = new ClickHouseSelect(); + ClickHouseAggregate.ClickHouseAggregateFunction windowFunction = Randomly.fromOptions( + ClickHouseAggregate.ClickHouseAggregateFunction.MIN, + ClickHouseAggregate.ClickHouseAggregateFunction.MAX, + ClickHouseAggregate.ClickHouseAggregateFunction.SUM); + ClickHouseAggregate aggregate = new ClickHouseAggregate( + gen.generateExpressions(ClickHouseSchema.ClickHouseLancerDataType.getRandom(), 1), windowFunction); + select.setFetchColumns(Arrays.asList(aggregate)); + List from = ClickHouseCommon.getTableRefs(targetTables.getTables(), s); + select.setFromList(from); + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + String originalQuery = ClickHouseVisitor.asString(select); + + ClickHouseExpression whereClause = gen + .generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + ClickHouseUnaryPrefixOperation negatedClause = new ClickHouseUnaryPrefixOperation(whereClause, + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.NOT); + ClickHouseUnaryPostfixOperation notNullClause = new ClickHouseUnaryPostfixOperation(whereClause, + ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.IS_NULL, false); + + ClickHouseSelect leftSelect = getSelect(aggregate, from, whereClause); + ClickHouseSelect middleSelect = getSelect(aggregate, from, negatedClause); + ClickHouseSelect rightSelect = getSelect(aggregate, from, notNullClause); + String metamorphicText = "SELECT " + aggregate.getFunc().toString() + "(aggr) FROM ("; + metamorphicText += ClickHouseVisitor.asString(leftSelect) + " UNION ALL " + + ClickHouseVisitor.asString(middleSelect) + " UNION ALL " + ClickHouseVisitor.asString(rightSelect); + metamorphicText += ")"; + + String firstResult; + String secondResult; + QueryAdapter q = new QueryAdapter(originalQuery); + try (ResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + firstResult = result.getString(1); + } catch (Exception e) { + // TODO + throw new IgnoreMeException(); + } + + QueryAdapter q2 = new QueryAdapter(metamorphicText); + try (ResultSet result = q2.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + secondResult = result.getString(1); + } catch (Exception e) { + // TODO + throw new IgnoreMeException(); + } + state.getState().queryString = "--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " + + secondResult; + if ((firstResult == null && secondResult != null + || firstResult != null && !firstResult.contentEquals(secondResult)) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + + throw new AssertionError(); + + } + + } + + private ClickHouseSelect getSelect(ClickHouseAggregate aggregate, List from, + ClickHouseExpression whereClause) { + ClickHouseSelect leftSelect = new ClickHouseSelect(); + leftSelect.setFetchColumns( + Arrays.asList(new ClickHouseExpression.ClickHousePostfixText(aggregate, " as aggr", null))); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + if (Randomly.getBooleanWithRatherLowProbability()) { + leftSelect.setGroupByClause(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + if (Randomly.getBoolean()) { + leftSelect.setOrderByExpressions(gen.generateOrderBys()); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java new file mode 100644 index 000000000..72523fdf1 --- /dev/null +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java @@ -0,0 +1,40 @@ +package sqlancer.clickhouse.oracle.tlp; + +import sqlancer.ComparatorHelper; +import sqlancer.clickhouse.ClickHouseProvider; +import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.ClickHouseSelect; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class ClickHouseTLPDistinctOracle extends ClickHouseTLPBase { + + public ClickHouseTLPDistinctOracle(ClickHouseProvider.ClickHouseGlobalState state) { + super(state); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setSelectType(ClickHouseSelect.SelectType.DISTINCT); + select.setWhereClause(null); + String originalQueryString = ClickHouseVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(predicate); + String firstQueryString = ClickHouseVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = ClickHouseVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = ClickHouseVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, false, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + +} diff --git a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java index afbe76227..3977b0bdf 100644 --- a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java +++ b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java @@ -33,8 +33,6 @@ void getExpectedValueFloat64EqualsFloat64() { ClickHouseConstant oneFConst = ClickHouseConstant.createFloat64Constant(1.0); ClickHouseConstant zeroConst = ClickHouseConstant.createFloat64Constant(0); ClickHouseConstant zeroFConst = ClickHouseConstant.createFloat64Constant(0.0); - ClickHouseConstant negativeConst = ClickHouseConstant.createFloat64Constant(-100.0); - ClickHouseConstant positiveConst = ClickHouseConstant.createFloat64Constant(10000.0); assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); assertEquals(oneFConst.applyEquals(oneFConst).asInt(), 1); diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 5626ac3b3..6ce0b5838 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -6,19 +6,18 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import sqlancer.Main; -import sqlancer.dbms.TestConfig; public class TestClickHouse { @Test - public void testClickHouseTLPWhereHaving() { + public void testClickHouseTLPWhereGroupBy() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); Assertions.assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", TestConfig.SECONDS, "--num-queries", - TestConfig.NUM_QUERIES, "--num-threads", "50", "clickhouse", "--oracle", "TLPWhere", "--oracle", - "TLPHaving" })); + Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, + "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", + "TLPWhere", "--oracle", "TLPGroupBy" })); } @Test @@ -27,8 +26,9 @@ public void testClickHouseTLPWhere() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); Assertions.assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", TestConfig.SECONDS, "--num-queries", - TestConfig.NUM_QUERIES, "--num-threads", "50", "clickhouse", "--oracle", "TLPWhere" })); + Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, + "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", + "TLPWhere" })); } @Test @@ -36,8 +36,11 @@ public void testClickHouseTLPHaving() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - assertEquals(0, Main.executeMain(new String[] { "--timeout-seconds", TestConfig.SECONDS, "--num-queries", - TestConfig.NUM_QUERIES, "--num-threads", "30", "clickhouse", "--oracle", "TLPHaving" })); + assertEquals(0, + Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", "0", "--num-threads", "5", + "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPHaving" })); // Disabled + // in CI + // https://github.com/ClickHouse/ClickHouse/issues/12264 } @Test @@ -45,7 +48,31 @@ public void testClickHouseTLPGroupBy() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - assertEquals(0, Main.executeMain(new String[] { "--timeout-seconds", TestConfig.SECONDS, "--num-queries", - TestConfig.NUM_QUERIES, "--num-threads", "30", "clickhouse", "--oracle", "TLPGroupBy" })); + assertEquals(0, + Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, + "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", + "TLPGroupBy" })); } + + @Test + public void testClickHouseTLPDistinct() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", "0", "--num-threads", "5", + "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPDistinct" })); + } + + @Test + public void testClickHouseTLPAggregate() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, + "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", + "TLPAggregate" })); + } + } From 234a49f7ad09365e6398f7a375417cb39bfb5135 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jul 2020 17:43:33 +0200 Subject: [PATCH 0009/1284] [SQLite3] Use the statement executor --- src/sqlancer/sqlite3/SQLite3Provider.java | 151 +++++++++------------- 1 file changed, 63 insertions(+), 88 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 5712b0b48..b265777cf 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -11,9 +11,9 @@ import java.util.Arrays; import java.util.List; +import sqlancer.AbstractAction; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -21,6 +21,7 @@ import sqlancer.Randomly; import sqlancer.StateToReproduce; import sqlancer.StateToReproduce.SQLite3StateToReproduce; +import sqlancer.StatementExecutor; import sqlancer.TestOracle; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; @@ -61,7 +62,7 @@ public SQLite3Provider() { super(SQLite3GlobalState.class, SQLite3Options.class); } - public enum Action { + public enum Action implements AbstractAction { PRAGMA(SQLite3PragmaGenerator::insertPragma), // INDEX(SQLite3IndexGenerator::insertIndex), // INSERT(SQLite3InsertGenerator::insertRow), // @@ -147,6 +148,7 @@ public enum Action { this.queryProvider = queryProvider; } + @Override public Query getQuery(SQLite3GlobalState state) throws SQLException { return queryProvider.getQuery(state); } @@ -175,10 +177,61 @@ private enum TableType { NORMAL, FTS, RTREE } + private static int mapActions(SQLite3GlobalState globalState, Action a) { + int nrPerformed = 0; + Randomly r = globalState.getRandomly(); + switch (a) { + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case DELETE: + case DROP_VIEW: + case DROP_INDEX: + nrPerformed = r.getInteger(0, 0); + break; + case ALTER: + nrPerformed = r.getInteger(0, 0); + break; + case EXPLAIN: + case CREATE_TRIGGER: + case DROP_TABLE: + nrPerformed = r.getInteger(0, 0); + break; + case VACUUM: + case CHECK_RTREE_TABLE: + nrPerformed = r.getInteger(0, 3); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + case MANIPULATE_STAT_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case VIRTUAL_TABLE_ACTION: + case UPDATE: + nrPerformed = r.getInteger(0, 30); + break; + case PRAGMA: + nrPerformed = r.getInteger(0, 20); + break; + case TRANSACTION_START: + case REINDEX: + case ANALYZE: + case ROLLBACK_TRANSACTION: + case COMMIT: + default: + nrPerformed = r.getInteger(1, 10); + break; + } + return nrPerformed; + } + @Override public void generateDatabase(SQLite3GlobalState globalState) throws SQLException { SQLite3Options sqliteOptions = globalState.getDmbsSpecificOptions(); - MainOptions options = globalState.getOptions(); Randomly r = new Randomly(SQLite3SpecialStringGenerator::generate); globalState.setSqliteOptions(sqliteOptions); globalState.setRandomly(r); @@ -205,92 +258,14 @@ public void generateDatabase(SQLite3GlobalState globalState) throws SQLException "CREATE VIRTUAL TABLE IF NOT EXISTS stat USING dbstat(main)"); globalState.executeStatement(tableQuery); } - int[] nrRemaining = new int[Action.values().length]; - List actions = new ArrayList<>(); - int total = 0; - for (i = 0; i < Action.values().length; i++) { - Action action = Action.values()[i]; - int nrPerformed = 0; - switch (action) { - case CREATE_VIEW: - nrPerformed = r.getInteger(0, 2); - break; - case DELETE: - case DROP_VIEW: - case DROP_INDEX: - nrPerformed = r.getInteger(0, 0); - break; - case ALTER: - nrPerformed = r.getInteger(0, 0); - break; - case EXPLAIN: - case CREATE_TRIGGER: - case DROP_TABLE: - nrPerformed = r.getInteger(0, 0); - break; - case VACUUM: - case CHECK_RTREE_TABLE: - nrPerformed = r.getInteger(0, 3); - break; - case INSERT: - nrPerformed = r.getInteger(0, options.getMaxNumberInserts()); - break; - case MANIPULATE_STAT_TABLE: - nrPerformed = r.getInteger(0, 5); - break; - case INDEX: - nrPerformed = r.getInteger(0, 5); - break; - case VIRTUAL_TABLE_ACTION: - case UPDATE: - nrPerformed = r.getInteger(0, 30); - break; - case PRAGMA: - nrPerformed = r.getInteger(0, 20); - break; - case TRANSACTION_START: - case REINDEX: - case ANALYZE: - case ROLLBACK_TRANSACTION: - case COMMIT: - default: - nrPerformed = r.getInteger(1, 10); - break; - } - if (nrPerformed != 0) { - actions.add(action); - } - nrRemaining[action.ordinal()] = nrPerformed; - total += nrPerformed; - } - while (total != 0) { - Action nextAction = null; - int selection = r.getInteger(0, total); - int previousRange = 0; - for (i = 0; i < nrRemaining.length; i++) { - if (previousRange <= selection && selection < previousRange + nrRemaining[i]) { - nextAction = Action.values()[i]; - break; - } else { - previousRange += nrRemaining[i]; - } - } - assert nextAction != null; - assert nrRemaining[nextAction.ordinal()] > 0; - nrRemaining[nextAction.ordinal()]--; - Query query = null; - try { - query = nextAction.getQuery(globalState); - globalState.executeStatement(query); - } catch (IgnoreMeException e) { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + SQLite3Provider::mapActions, (q) -> { + if (q.couldAffectSchema() && globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); - } - if (query != null && query.couldAffectSchema() - && globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - total--; - } Query query = SQLite3TransactionGenerator.generateCommit(globalState); globalState.executeStatement(query); From d6c80368c7f4b63adee7398e00ce92425e99aaa2 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jul 2020 17:52:34 +0200 Subject: [PATCH 0010/1284] [SQLite3] Remove redundant options --- src/sqlancer/sqlite3/SQLite3Provider.java | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index b265777cf..14ab84fa8 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -156,16 +156,6 @@ public Query getQuery(SQLite3GlobalState state) throws SQLException { public static class SQLite3GlobalState extends GlobalState { - private SQLite3Options sqliteOptions; - - public void setSqliteOptions(SQLite3Options sqliteOptions) { - this.sqliteOptions = sqliteOptions; - } - - public SQLite3Options getSqliteOptions() { - return sqliteOptions; - } - @Override protected void updateSchema() throws SQLException { setSchema(SQLite3Schema.fromConnection(this)); @@ -231,9 +221,7 @@ private static int mapActions(SQLite3GlobalState globalState, Action a) { @Override public void generateDatabase(SQLite3GlobalState globalState) throws SQLException { - SQLite3Options sqliteOptions = globalState.getDmbsSpecificOptions(); Randomly r = new Randomly(SQLite3SpecialStringGenerator::generate); - globalState.setSqliteOptions(sqliteOptions); globalState.setRandomly(r); if (globalState.getDmbsSpecificOptions().generateDatabase) { @@ -277,7 +265,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws SQLException @Override protected TestOracle getTestOracle(SQLite3GlobalState globalState) throws SQLException { - TestOracle oracle = globalState.getSqliteOptions().oracle.create(globalState); + TestOracle oracle = globalState.getDmbsSpecificOptions().oracle.create(globalState); if (oracle.onlyWorksForNonEmptyTables()) { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { int nrRows = SQLite3Schema.getNrRows(globalState, table.getName()); @@ -304,10 +292,10 @@ private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) private Query getTableQuery(SQLite3GlobalState globalState, int i) throws AssertionError { Query tableQuery; List options = new ArrayList<>(Arrays.asList(TableType.values())); - if (!globalState.getSqliteOptions().testFts) { + if (!globalState.getDmbsSpecificOptions().testFts) { options.remove(TableType.FTS); } - if (!globalState.getSqliteOptions().testRtree) { + if (!globalState.getDmbsSpecificOptions().testRtree) { options.remove(TableType.RTREE); } switch (Randomly.fromList(options)) { From f19414826cc059f6e4ec96bb71678911a49cff75 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 13 Jul 2020 10:31:37 +0200 Subject: [PATCH 0011/1284] Fix the thread exiting logic --- src/sqlancer/Main.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 1295ac990..7f41ecaef 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -440,12 +440,17 @@ private void runThread(final String databaseName) { try { if (options.getMaxGeneratedDatabases() == -1) { // run without a limit - while (true) { - run(options, execService, executorFactory, seed, databaseName); + boolean continueRunning = true; + while (continueRunning) { + continueRunning = run(options, execService, executorFactory, seed, databaseName); } } else { for (int i = 0; i < options.getMaxGeneratedDatabases(); i++) { - run(options, execService, executorFactory, seed, databaseName); + boolean continueRunning = run(options, execService, executorFactory, seed, + databaseName); + if (!continueRunning) { + break; + } } } } finally { @@ -456,19 +461,20 @@ private void runThread(final String databaseName) { } } - private void run(MainOptions options, ExecutorService execService, + private boolean run(MainOptions options, ExecutorService execService, DBMSExecutorFactory executorFactory, final long seed, final String databaseName) { DBMSExecutor executor = executorFactory.getDBMSExecutor(databaseName, seed); try { executor.run(); + return true; } catch (IgnoreMeException e) { - return; + return true; } catch (Throwable reduce) { reduce.printStackTrace(); executor.getStateToReproduce().exception = reduce.getMessage(); executor.getLogger().logFileWriter = null; executor.getLogger().logException(reduce, executor.getStateToReproduce()); - return; + return false; } finally { try { if (options.logEachSelect()) { From 760266b721d1fb1f9624ac371e2ac576395a84c3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 13 Jul 2020 13:44:50 +0200 Subject: [PATCH 0012/1284] Remove the ClickHouse Travis CI tests since they currently always fail --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 13b03cdb0..76418fdb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,5 +103,3 @@ matrix: - docker pull yandex/clickhouse-server:latest - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest - sleep 5 - script: - - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse test From 605a1041c80a749cdd457b643feff118d21e5b24 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2020 12:13:16 +0000 Subject: [PATCH 0013/1284] Bump mysql-connector-java from 8.0.20 to 8.0.21 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.20 to 8.0.21. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.20...8.0.21) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8669c0664..5d3b3a850 100644 --- a/pom.xml +++ b/pom.xml @@ -201,7 +201,7 @@ mysql mysql-connector-java - 8.0.20 + 8.0.21 org.mariadb.jdbc From 9f1385859d42954d6f82bed834993c978bfbf962 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2020 12:57:02 +0000 Subject: [PATCH 0014/1284] Bump formatter-maven-plugin from 2.12.0 to 2.12.1 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.12.0 to 2.12.1. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/formatter-maven-plugin-2.12.1/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.12.0...formatter-maven-plugin-2.12.1) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5d3b3a850..f9c0e0c90 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.12.0 + 2.12.1 eclipseformat From 6e85452c567d8a873e4f46a896b6995f478287d7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Jul 2020 13:39:45 +0200 Subject: [PATCH 0015/1284] Add a method to specify whether a db creation action should be retried --- src/sqlancer/AbstractAction.java | 10 ++++++++++ src/sqlancer/StatementExecutor.java | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/AbstractAction.java b/src/sqlancer/AbstractAction.java index d2181e041..9238d8208 100644 --- a/src/sqlancer/AbstractAction.java +++ b/src/sqlancer/AbstractAction.java @@ -6,4 +6,14 @@ public interface AbstractAction { Query getQuery(G globalState) throws SQLException; + /** + * Specifies whether it makes sense to request a {@link Query}, when the previous call to {@link #getQuery(Object)} + * returned a query that failed executing. + * + * @return whether retrying getting queries makes sense, if the first query failed executing. + */ + default boolean canBeRetried() { + return true; + } + } diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index a344a9286..edf46446a 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -65,7 +65,8 @@ public void executeStatements() throws SQLException { do { query = nextAction.getQuery(globalState); success = globalState.executeStatement(query); - } while (!success && nrTries++ < globalState.getOptions().getNrStatementRetryCount()); + } while (nextAction.canBeRetried() && !success + && nrTries++ < globalState.getOptions().getNrStatementRetryCount()); } catch (IgnoreMeException e) { } From 3d6952392b7fb80f6afcf377423f82963fc6a602 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Jul 2020 13:40:22 +0200 Subject: [PATCH 0016/1284] [SQLite3] Do not retry the transaction statements since they repeatedly fail --- src/sqlancer/sqlite3/SQLite3Provider.java | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 14ab84fa8..e4d83cf50 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -70,12 +70,28 @@ public enum Action implements AbstractAction { REINDEX(SQLite3ReindexGenerator::executeReindex), // ANALYZE(SQLite3AnalyzeGenerator::generateAnalyze), // DELETE(SQLite3DeleteGenerator::deleteContent), // - TRANSACTION_START(SQLite3TransactionGenerator::generateBeginTransaction), // + TRANSACTION_START(SQLite3TransactionGenerator::generateBeginTransaction) { + @Override + public boolean canBeRetried() { + return false; + } + + }, // ALTER(SQLite3AlterTable::alterTable), // DROP_INDEX(SQLite3DropIndexGenerator::dropIndex), // UPDATE(SQLite3UpdateGenerator::updateRow), // - ROLLBACK_TRANSACTION(SQLite3TransactionGenerator::generateRollbackTransaction), // - COMMIT(SQLite3TransactionGenerator::generateCommit), // + ROLLBACK_TRANSACTION(SQLite3TransactionGenerator::generateRollbackTransaction) { + @Override + public boolean canBeRetried() { + return false; + } + }, // + COMMIT(SQLite3TransactionGenerator::generateCommit) { + @Override + public boolean canBeRetried() { + return false; + } + }, // DROP_TABLE(SQLite3DropTableGenerator::dropTable), // DROP_VIEW(SQLite3ViewGenerator::dropView), // EXPLAIN(SQLite3ExplainGenerator::explain), // From 2610e1fd9989bd491c92995e920232d6f89d3fc9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Jul 2020 22:20:30 +0200 Subject: [PATCH 0017/1284] Print an execution summary before exiting SQLancer --- src/sqlancer/Main.java | 25 +++++++++++++++++++++++++ src/sqlancer/MainOptions.java | 7 +++++++ 2 files changed, 32 insertions(+) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 7f41ecaef..f0fb2df61 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -415,6 +415,30 @@ public static int executeMain(String... args) throws AssertionError { if (options.printProgressInformation()) { startProgressMonitor(); + if (options.printProgressSummary()) { + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + + @Override + public void run() { + System.out.println("Overall execution statistics"); + System.out.println("============================"); + System.out.println(formatInteger(nrQueries.get()) + " queries"); + System.out.println(formatInteger(nrDatabases.get()) + " databases"); + System.out.println( + formatInteger(nrSuccessfulActions.get()) + " successfully-executed statements"); + System.out.println( + formatInteger(nrUnsuccessfulActions.get()) + " unsuccessfuly-executed statements"); + } + + private String formatInteger(long intValue) { + if (intValue > 1000) { + return String.format("%,9dk", intValue / 1000); + } else { + return String.format("%,10d", intValue); + } + } + })); + } } ExecutorService execService = Executors.newFixedThreadPool(options.getNumberConcurrentThreads()); @@ -499,6 +523,7 @@ private boolean run(MainOptions options, ExecutorService execService, } catch (InterruptedException e) { e.printStackTrace(); } + return threadsShutdown == 0 ? 0 : options.getErrorExitCode(); } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 3e3c52441..d9999076a 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -48,6 +48,9 @@ public class MainOptions { @Parameter(names = "--print-progress-information", description = "Whether to print progress information such as the number of databases generated or queries issued", arity = 1) private boolean printProgressInformation = true; // NOPMD + @Parameter(names = "--print-progress-summary", description = "Whether to print an execution summary when exiting SQLancer", arity = 1) + private boolean printProgressSummary; // NOPMD + @Parameter(names = "--timeout-seconds", description = "The timeout in seconds") private int timeoutSeconds = -1; // NOPMD @@ -135,6 +138,10 @@ public boolean printProgressInformation() { return printProgressInformation; } + public boolean printProgressSummary() { + return printProgressSummary; + } + public int getTimeoutSeconds() { return timeoutSeconds; } From 62b199f90035a46c8cfcc8530eedb6effa6749c4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Jul 2020 23:09:29 +0200 Subject: [PATCH 0018/1284] Add a ExpressionGenerator marker interface --- src/sqlancer/gen/ExpressionGenerator.java | 5 +++++ src/sqlancer/gen/TypedExpressionGenerator.java | 2 +- src/sqlancer/gen/UntypedExpressionGenerator.java | 2 +- src/sqlancer/postgres/gen/PostgresExpressionGenerator.java | 3 ++- src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java | 3 ++- 5 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 src/sqlancer/gen/ExpressionGenerator.java diff --git a/src/sqlancer/gen/ExpressionGenerator.java b/src/sqlancer/gen/ExpressionGenerator.java new file mode 100644 index 000000000..3e49f591e --- /dev/null +++ b/src/sqlancer/gen/ExpressionGenerator.java @@ -0,0 +1,5 @@ +package sqlancer.gen; + +public interface ExpressionGenerator { + +} diff --git a/src/sqlancer/gen/TypedExpressionGenerator.java b/src/sqlancer/gen/TypedExpressionGenerator.java index 462643300..a5c23d867 100644 --- a/src/sqlancer/gen/TypedExpressionGenerator.java +++ b/src/sqlancer/gen/TypedExpressionGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; -public abstract class TypedExpressionGenerator { +public abstract class TypedExpressionGenerator implements ExpressionGenerator { protected List columns = Collections.emptyList(); protected boolean allowAggregates; diff --git a/src/sqlancer/gen/UntypedExpressionGenerator.java b/src/sqlancer/gen/UntypedExpressionGenerator.java index df14fd030..f0c930b8d 100644 --- a/src/sqlancer/gen/UntypedExpressionGenerator.java +++ b/src/sqlancer/gen/UntypedExpressionGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; -public abstract class UntypedExpressionGenerator { +public abstract class UntypedExpressionGenerator implements ExpressionGenerator { protected List columns; protected boolean allowAggregates; diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index d5a0ec3b9..747923a8e 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.gen.ExpressionGenerator; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; @@ -49,7 +50,7 @@ import sqlancer.postgres.ast.PostgresPrefixOperation.PrefixOperator; import sqlancer.postgres.ast.PostgresSimilarTo; -public class PostgresExpressionGenerator { +public class PostgresExpressionGenerator implements ExpressionGenerator { private final int maxDepth; diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index db8600f16..9cae6b73b 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -7,6 +7,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.gen.ExpressionGenerator; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; @@ -43,7 +44,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3RowValue; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3ExpressionGenerator { +public class SQLite3ExpressionGenerator implements ExpressionGenerator { private SQLite3RowValue rw; private final SQLite3GlobalState globalState; From 2082fb96e8ce40b2041f04004e3e66a802cb1ca8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Jul 2020 23:25:23 +0200 Subject: [PATCH 0019/1284] Introduce a generatePredicate() method in the ExpressionGenerator class --- .../clickhouse/gen/ClickHouseExpressionGenerator.java | 5 +++++ .../cockroachdb/gen/CockroachDBExpressionGenerator.java | 5 +++++ src/sqlancer/gen/ExpressionGenerator.java | 7 +++++++ src/sqlancer/gen/UntypedExpressionGenerator.java | 5 +++++ src/sqlancer/postgres/gen/PostgresExpressionGenerator.java | 5 +++++ src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java | 5 +++++ 6 files changed, 32 insertions(+) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 5cabcf358..a99de13e7 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -165,4 +165,9 @@ private ClickHouseExpression getAggregate(ClickHouseDataType dataType) { ClickHouseAggregate.ClickHouseAggregateFunction agg = Randomly.fromList(aggregates); return generateArgsForAggregate(dataType, agg); } + + @Override + public ClickHouseExpression generatePredicate() { + return generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index bc5f765c2..b5ccd0ed1 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -343,4 +343,9 @@ protected CockroachDBExpression generateColumn(CockroachDBCompositeDataType type return columnReference; } + @Override + public CockroachDBExpression generatePredicate() { + return generateExpression(CockroachDBDataType.BOOL.get()); + } + } diff --git a/src/sqlancer/gen/ExpressionGenerator.java b/src/sqlancer/gen/ExpressionGenerator.java index 3e49f591e..98f4290b5 100644 --- a/src/sqlancer/gen/ExpressionGenerator.java +++ b/src/sqlancer/gen/ExpressionGenerator.java @@ -2,4 +2,11 @@ public interface ExpressionGenerator { + /** + * Generates a boolean predicate. + * + * @return an expression that can be used in a boolean context. + */ + E generatePredicate(); + } diff --git a/src/sqlancer/gen/UntypedExpressionGenerator.java b/src/sqlancer/gen/UntypedExpressionGenerator.java index f0c930b8d..8565b15e0 100644 --- a/src/sqlancer/gen/UntypedExpressionGenerator.java +++ b/src/sqlancer/gen/UntypedExpressionGenerator.java @@ -63,4 +63,9 @@ public E generateHavingClause() { return expr; } + @Override + public E generatePredicate() { + return generateExpression(); + } + } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 747923a8e..970650684 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -575,4 +575,9 @@ public PostgresExpressionGenerator allowAggregates(boolean value) { return this; } + @Override + public PostgresExpression generatePredicate() { + return generateExpression(0); + } + } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 9cae6b73b..d90743c22 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -641,4 +641,9 @@ public SQLite3Expression getHavingClause() { return generateExpression(); } + @Override + public SQLite3Expression generatePredicate() { + return generateExpression(); + } + } From c8cab2fb0b1d22ecf2a06a717447332def0ce633 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Jul 2020 23:35:56 +0200 Subject: [PATCH 0020/1284] Introduce a negatePredicate() method in the ExpressionGenerator class --- .../gen/ClickHouseExpressionGenerator.java | 15 ++++++++++----- .../gen/CockroachDBExpressionGenerator.java | 5 +++++ .../duckdb/gen/DuckDBExpressionGenerator.java | 5 +++++ src/sqlancer/gen/ExpressionGenerator.java | 10 ++++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 5 +++++ .../postgres/gen/PostgresExpressionGenerator.java | 5 +++++ .../sqlite3/gen/SQLite3ExpressionGenerator.java | 5 +++++ src/sqlancer/tidb/TiDBExpressionGenerator.java | 5 +++++ 8 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index a99de13e7..ebc4fee2a 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -1,12 +1,15 @@ package sqlancer.clickhouse.gen; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; - import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; @@ -15,12 +18,9 @@ import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; +import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; import sqlancer.gen.TypedExpressionGenerator; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - public class ClickHouseExpressionGenerator extends TypedExpressionGenerator { @@ -170,4 +170,9 @@ private ClickHouseExpression getAggregate(ClickHouseDataType dataType) { public ClickHouseExpression generatePredicate() { return generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); } + + @Override + public ClickHouseExpression negatePredicate(ClickHouseExpression predicate) { + return new ClickHouseUnaryPrefixOperation(predicate, ClickHouseUnaryPrefixOperator.NOT); + } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index b5ccd0ed1..b83a6c514 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -348,4 +348,9 @@ public CockroachDBExpression generatePredicate() { return generateExpression(CockroachDBDataType.BOOL.get()); } + @Override + public CockroachDBExpression negatePredicate(CockroachDBExpression predicate) { + return new CockroachDBNotOperation(predicate); + } + } diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index f00685dde..9dc4d9169 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -433,4 +433,9 @@ public Node generateAggregate() { return generateArgsForAggregate(aggrFunc); } + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, DuckDBUnaryPrefixOperator.NOT); + } + } diff --git a/src/sqlancer/gen/ExpressionGenerator.java b/src/sqlancer/gen/ExpressionGenerator.java index 98f4290b5..c380c9a77 100644 --- a/src/sqlancer/gen/ExpressionGenerator.java +++ b/src/sqlancer/gen/ExpressionGenerator.java @@ -9,4 +9,14 @@ public interface ExpressionGenerator { */ E generatePredicate(); + /** + * Negates a predicate (i.e., uses a NOT operator). + * + * @param predicate + * the boolean predicate. + * + * @return the negated predicate. + */ + E negatePredicate(E predicate); + } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index fb8baf108..ff711122d 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -188,4 +188,9 @@ protected MySQLExpression generateColumn() { return MySQLColumnReference.create(c, val); } + @Override + public MySQLExpression negatePredicate(MySQLExpression predicate) { + return new MySQLUnaryPrefixOperation(predicate, MySQLUnaryPrefixOperator.NOT); + } + } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 970650684..c998dd847 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -580,4 +580,9 @@ public PostgresExpression generatePredicate() { return generateExpression(0); } + @Override + public PostgresExpression negatePredicate(PostgresExpression predicate) { + return new PostgresPrefixOperation(predicate, PostgresPrefixOperation.PrefixOperator.NOT); + } + } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index d90743c22..3e89d5f9d 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -646,4 +646,9 @@ public SQLite3Expression generatePredicate() { return generateExpression(); } + @Override + public SQLite3Expression negatePredicate(SQLite3Expression predicate) { + return new SQLite3UnaryOperation(UnaryOperator.NOT, predicate); + } + } diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index 271a86955..e2c44bace 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -170,4 +170,9 @@ public List generateOrderBys() { return newExpressions; } + @Override + public TiDBExpression negatePredicate(TiDBExpression predicate) { + return new TiDBUnaryPrefixOperation(predicate, TiDBUnaryPrefixOperator.NOT); + } + } From 44b292ae881c576275dfe903585a2154c00cc252 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Jul 2020 23:45:22 +0200 Subject: [PATCH 0021/1284] Introduce an isNull() method in the ExpressionGenerator class --- .../clickhouse/gen/ClickHouseExpressionGenerator.java | 6 ++++++ .../gen/CockroachDBExpressionGenerator.java | 5 +++++ src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java | 5 +++++ src/sqlancer/gen/ExpressionGenerator.java | 10 ++++++++++ src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 5 +++++ .../postgres/gen/PostgresExpressionGenerator.java | 5 +++++ .../sqlite3/gen/SQLite3ExpressionGenerator.java | 6 ++++++ src/sqlancer/tidb/TiDBExpressionGenerator.java | 5 +++++ 8 files changed, 47 insertions(+) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index ebc4fee2a..f5abe276c 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -18,6 +18,7 @@ import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; +import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; import sqlancer.gen.TypedExpressionGenerator; @@ -175,4 +176,9 @@ public ClickHouseExpression generatePredicate() { public ClickHouseExpression negatePredicate(ClickHouseExpression predicate) { return new ClickHouseUnaryPrefixOperation(predicate, ClickHouseUnaryPrefixOperator.NOT); } + + @Override + public ClickHouseExpression isNull(ClickHouseExpression expr) { + return new ClickHouseUnaryPostfixOperation(expr, ClickHouseUnaryPostfixOperator.IS_NULL, false); + } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index b83a6c514..5fe4d6d0d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -353,4 +353,9 @@ public CockroachDBExpression negatePredicate(CockroachDBExpression predicate) { return new CockroachDBNotOperation(predicate); } + @Override + public CockroachDBExpression isNull(CockroachDBExpression expr) { + return new CockroachDBUnaryPostfixOperation(expr, CockroachDBUnaryPostfixOperator.IS_NULL); + } + } diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 9dc4d9169..eebee9d18 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -438,4 +438,9 @@ public Node negatePredicate(Node predicate) return new NewUnaryPrefixOperatorNode<>(predicate, DuckDBUnaryPrefixOperator.NOT); } + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, DuckDBUnaryPostfixOperator.IS_NULL); + } + } diff --git a/src/sqlancer/gen/ExpressionGenerator.java b/src/sqlancer/gen/ExpressionGenerator.java index c380c9a77..e3f2f05ec 100644 --- a/src/sqlancer/gen/ExpressionGenerator.java +++ b/src/sqlancer/gen/ExpressionGenerator.java @@ -19,4 +19,14 @@ public interface ExpressionGenerator { */ E negatePredicate(E predicate); + /** + * Checks if an expression evaluates to NULL (i.e., implements the IS NULL operator). + * + * @param expr + * the expression + * + * @return an expression that checks whether the expression evaluates to NULL. + */ + E isNull(E expr); + } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index ff711122d..76af4a059 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -193,4 +193,9 @@ public MySQLExpression negatePredicate(MySQLExpression predicate) { return new MySQLUnaryPrefixOperation(predicate, MySQLUnaryPrefixOperator.NOT); } + @Override + public MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, MySQLUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + } + } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index c998dd847..308754936 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -585,4 +585,9 @@ public PostgresExpression negatePredicate(PostgresExpression predicate) { return new PostgresPrefixOperation(predicate, PostgresPrefixOperation.PrefixOperator.NOT); } + @Override + public PostgresExpression isNull(PostgresExpression expr) { + return new PostgresPostfixOperation(expr, PostfixOperator.IS_NULL); + } + } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 3e89d5f9d..8963e4366 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -29,6 +29,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; @@ -651,4 +652,9 @@ public SQLite3Expression negatePredicate(SQLite3Expression predicate) { return new SQLite3UnaryOperation(UnaryOperator.NOT, predicate); } + @Override + public SQLite3Expression isNull(SQLite3Expression expr) { + return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, expr); + } + } diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index e2c44bace..7440c0da3 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -175,4 +175,9 @@ public TiDBExpression negatePredicate(TiDBExpression predicate) { return new TiDBUnaryPrefixOperation(predicate, TiDBUnaryPrefixOperator.NOT); } + @Override + public TiDBExpression isNull(TiDBExpression expr) { + return new TiDBUnaryPostfixOperation(expr, TiDBUnaryPostfixOperator.IS_NULL); + } + } From a78e62c5e8a221bfef3fb63f42f4a2d68a5ecef3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2020 08:27:19 +0000 Subject: [PATCH 0022/1284] Bump sqlite-jdbc from 3.32.3 to 3.32.3.1 Bumps [sqlite-jdbc](https://github.com/xerial/sqlite-jdbc) from 3.32.3 to 3.32.3.1. - [Release notes](https://github.com/xerial/sqlite-jdbc/releases) - [Changelog](https://github.com/xerial/sqlite-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/xerial/sqlite-jdbc/compare/3.32.3...3.32.3.1) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f9c0e0c90..9e8dda10a 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ org.xerial sqlite-jdbc - 3.32.3 + 3.32.3.1 mysql From e8b8339cff213ab3198a664e4ec88472a22b76fd Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 15 Jul 2020 19:45:56 +0200 Subject: [PATCH 0023/1284] [MySQL] Fix compiler warnings --- src/sqlancer/mysql/ast/MySQLComputableFunction.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/mysql/ast/MySQLComputableFunction.java b/src/sqlancer/mysql/ast/MySQLComputableFunction.java index d049a8ba3..8cfb5f70c 100644 --- a/src/sqlancer/mysql/ast/MySQLComputableFunction.java +++ b/src/sqlancer/mysql/ast/MySQLComputableFunction.java @@ -43,7 +43,7 @@ public enum MySQLFunction { BIT_COUNT(1, "BIT_COUNT") { @Override - public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression[] args) { + public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression... args) { MySQLConstant arg = evaluatedArgs[0]; if (arg.isNull()) { return MySQLConstant.createNullConstant(); @@ -74,7 +74,7 @@ public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression[] args COALESCE(2, "COALESCE") { @Override - public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { + public MySQLConstant apply(MySQLConstant[] args, MySQLExpression... origArgs) { MySQLConstant result = MySQLConstant.createNullConstant(); for (MySQLConstant arg : args) { if (!arg.isNull()) { @@ -97,7 +97,7 @@ public boolean isVariadic() { IF(3, "IF") { @Override - public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { + public MySQLConstant apply(MySQLConstant[] args, MySQLExpression... origArgs) { MySQLConstant cond = args[0]; MySQLConstant left = args[1]; MySQLConstant right = args[2]; @@ -118,7 +118,7 @@ public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { IFNULL(2, "IFNULL") { @Override - public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { + public MySQLConstant apply(MySQLConstant[] args, MySQLExpression... origArgs) { MySQLConstant result; if (args[0].isNull()) { result = args[1]; @@ -132,14 +132,14 @@ public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { LEAST(2, "LEAST", true) { @Override - public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression[] args) { + public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression... args) { return aggregate(evaluatedArgs, (min, cur) -> cur.isLessThan(min).asBooleanNotNull() ? cur : min); } }, GREATEST(2, "GREATEST", true) { @Override - public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression[] args) { + public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression... args) { return aggregate(evaluatedArgs, (max, cur) -> cur.isLessThan(max).asBooleanNotNull() ? max : cur); } }; From cf6a83eabbc282b80414a4a352fb594fd85e3992 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Fri, 10 Jul 2020 19:07:33 +0300 Subject: [PATCH 0024/1284] standardize postgres error loggs --- src/sqlancer/ComparatorHelper.java | 4 +++- src/sqlancer/StateToReproduce.java | 5 +++++ .../postgres/oracle/PostgresNoRECOracle.java | 11 ++++++++--- .../oracle/PostgresPivotedQuerySynthesisOracle.java | 5 ++++- .../oracle/tlp/PostgresTLPAggregateOracle.java | 12 +++++++++--- 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index d850858ab..e6d545553 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -91,6 +91,7 @@ public static void assumeResultSetsAreEqual(List resultSet, List combinedString.stream().collect(Collectors.joining(";")), secondResultSet.size()); state.getState().statements.add(new QueryAdapter(firstQueryString)); state.getState().statements.add(new QueryAdapter(secondQueryString)); + state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); throw new AssertionError(assertionMessage); @@ -104,12 +105,13 @@ public static void assumeResultSetsAreEqual(List resultSet, List firstResultSetMisses.removeAll(secondHashSet); Set secondResultSetMisses = new HashSet<>(secondHashSet); secondResultSetMisses.removeAll(firstHashSet); - String queryFormatString = "%s; -- misses: %s"; + String queryFormatString = "--%s;\n-- misses: %s"; String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); String secondQueryString = String.format(queryFormatString, combinedString.stream().collect(Collectors.joining(";")), secondResultSetMisses); state.getState().statements.add(new QueryAdapter(firstQueryString)); state.getState().statements.add(new QueryAdapter(secondQueryString)); + state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", firstQueryString, secondQueryString); throw new AssertionError(assertionMessage); diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index ea52d7bbb..7e9d32db3 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -20,6 +20,11 @@ public class StateToReproduce { public final List statements = new ArrayList<>(); + + /** + * The string printed at the bottom of the error log file, which contains + * the queries that caused the test to fail and information about their results. + */ public String queryString; private final String databaseName; diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index b58e1cbed..1478dae81 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -78,9 +78,14 @@ public void check() throws SQLException { throw new IgnoreMeException(); } if (firstCount != secondCount) { - state.queryString = firstCount + " " + secondCount + " " + firstQueryString + ";\n" + secondQueryString - + ";"; - throw new AssertionError(firstQueryString + secondQueryString + firstCount + " " + secondCount); + String queryFormatString = "%s; -- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, firstQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, secondQueryString, secondCount); + state.statements.add(new QueryAdapter(firstQueryStringWithCount)); + state.statements.add(new QueryAdapter(secondQueryStringWithCount)); + state.queryString = String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); } } diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 488b29054..c02ebdadd 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -27,6 +27,7 @@ import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.QueryAdapter; public class PostgresPivotedQuerySynthesisOracle implements TestOracle { @@ -57,7 +58,9 @@ public void check() throws SQLException { boolean isContainedIn = isContainedIn(queryString, options, logger); if (!isContainedIn) { - throw new AssertionError(queryString); + state.statements.add(new QueryAdapter(queryString)); + String assertionMessage = String.format("the query doesn't contain at least 1 row!\n%s", queryString); + throw new AssertionError(assertionMessage); } } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index bb1d95e08..e698d01b3 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -63,15 +63,21 @@ public void check() throws SQLException { metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); secondResult = getAggregateResult(metamorphicQuery); - state.getState().queryString = "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult - + "\n-- " + secondResult; + String queryFormatString = "--%s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().statements.add(new QueryAdapter(firstQueryString)); + state.getState().statements.add(new QueryAdapter(secondQueryString)); + state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); if (firstResult == null && secondResult != null || firstResult != null && (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { if (secondResult.contains("Inf")) { throw new IgnoreMeException(); // FIXME: average computation } - throw new AssertionError(); + String assertionMessage = String.format("the results mismatch!\n%s\n%s", + firstQueryString, secondQueryString); + throw new AssertionError(assertionMessage); } } From 0bfeed32990cbca96df8c15932253bbc9b13fcc7 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Fri, 10 Jul 2020 19:54:50 +0300 Subject: [PATCH 0025/1284] prevent printing last query on error log twice --- src/sqlancer/ComparatorHelper.java | 2 -- src/sqlancer/postgres/oracle/PostgresNoRECOracle.java | 2 -- .../postgres/oracle/PostgresPivotedQuerySynthesisOracle.java | 1 - .../postgres/oracle/tlp/PostgresTLPAggregateOracle.java | 2 -- 4 files changed, 7 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index e6d545553..0e304c313 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -89,8 +89,6 @@ public static void assumeResultSetsAreEqual(List resultSet, List String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); String secondQueryString = String.format(queryFormatString, combinedString.stream().collect(Collectors.joining(";")), secondResultSet.size()); - state.getState().statements.add(new QueryAdapter(firstQueryString)); - state.getState().statements.add(new QueryAdapter(secondQueryString)); state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 1478dae81..f17e87029 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -81,8 +81,6 @@ public void check() throws SQLException { String queryFormatString = "%s; -- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, firstQueryString, firstCount); String secondQueryStringWithCount = String.format(queryFormatString, secondQueryString, secondCount); - state.statements.add(new QueryAdapter(firstQueryStringWithCount)); - state.statements.add(new QueryAdapter(secondQueryStringWithCount)); state.queryString = String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount); String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, firstQueryStringWithCount, secondQueryStringWithCount); throw new AssertionError(assertionMessage); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index c02ebdadd..2d39a3794 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -58,7 +58,6 @@ public void check() throws SQLException { boolean isContainedIn = isContainedIn(queryString, options, logger); if (!isContainedIn) { - state.statements.add(new QueryAdapter(queryString)); String assertionMessage = String.format("the query doesn't contain at least 1 row!\n%s", queryString); throw new AssertionError(assertionMessage); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index e698d01b3..827bd4d77 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -66,8 +66,6 @@ public void check() throws SQLException { String queryFormatString = "--%s;\n-- result: %s"; String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); - state.getState().statements.add(new QueryAdapter(firstQueryString)); - state.getState().statements.add(new QueryAdapter(secondQueryString)); state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); if (firstResult == null && secondResult != null || firstResult != null && (!firstResult.contentEquals(secondResult) From 65ebc55ec9cf03d3e60f55c43f2d1c3bd314596a Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 13 Jul 2020 11:18:45 +0300 Subject: [PATCH 0026/1284] standardize postgres error & current logs, add Aggregate TLP queries to current logs, avoid duplicate TLP queries on error logs, standardize error logs for other oracles --- src/sqlancer/ComparatorHelper.java | 7 +++---- .../postgres/oracle/PostgresNoRECOracle.java | 2 +- .../PostgresPivotedQuerySynthesisOracle.java | 5 +++-- .../oracle/tlp/PostgresTLPAggregateOracle.java | 14 +++++++++++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 0e304c313..93448a9c4 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -85,7 +85,7 @@ public static List getResultSetFirstColumnAsString(String queryString, S public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, String originalQueryString, List combinedString, GlobalState state) { if (resultSet.size() != secondResultSet.size()) { - String queryFormatString = "%s; -- cardinality: %d"; + String queryFormatString = "-- %s;\n-- cardinality: %d"; String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); String secondQueryString = String.format(queryFormatString, combinedString.stream().collect(Collectors.joining(";")), secondResultSet.size()); @@ -103,12 +103,11 @@ public static void assumeResultSetsAreEqual(List resultSet, List firstResultSetMisses.removeAll(secondHashSet); Set secondResultSetMisses = new HashSet<>(secondHashSet); secondResultSetMisses.removeAll(firstHashSet); - String queryFormatString = "--%s;\n-- misses: %s"; + String queryFormatString = "-- %s;\n-- misses: %s"; String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); String secondQueryString = String.format(queryFormatString, combinedString.stream().collect(Collectors.joining(";")), secondResultSetMisses); - state.getState().statements.add(new QueryAdapter(firstQueryString)); - state.getState().statements.add(new QueryAdapter(secondQueryString)); + // update the SELECT queries to be logged at the bottom of the error log file state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", firstQueryString, secondQueryString); diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index f17e87029..f2b5610b8 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -78,7 +78,7 @@ public void check() throws SQLException { throw new IgnoreMeException(); } if (firstCount != secondCount) { - String queryFormatString = "%s; -- count: %d"; + String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, firstQueryString, firstCount); String secondQueryStringWithCount = String.format(queryFormatString, secondQueryString, secondCount); state.queryString = String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 2d39a3794..4d75fa05c 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -58,7 +58,7 @@ public void check() throws SQLException { boolean isContainedIn = isContainedIn(queryString, options, logger); if (!isContainedIn) { - String assertionMessage = String.format("the query doesn't contain at least 1 row!\n%s", queryString); + String assertionMessage = String.format("the query doesn't contain at least 1 row!\n-- %s;", queryString); throw new AssertionError(assertionMessage); } @@ -175,7 +175,8 @@ private boolean isContainedIn(String queryString, MainOptions options, StateLogg } } String resultingQueryString = sb.toString(); - state.queryString = resultingQueryString; + // log both SELECT queries at the bottom of the error log file + state.queryString = String.format("-- %s;\n-- %s;", queryString, resultingQueryString); if (options.logEachSelect()) { logger.writeCurrent(resultingQueryString); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index 827bd4d77..5b86ba00b 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -6,6 +6,7 @@ import java.util.Arrays; import java.util.List; +import java.io.IOException; import org.postgresql.util.PSQLException; import sqlancer.ComparatorHelper; @@ -63,7 +64,7 @@ public void check() throws SQLException { metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); secondResult = getAggregateResult(metamorphicQuery); - String queryFormatString = "--%s;\n-- result: %s"; + String queryFormatString = "-- %s;\n-- result: %s"; String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); @@ -98,6 +99,17 @@ private String createMetamorphicUnionQuery(PostgresSelect select, PostgresAggreg } private String getAggregateResult(String queryString) throws SQLException { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } String resultString; QueryAdapter q = new QueryAdapter(queryString, errors); try (ResultSet result = q.executeAndGet(state)) { From 35dd107f21dac9e7485d46f62af267efdd85ddb7 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 16 Jul 2020 10:19:51 +0300 Subject: [PATCH 0027/1284] clear left-over query string from previous test --- src/sqlancer/postgres/oracle/PostgresNoRECOracle.java | 2 ++ .../postgres/oracle/PostgresPivotedQuerySynthesisOracle.java | 2 ++ src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index f2b5610b8..5d9e210e8 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -62,6 +62,8 @@ public PostgresNoRECOracle(PostgresGlobalState globalState) { @Override public void check() throws SQLException { + // clear left-over query string from previous test + state.queryString = null; PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonFetchErrors(errors); PostgresTables randomTables = s.getRandomTableNonEmptyTables(); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 4d75fa05c..875aa9dd9 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -50,6 +50,8 @@ public PostgresPivotedQuerySynthesisOracle(PostgresGlobalState globalState) thro @Override public void check() throws SQLException { + // clear left-over query string from previous test + state.queryString = null; String queryString = getQueryThatContainsAtLeastOneRow(state); state.queryString = queryString; if (options.logEachSelect()) { diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 22abc849b..9bf1e50bf 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -48,6 +48,8 @@ public PostgresTLPBase(PostgresGlobalState state) { @Override public void check() throws SQLException { + // clear left-over query string from previous test + state.getState().queryString = null; s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new PostgresExpressionGenerator(state).setColumns(targetTables.getColumns()); From 116aaec90a57fe84b86b1611dd376dddfbf82621 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 16 Jul 2020 12:51:18 +0300 Subject: [PATCH 0028/1284] format source code --- src/sqlancer/StateToReproduce.java | 4 ++-- src/sqlancer/postgres/oracle/PostgresNoRECOracle.java | 3 ++- .../postgres/oracle/PostgresPivotedQuerySynthesisOracle.java | 1 - .../postgres/oracle/tlp/PostgresTLPAggregateOracle.java | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 7e9d32db3..b27717589 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -22,8 +22,8 @@ public class StateToReproduce { public final List statements = new ArrayList<>(); /** - * The string printed at the bottom of the error log file, which contains - * the queries that caused the test to fail and information about their results. + * The string printed at the bottom of the error log file, which contains the queries that caused the test to fail + * and information about their results. */ public String queryString; diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 5d9e210e8..29adf56a5 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -84,7 +84,8 @@ public void check() throws SQLException { String firstQueryStringWithCount = String.format(queryFormatString, firstQueryString, firstCount); String secondQueryStringWithCount = String.format(queryFormatString, secondQueryString, secondCount); state.queryString = String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, firstQueryStringWithCount, secondQueryStringWithCount); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, + firstQueryStringWithCount, secondQueryStringWithCount); throw new AssertionError(assertionMessage); } } diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 875aa9dd9..5d3ac521d 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -27,7 +27,6 @@ import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.gen.PostgresExpressionGenerator; -import sqlancer.QueryAdapter; public class PostgresPivotedQuerySynthesisOracle implements TestOracle { diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index 5b86ba00b..b1edd66cd 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -74,8 +74,8 @@ public void check() throws SQLException { if (secondResult.contains("Inf")) { throw new IgnoreMeException(); // FIXME: average computation } - String assertionMessage = String.format("the results mismatch!\n%s\n%s", - firstQueryString, secondQueryString); + String assertionMessage = String.format("the results mismatch!\n%s\n%s", firstQueryString, + secondQueryString); throw new AssertionError(assertionMessage); } From d9fab80e9b52f3decf2f57ae1d842d6450a08cc0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 15 Jul 2020 20:21:19 +0200 Subject: [PATCH 0029/1284] Add a base class for all TLP oracles --- .../TernaryLogicPartitioningOracleBase.java | 13 +++++++++++++ .../clickhouse/oracle/tlp/ClickHouseTLPBase.java | 3 ++- .../cockroachdb/oracle/tlp/CockroachDBTLPBase.java | 4 +++- .../duckdb/test/DuckDBQueryPartitioningBase.java | 4 +++- .../mysql/oracle/MySQLQueryPartitioningBase.java | 4 +++- .../postgres/oracle/tlp/PostgresTLPBase.java | 3 ++- src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java | 3 ++- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 3 ++- 8 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/TernaryLogicPartitioningOracleBase.java diff --git a/src/sqlancer/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/TernaryLogicPartitioningOracleBase.java new file mode 100644 index 000000000..b01d98348 --- /dev/null +++ b/src/sqlancer/TernaryLogicPartitioningOracleBase.java @@ -0,0 +1,13 @@ +package sqlancer; + +/** + * This is the base class of the Ternary Logic Partitioning (TLP) oracles. The core idea of TLP is to partition a given + * so-called original query to three so-called partitioning queries, each of which computes a partition of the original + * query's result. + * + * @param + * the expression type + */ +public abstract class TernaryLogicPartitioningOracleBase { + +} diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 5c86e3bab..80cd0c9b9 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -2,6 +2,7 @@ import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Randomly; +import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; @@ -26,7 +27,7 @@ import java.util.Set; import java.util.stream.Collectors; -public class ClickHouseTLPBase implements TestOracle { +public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { final ClickHouseGlobalState state; final Set errors = new HashSet<>(); diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index d141a68d9..8cfbd4bc8 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -8,6 +8,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -26,7 +27,8 @@ import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; -public class CockroachDBTLPBase implements TestOracle { +public class CockroachDBTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { final CockroachDBGlobalState state; final Set errors = new HashSet<>(); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index d27c2cabe..9147642f0 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -8,6 +8,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; import sqlancer.ast.newast.ColumnReferenceNode; import sqlancer.ast.newast.NewUnaryPostfixOperatorNode; @@ -27,7 +28,8 @@ import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPostfixOperator; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPrefixOperator; -public class DuckDBQueryPartitioningBase implements TestOracle { +public class DuckDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase> + implements TestOracle { final DuckDBGlobalState state; final Set errors = new HashSet<>(); diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index c67018344..5c3a180e8 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -7,6 +7,7 @@ import java.util.Set; import java.util.stream.Collectors; +import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; @@ -22,7 +23,8 @@ import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; import sqlancer.mysql.gen.MySQLExpressionGenerator; -public abstract class MySQLQueryPartitioningBase implements TestOracle { +public abstract class MySQLQueryPartitioningBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { final MySQLGlobalState state; final Set errors = new HashSet<>(); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 9bf1e50bf..740af1c49 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -8,6 +8,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; @@ -27,7 +28,7 @@ import sqlancer.postgres.gen.PostgresExpressionGenerator; import sqlancer.postgres.oracle.PostgresNoRECOracle; -public class PostgresTLPBase implements TestOracle { +public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { final PostgresGlobalState state; final Set errors = new HashSet<>(); diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index 3e5f2e119..1a7f8315b 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -8,6 +8,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -26,7 +27,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3TLPBase implements TestOracle { +public class SQLite3TLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { final SQLite3GlobalState state; final Set errors = new HashSet<>(); diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index 5ccc1d42d..a1ac560d2 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -8,6 +8,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; @@ -26,7 +27,7 @@ import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; import sqlancer.tidb.gen.TiDBHintGenerator; -public abstract class TiDBTLPBase implements TestOracle { +public abstract class TiDBTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { final TiDBGlobalState state; final Set errors = new HashSet<>(); From 6d9136a72a2c7396259c36d05cc1e100879bc118 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 15 Jul 2020 21:35:35 +0200 Subject: [PATCH 0030/1284] Move the ternary predicate variant fields into the TLP base class --- src/sqlancer/TernaryLogicPartitioningOracleBase.java | 7 +++++++ src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java | 3 --- .../cockroachdb/oracle/tlp/CockroachDBTLPBase.java | 3 --- src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java | 3 --- src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java | 3 --- src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 3 --- src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java | 3 --- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 3 --- 8 files changed, 7 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/TernaryLogicPartitioningOracleBase.java index b01d98348..bdbd1c4d2 100644 --- a/src/sqlancer/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/TernaryLogicPartitioningOracleBase.java @@ -10,4 +10,11 @@ */ public abstract class TernaryLogicPartitioningOracleBase { + protected E predicate; + protected E negatedPredicate; + protected E isNullPredicate; + + protected TernaryLogicPartitioningOracleBase() { + } + } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 80cd0c9b9..993baa8ee 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -36,9 +36,6 @@ public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase predicate; - Node negatedPredicate; - Node isNullPredicate; public DuckDBQueryPartitioningBase(DuckDBGlobalState state) { this.state = state; diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index 5c3a180e8..b22b5721c 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -33,9 +33,6 @@ public abstract class MySQLQueryPartitioningBase extends TernaryLogicPartitionin MySQLTables targetTables; MySQLExpressionGenerator gen; MySQLSelect select; - MySQLExpression predicate; - MySQLExpression negatedPredicate; - MySQLExpression isNullPredicate; public MySQLQueryPartitioningBase(MySQLGlobalState state) { this.state = state; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 740af1c49..1be289b07 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -37,9 +37,6 @@ public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase Date: Wed, 15 Jul 2020 21:44:05 +0200 Subject: [PATCH 0031/1284] Move the errors fields into the TLP base class --- .../TernaryLogicPartitioningOracleBase.java | 5 +++++ .../oracle/tlp/ClickHouseTLPBase.java | 19 ++++++++----------- .../oracle/tlp/CockroachDBTLPBase.java | 3 --- .../test/DuckDBQueryPartitioningBase.java | 3 --- .../oracle/MySQLQueryPartitioningBase.java | 3 --- .../postgres/oracle/tlp/PostgresTLPBase.java | 3 --- .../sqlite3/oracle/tlp/SQLite3TLPBase.java | 3 --- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 3 --- 8 files changed, 13 insertions(+), 29 deletions(-) diff --git a/src/sqlancer/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/TernaryLogicPartitioningOracleBase.java index bdbd1c4d2..822e19481 100644 --- a/src/sqlancer/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/TernaryLogicPartitioningOracleBase.java @@ -1,5 +1,8 @@ package sqlancer; +import java.util.HashSet; +import java.util.Set; + /** * This is the base class of the Ternary Logic Partitioning (TLP) oracles. The core idea of TLP is to partition a given * so-called original query to three so-called partitioning queries, each of which computes a partition of the original @@ -14,6 +17,8 @@ public abstract class TernaryLogicPartitioningOracleBase { protected E negatedPredicate; protected E isNullPredicate; + protected final Set errors = new HashSet<>(); + protected TernaryLogicPartitioningOracleBase() { } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 993baa8ee..20c820538 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -1,11 +1,19 @@ package sqlancer.clickhouse.oracle.tlp; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Randomly; import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; @@ -16,21 +24,10 @@ import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { final ClickHouseGlobalState state; - final Set errors = new HashSet<>(); ClickHouseSchema s; ClickHouseTables targetTables; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index cac1dea0f..610d28aa8 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -2,9 +2,7 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -31,7 +29,6 @@ public class CockroachDBTLPBase extends TernaryLogicPartitioningOracleBase errors = new HashSet<>(); CockroachDBSchema s; CockroachDBTables targetTables; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index a0c742a45..b56a0de0c 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -2,9 +2,7 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -32,7 +30,6 @@ public class DuckDBQueryPartitioningBase extends TernaryLogicPartitioningOracleB implements TestOracle { final DuckDBGlobalState state; - final Set errors = new HashSet<>(); DuckDBSchema s; DuckDBTables targetTables; diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index b22b5721c..bfb246da0 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -2,9 +2,7 @@ import java.sql.SQLException; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.TernaryLogicPartitioningOracleBase; @@ -27,7 +25,6 @@ public abstract class MySQLQueryPartitioningBase extends TernaryLogicPartitionin implements TestOracle { final MySQLGlobalState state; - final Set errors = new HashSet<>(); MySQLSchema s; MySQLTables targetTables; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 1be289b07..4d396cb39 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -2,9 +2,7 @@ import java.sql.SQLException; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -31,7 +29,6 @@ public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { final PostgresGlobalState state; - final Set errors = new HashSet<>(); PostgresSchema s; PostgresTables targetTables; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index e97249a0f..e0eddbbe2 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -2,9 +2,7 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -30,7 +28,6 @@ public class SQLite3TLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { final SQLite3GlobalState state; - final Set errors = new HashSet<>(); SQLite3Schema s; SQLite3Tables targetTables; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index 4f0392e21..032bc560e 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -2,9 +2,7 @@ import java.sql.SQLException; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -30,7 +28,6 @@ public abstract class TiDBTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { final TiDBGlobalState state; - final Set errors = new HashSet<>(); TiDBSchema s; TiDBTables targetTables; From f76f4dd651072038299f1e8ae2601707f974afba Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 15 Jul 2020 22:09:47 +0200 Subject: [PATCH 0032/1284] Initialize the ternary predicate variants in the TLP base class --- .../TernaryLogicPartitioningOracleBase.java | 25 ++++++++++++++++++- .../oracle/tlp/ClickHouseTLPBase.java | 14 +++++------ .../oracle/tlp/CockroachDBTLPBase.java | 13 +++++----- .../test/DuckDBQueryPartitioningBase.java | 14 +++++------ .../oracle/MySQLQueryPartitioningBase.java | 14 +++++------ .../postgres/oracle/tlp/PostgresTLPBase.java | 12 ++++----- .../sqlite3/oracle/tlp/SQLite3TLPBase.java | 14 +++++------ src/sqlancer/tidb/oracle/TiDBTLPBase.java | 14 +++++------ 8 files changed, 72 insertions(+), 48 deletions(-) diff --git a/src/sqlancer/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/TernaryLogicPartitioningOracleBase.java index 822e19481..4e23989fa 100644 --- a/src/sqlancer/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/TernaryLogicPartitioningOracleBase.java @@ -3,6 +3,8 @@ import java.util.HashSet; import java.util.Set; +import sqlancer.gen.ExpressionGenerator; + /** * This is the base class of the Ternary Logic Partitioning (TLP) oracles. The core idea of TLP is to partition a given * so-called original query to three so-called partitioning queries, each of which computes a partition of the original @@ -11,7 +13,7 @@ * @param * the expression type */ -public abstract class TernaryLogicPartitioningOracleBase { +public abstract class TernaryLogicPartitioningOracleBase implements TestOracle { protected E predicate; protected E negatedPredicate; @@ -22,4 +24,25 @@ public abstract class TernaryLogicPartitioningOracleBase { protected TernaryLogicPartitioningOracleBase() { } + protected void initializeTernaryPredicateVariants() { + ExpressionGenerator gen = getGen(); + if (gen == null) { + throw new IllegalStateException(); + } + predicate = gen.generatePredicate(); + if (predicate == null) { + throw new IllegalStateException(); + } + negatedPredicate = gen.negatePredicate(predicate); + if (negatedPredicate == null) { + throw new IllegalStateException(); + } + isNullPredicate = gen.isNull(predicate); + if (isNullPredicate == null) { + throw new IllegalStateException(); + } + } + + protected abstract ExpressionGenerator getGen(); + } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 20c820538..e469730e6 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -18,12 +18,9 @@ import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; -import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator; -import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; +import sqlancer.gen.ExpressionGenerator; public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { @@ -45,6 +42,7 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new ClickHouseExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new ClickHouseSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); @@ -53,9 +51,6 @@ public void check() throws SQLException { select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); select.setFromTables(tableRefs); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new ClickHouseUnaryPrefixOperation(predicate, ClickHouseUnaryPrefixOperator.NOT); - isNullPredicate = new ClickHouseUnaryPostfixOperation(predicate, ClickHouseUnaryPostfixOperator.IS_NULL, false); } List generateFetchColumns() { @@ -69,4 +64,9 @@ ClickHouseExpression generatePredicate() { return gen.generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); } + @Override + protected ExpressionGenerator getGen() { + return gen; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index 610d28aa8..48d5598af 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -17,13 +17,11 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; import sqlancer.cockroachdb.ast.CockroachDBColumnReference; import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBNotOperation; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; +import sqlancer.gen.ExpressionGenerator; public class CockroachDBTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { @@ -45,6 +43,7 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new CockroachDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new CockroachDBSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); @@ -54,9 +53,6 @@ public void check() throws SQLException { select.setJoinList(joins); select.setFromList(tableList); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new CockroachDBNotOperation(predicate); - isNullPredicate = new CockroachDBUnaryPostfixOperation(predicate, CockroachDBUnaryPostfixOperator.IS_NULL); } List generateFetchColumns() { @@ -74,4 +70,9 @@ CockroachDBExpression generatePredicate() { return gen.generateExpression(CockroachDBDataType.BOOL.get()); } + @Override + protected ExpressionGenerator getGen() { + return gen; + } + } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index b56a0de0c..d99ddb6a9 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -9,8 +9,6 @@ import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; import sqlancer.ast.newast.ColumnReferenceNode; -import sqlancer.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.ast.newast.Node; import sqlancer.ast.newast.TableReferenceNode; import sqlancer.duckdb.DuckDBErrors; @@ -23,8 +21,7 @@ import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBSelect; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPostfixOperator; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPrefixOperator; +import sqlancer.gen.ExpressionGenerator; public class DuckDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase> implements TestOracle { @@ -46,6 +43,7 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new DuckDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new DuckDBSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); @@ -55,9 +53,6 @@ public void check() throws SQLException { select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new NewUnaryPrefixOperatorNode<>(predicate, DuckDBUnaryPrefixOperator.NOT); - isNullPredicate = new NewUnaryPostfixOperatorNode<>(predicate, DuckDBUnaryPostfixOperator.IS_NULL); } List> generateFetchColumns() { @@ -75,4 +70,9 @@ Node generatePredicate() { return gen.generateExpression(); } + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + } diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index bfb246da0..551a779c7 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -7,6 +7,7 @@ import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; +import sqlancer.gen.ExpressionGenerator; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -16,9 +17,6 @@ import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLTableReference; -import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; import sqlancer.mysql.gen.MySQLExpressionGenerator; public abstract class MySQLQueryPartitioningBase extends TernaryLogicPartitioningOracleBase @@ -41,6 +39,7 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new MySQLExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new MySQLSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); @@ -50,10 +49,6 @@ public void check() throws SQLException { select.setFromList(tableList); select.setWhereClause(null); // select.setJoins(joins); - predicate = generatePredicate(); - negatedPredicate = new MySQLUnaryPrefixOperation(predicate, MySQLUnaryPrefixOperator.NOT); - isNullPredicate = new MySQLUnaryPostfixOperation(predicate, - MySQLUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); } List generateFetchColumns() { @@ -64,4 +59,9 @@ MySQLExpression generatePredicate() { return gen.generateExpression(); } + @Override + protected ExpressionGenerator getGen() { + return gen; + } + } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 4d396cb39..304e79818 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -8,6 +8,7 @@ import sqlancer.Randomly; import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; +import sqlancer.gen.ExpressionGenerator; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -16,9 +17,6 @@ import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresPostfixOperation; -import sqlancer.postgres.ast.PostgresPostfixOperation.PostfixOperator; -import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.ForClause; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; @@ -58,9 +56,6 @@ public void check() throws SQLException { select.setFromList(tableList); select.setWhereClause(null); select.setJoinClauses(joins); - predicate = generatePredicate(); - negatedPredicate = new PostgresPrefixOperation(predicate, PostgresPrefixOperation.PrefixOperator.NOT); - isNullPredicate = new PostgresPostfixOperation(predicate, PostfixOperator.IS_NULL); if (Randomly.getBoolean()) { select.setForClause(ForClause.getRandom()); } @@ -74,4 +69,9 @@ PostgresExpression generatePredicate() { return gen.generateExpression(PostgresDataType.BOOLEAN); } + @Override + protected ExpressionGenerator getGen() { + return gen; + } + } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index e0eddbbe2..969a26453 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -8,16 +8,13 @@ import sqlancer.Randomly; import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; +import sqlancer.gen.ExpressionGenerator; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.ast.SQLite3UnaryOperation; -import sqlancer.sqlite3.ast.SQLite3UnaryOperation.UnaryOperator; import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -45,6 +42,7 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new SQLite3ExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new SQLite3Select(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); @@ -53,9 +51,6 @@ public void check() throws SQLException { select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); select.setFromTables(tableRefs); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new SQLite3UnaryOperation(UnaryOperator.NOT, predicate); - isNullPredicate = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, predicate); } List generateFetchColumns() { @@ -73,4 +68,9 @@ SQLite3Expression generatePredicate() { return gen.generateExpression(); } + @Override + protected ExpressionGenerator getGen() { + return gen; + } + } diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index 032bc560e..43c31bab4 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -8,6 +8,7 @@ import sqlancer.Randomly; import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; +import sqlancer.gen.ExpressionGenerator; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -19,10 +20,6 @@ import sqlancer.tidb.ast.TiDBJoin; import sqlancer.tidb.ast.TiDBSelect; import sqlancer.tidb.ast.TiDBTableReference; -import sqlancer.tidb.ast.TiDBUnaryPostfixOperation; -import sqlancer.tidb.ast.TiDBUnaryPostfixOperation.TiDBUnaryPostfixOperator; -import sqlancer.tidb.ast.TiDBUnaryPrefixOperation; -import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; import sqlancer.tidb.gen.TiDBHintGenerator; public abstract class TiDBTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { @@ -44,6 +41,7 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new TiDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new TiDBSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); @@ -57,9 +55,6 @@ public void check() throws SQLException { select.setJoinList(joins); select.setFromList(tableList); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new TiDBUnaryPrefixOperation(predicate, TiDBUnaryPrefixOperator.NOT); - isNullPredicate = new TiDBUnaryPostfixOperation(predicate, TiDBUnaryPostfixOperator.IS_NULL); } List generateFetchColumns() { @@ -70,4 +65,9 @@ TiDBExpression generatePredicate() { return gen.generateExpression(); } + @Override + protected ExpressionGenerator getGen() { + return gen; + } + } From 409ba8c751a3f7d87dbcb32482c97fd1a00ce0b0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 15 Jul 2020 22:41:02 +0200 Subject: [PATCH 0033/1284] Move the global state into the TLP base class --- src/sqlancer/TernaryLogicPartitioningOracleBase.java | 6 ++++-- src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java | 7 +++---- .../cockroachdb/oracle/tlp/CockroachDBTLPBase.java | 8 +++----- src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java | 8 +++----- src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java | 8 +++----- src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 8 ++++---- src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java | 7 +++---- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 7 +++---- 8 files changed, 26 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/TernaryLogicPartitioningOracleBase.java index 4e23989fa..28f04d844 100644 --- a/src/sqlancer/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/TernaryLogicPartitioningOracleBase.java @@ -13,15 +13,17 @@ * @param * the expression type */ -public abstract class TernaryLogicPartitioningOracleBase implements TestOracle { +public abstract class TernaryLogicPartitioningOracleBase implements TestOracle { protected E predicate; protected E negatedPredicate; protected E isNullPredicate; + protected final S state; protected final Set errors = new HashSet<>(); - protected TernaryLogicPartitioningOracleBase() { + protected TernaryLogicPartitioningOracleBase(S state) { + this.state = state; } protected void initializeTernaryPredicateVariants() { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index e469730e6..999d3c1e4 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -22,9 +22,8 @@ import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; import sqlancer.gen.ExpressionGenerator; -public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { - - final ClickHouseGlobalState state; +public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { ClickHouseSchema s; ClickHouseTables targetTables; @@ -32,7 +31,7 @@ public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - final CockroachDBGlobalState state; +public class CockroachDBTLPBase extends + TernaryLogicPartitioningOracleBase implements TestOracle { CockroachDBSchema s; CockroachDBTables targetTables; @@ -34,7 +32,7 @@ public class CockroachDBTLPBase extends TernaryLogicPartitioningOracleBase> - implements TestOracle { - - final DuckDBGlobalState state; +public class DuckDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, DuckDBGlobalState> implements TestOracle { DuckDBSchema s; DuckDBTables targetTables; @@ -34,7 +32,7 @@ public class DuckDBQueryPartitioningBase extends TernaryLogicPartitioningOracleB DuckDBSelect select; public DuckDBQueryPartitioningBase(DuckDBGlobalState state) { - this.state = state; + super(state); DuckDBErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index 551a779c7..e6f776205 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -19,10 +19,8 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.gen.MySQLExpressionGenerator; -public abstract class MySQLQueryPartitioningBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - final MySQLGlobalState state; +public abstract class MySQLQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase implements TestOracle { MySQLSchema s; MySQLTables targetTables; @@ -30,7 +28,7 @@ public abstract class MySQLQueryPartitioningBase extends TernaryLogicPartitionin MySQLSelect select; public MySQLQueryPartitioningBase(MySQLGlobalState state) { - this.state = state; + super(state); MySQLErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 304e79818..df189259d 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -24,9 +24,8 @@ import sqlancer.postgres.gen.PostgresExpressionGenerator; import sqlancer.postgres.oracle.PostgresNoRECOracle; -public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { - - final PostgresGlobalState state; +public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { PostgresSchema s; PostgresTables targetTables; @@ -34,7 +33,7 @@ public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase tables = targetTables.getTables(); diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index 969a26453..ebe8a3af1 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -22,9 +22,8 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3TLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { - - final SQLite3GlobalState state; +public class SQLite3TLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { SQLite3Schema s; SQLite3Tables targetTables; @@ -32,7 +31,7 @@ public class SQLite3TLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { - - final TiDBGlobalState state; +public abstract class TiDBTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { TiDBSchema s; TiDBTables targetTables; @@ -32,7 +31,7 @@ public abstract class TiDBTLPBase extends TernaryLogicPartitioningOracleBase Date: Wed, 15 Jul 2020 23:16:39 +0200 Subject: [PATCH 0034/1284] Introduce an overridable generatePredicate() method in the TLP base class --- src/sqlancer/TernaryLogicPartitioningOracleBase.java | 6 +++++- src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java | 5 ----- src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java | 5 ----- .../cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java | 2 +- src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java | 4 ---- .../duckdb/test/DuckDBQueryPartitioningHavingTester.java | 2 +- src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java | 4 ---- src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 5 ----- .../postgres/oracle/tlp/PostgresTLPHavingOracle.java | 2 +- src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java | 4 ---- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 4 ---- src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java | 2 +- 12 files changed, 9 insertions(+), 36 deletions(-) diff --git a/src/sqlancer/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/TernaryLogicPartitioningOracleBase.java index 28f04d844..dca63bb45 100644 --- a/src/sqlancer/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/TernaryLogicPartitioningOracleBase.java @@ -26,12 +26,16 @@ protected TernaryLogicPartitioningOracleBase(S state) { this.state = state; } + protected E generatePredicate() { + return getGen().generatePredicate(); + } + protected void initializeTernaryPredicateVariants() { ExpressionGenerator gen = getGen(); if (gen == null) { throw new IllegalStateException(); } - predicate = gen.generatePredicate(); + predicate = generatePredicate(); if (predicate == null) { throw new IllegalStateException(); } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 999d3c1e4..39462589f 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.stream.Collectors; -import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Randomly; import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; @@ -59,10 +58,6 @@ List generateFetchColumns() { return columns; } - ClickHouseExpression generatePredicate() { - return gen.generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); - } - @Override protected ExpressionGenerator getGen() { return gen; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index ca3c8c6f7..4cf51ab62 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -12,7 +12,6 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; import sqlancer.cockroachdb.ast.CockroachDBColumnReference; @@ -64,10 +63,6 @@ List generateFetchColumns() { return columns; } - CockroachDBExpression generatePredicate() { - return gen.generateExpression(CockroachDBDataType.BOOL.get()); - } - @Override protected ExpressionGenerator getGen() { return gen; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java index 5aeb8e29b..d5fcf9812 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java @@ -47,7 +47,7 @@ public void check() throws SQLException { } @Override - CockroachDBExpression generatePredicate() { + protected CockroachDBExpression generatePredicate() { return gen.generateHavingClause(); } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 6bc8cc56f..6e8cee2d8 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -64,10 +64,6 @@ List> generateFetchColumns() { return columns; } - Node generatePredicate() { - return gen.generateExpression(); - } - @Override protected ExpressionGenerator> getGen() { return gen; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index a6b625463..4215bf080 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -50,7 +50,7 @@ public void check() throws SQLException { } @Override - Node generatePredicate() { + protected Node generatePredicate() { return gen.generateHavingClause(); } diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index e6f776205..c01b39aea 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -53,10 +53,6 @@ List generateFetchColumns() { return Arrays.asList(MySQLColumnReference.create(targetTables.getColumns().get(0), null)); } - MySQLExpression generatePredicate() { - return gen.generateExpression(); - } - @Override protected ExpressionGenerator getGen() { return gen; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index df189259d..3b74340a6 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -11,7 +11,6 @@ import sqlancer.gen.ExpressionGenerator; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresSchema.PostgresTables; import sqlancer.postgres.ast.PostgresColumnValue; @@ -65,10 +64,6 @@ List generateFetchColumns() { return Arrays.asList(new PostgresColumnValue(targetTables.getColumns().get(0), null)); } - PostgresExpression generatePredicate() { - return gen.generateExpression(PostgresDataType.BOOLEAN); - } - @Override protected ExpressionGenerator getGen() { return gen; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java index 6635ed0fa..01a0d57f5 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java @@ -48,7 +48,7 @@ public void check() throws SQLException { } @Override - PostgresExpression generatePredicate() { + protected PostgresExpression generatePredicate() { return gen.generateHavingClause(); } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index ebe8a3af1..7118c9df0 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -63,10 +63,6 @@ List generateFetchColumns() { return columns; } - SQLite3Expression generatePredicate() { - return gen.generateExpression(); - } - @Override protected ExpressionGenerator getGen() { return gen; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index 7443eb85b..5e05fe9e2 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -60,10 +60,6 @@ List generateFetchColumns() { return Arrays.asList(new TiDBColumnReference(targetTables.getColumns().get(0))); } - TiDBExpression generatePredicate() { - return gen.generateExpression(); - } - @Override protected ExpressionGenerator getGen() { return gen; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java index 74b92cd78..5e5126420 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java @@ -48,7 +48,7 @@ public void check() throws SQLException { } @Override - TiDBExpression generatePredicate() { + protected TiDBExpression generatePredicate() { return gen.generateHavingClause(); } } From c2f83b05f50523b31e5ed55b8d2fe3a62729869a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 15 Jul 2020 23:42:45 +0200 Subject: [PATCH 0035/1284] [Postgres] Fix the predicate generation --- src/sqlancer/postgres/gen/PostgresExpressionGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 308754936..ce9b6bce8 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -577,7 +577,7 @@ public PostgresExpressionGenerator allowAggregates(boolean value) { @Override public PostgresExpression generatePredicate() { - return generateExpression(0); + return generateExpression(PostgresDataType.BOOLEAN); } @Override From abe2d9dc584b9ec6c59356070d8c8a3f632a616c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 16 Jul 2020 13:52:11 +0200 Subject: [PATCH 0036/1284] Add a common NoREC base --- src/sqlancer/NoRECBase.java | 26 ++++++++ .../oracle/CockroachDBNoRECOracle.java | 35 +++++------ .../duckdb/test/DuckDBNoRECOracle.java | 44 +++++--------- .../mariadb/oracle/MariaDBNoRECOracle.java | 39 ++++-------- .../postgres/oracle/PostgresNoRECOracle.java | 60 +++++++------------ .../sqlite3/oracle/SQLite3NoRECOracle.java | 44 +++++--------- 6 files changed, 104 insertions(+), 144 deletions(-) create mode 100644 src/sqlancer/NoRECBase.java diff --git a/src/sqlancer/NoRECBase.java b/src/sqlancer/NoRECBase.java new file mode 100644 index 000000000..92ec27b0e --- /dev/null +++ b/src/sqlancer/NoRECBase.java @@ -0,0 +1,26 @@ +package sqlancer; + +import java.sql.Connection; +import java.util.HashSet; +import java.util.Set; + +import sqlancer.Main.StateLogger; + +public abstract class NoRECBase> implements TestOracle { + + protected final S state; + protected final Set errors = new HashSet<>(); + protected final StateLogger logger; + protected final MainOptions options; + protected final Connection con; + protected String optimizedQueryString; + protected String unoptimizedQueryString; + + public NoRECBase(S state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } + +} diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 1a331232e..6ce80fb73 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -4,13 +4,13 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; +import sqlancer.NoRECBase; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -30,31 +30,26 @@ import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -public class CockroachDBNoRECOracle implements TestOracle { +public class CockroachDBNoRECOracle extends NoRECBase implements TestOracle { - private final CockroachDBGlobalState globalState; - private final Set errors = new HashSet<>(); - private String optimizableQueryString; - private String unoptimizedQuery; private CockroachDBExpressionGenerator gen; public CockroachDBNoRECOracle(CockroachDBGlobalState globalState) { - this.globalState = globalState; + super(globalState); CockroachDBErrors.addExpressionErrors(errors); CockroachDBErrors.addTransactionErrors(errors); errors.add("unable to vectorize execution plan"); // SET vectorize=experimental_always; errors.add(" mismatched physical types at index"); // SET vectorize=experimental_always; - } @Override public void check() throws SQLException { - CockroachDBTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + CockroachDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); List tableL = tables.getTables().stream().map(t -> new CockroachDBTableReference(t)) .collect(Collectors.toList()); List tableList = CockroachDBCommon.getTableReferences(tableL); - gen = new CockroachDBExpressionGenerator(globalState).setColumns(tables.getColumns()); - List joinExpressions = getJoins(tableList, globalState); + gen = new CockroachDBExpressionGenerator(state).setColumns(tables.getColumns()); + List joinExpressions = getJoins(tableList, state); CockroachDBExpression whereCondition = gen.generateExpression(CockroachDBDataType.BOOL.get()); int optimizableCount = getOptimizedResult(whereCondition, tableList, errors, joinExpressions); if (optimizableCount == -1) { @@ -65,7 +60,7 @@ public void check() throws SQLException { throw new IgnoreMeException(); } if (optimizableCount != nonOptimizableCount) { - globalState.getState().queryString = optimizableQueryString + ";\n" + unoptimizedQuery + ";"; + state.getState().queryString = optimizedQueryString + ";\n" + unoptimizedQueryString + ";"; throw new AssertionError(CockroachDBVisitor.asString(whereCondition)); } } @@ -114,12 +109,12 @@ private int getOptimizedResult(CockroachDBExpression whereCondition, List tableList, @@ -132,12 +127,12 @@ private int getNonOptimizedResult(CockroachDBExpression whereCondition, List CockroachDBVisitor.asString(j)).collect(Collectors.joining(", ")) + ")"; - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(s); + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(s); } - this.unoptimizedQuery = s; + this.unoptimizedQueryString = s; Query q = new QueryAdapter(s, errors); - return getCount(globalState, q); + return getCount(state, q); } private int getCount(GlobalState globalState, Query q) throws AssertionError { diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 895b65ae5..148551b97 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -5,14 +5,11 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; +import sqlancer.NoRECBase; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -36,38 +33,26 @@ import sqlancer.duckdb.gen.DuckDBExpressionGenerator; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBCastOperation; -public class DuckDBNoRECOracle implements TestOracle { +public class DuckDBNoRECOracle extends NoRECBase implements TestOracle { private final DuckDBSchema s; - private final Connection con; - private String firstQueryString; - private String secondQueryString; - private final StateLogger logger; - private final MainOptions options; - private final Set errors = new HashSet<>(); - private final DuckDBGlobalState globalState; public DuckDBNoRECOracle(DuckDBGlobalState globalState) { + super(globalState); this.s = globalState.getSchema(); - this.con = globalState.getConnection(); - this.logger = globalState.getLogger(); - this.options = globalState.getOptions(); - this.globalState = globalState; DuckDBErrors.addExpressionErrors(errors); } @Override public void check() throws SQLException { - // DuckDBCommon.addCommonExpressionErrors(errors); - // DuckDBCommon.addCommonFetchErrors(errors); DuckDBTables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState).setColumns(columns); + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(state).setColumns(columns); Node randomWhereCondition = gen.generateExpression(); List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, globalState); + List> joins = DuckDBJoin.getJoins(tableList, state); int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, randomWhereCondition, joins); @@ -76,7 +61,7 @@ public void check() throws SQLException { } if (firstCount != secondCount) { throw new AssertionError( - firstQueryString + "; -- " + firstCount + "\n" + secondQueryString + " -- " + secondCount); + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); } } @@ -95,14 +80,14 @@ private int getSecondQuery(List> tableList, Node> tabl select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( - new DuckDBExpressionGenerator(globalState).setColumns(columns).generateOrderBys()); + select.setOrderByExpressions(new DuckDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); } // select.setSelectType(SelectType.ALL); select.setJoinList(joins); int firstCount = 0; try (Statement stat = con.createStatement()) { - firstQueryString = DuckDBToStringVisitor.asString(select); + optimizedQueryString = DuckDBToStringVisitor.asString(select); if (options.logEachSelect()) { - logger.writeCurrent(firstQueryString); + logger.writeCurrent(optimizedQueryString); } - try (ResultSet rs = stat.executeQuery(firstQueryString)) { + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { while (rs.next()) { firstCount++; } diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index 68a4e8585..4f7b9b413 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -1,17 +1,14 @@ package sqlancer.mariadb.oracle; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import sqlancer.IgnoreMeException; +import sqlancer.NoRECBase; import sqlancer.QueryAdapter; -import sqlancer.Randomly; -import sqlancer.StateToReproduce; import sqlancer.TestOracle; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema; @@ -30,24 +27,14 @@ import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; -public class MariaDBNoRECOracle implements TestOracle { +public class MariaDBNoRECOracle extends NoRECBase implements TestOracle { private final MariaDBSchema s; - private final Randomly r; - private final Connection con; - private String firstQueryString; - private String secondQueryString; - private final List errors = new ArrayList<>(); private static final int NOT_FOUND = -1; - private final StateToReproduce state; - private final MariaDBGlobalState globalState; public MariaDBNoRECOracle(MariaDBGlobalState globalState) { + super(globalState); this.s = globalState.getSchema(); - this.r = globalState.getRandomly(); - this.con = globalState.getConnection(); - this.state = globalState.getState(); - this.globalState = globalState; errors.add("is out of range"); // regex errors.add("unmatched parentheses"); @@ -66,8 +53,8 @@ public MariaDBNoRECOracle(MariaDBGlobalState globalState) { public void check() throws SQLException { MariaDBTable randomTable = s.getRandomTable(); List columns = randomTable.getColumns(); - MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(r).setColumns(columns).setCon(con) - .setState(state); + MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(columns) + .setCon(con).setState(state.getState()); MariaDBExpression randomWhereCondition = gen.getRandomExpression(); List groupBys = Collections.emptyList(); // getRandomExpressions(columns); int optimizedCount = getOptimizedQuery(randomTable, randomWhereCondition, groupBys); @@ -76,7 +63,7 @@ public void check() throws SQLException { throw new IgnoreMeException(); } if (optimizedCount != unoptimizedCount) { - state.queryString = firstQueryString + ";\n" + secondQueryString + ";"; + state.getState().queryString = optimizedQueryString + ";\n" + unoptimizedQueryString + ";"; throw new AssertionError(optimizedCount + " " + unoptimizedCount); } } @@ -93,9 +80,9 @@ private int getUnoptimizedQuery(MariaDBTable randomTable, MariaDBExpression rand select.setSelectType(MariaDBSelectType.ALL); int secondCount = 0; - secondQueryString = "SELECT SUM(count) FROM (" + MariaDBVisitor.asString(select) + ") as asdf"; - QueryAdapter q = new QueryAdapter(secondQueryString, errors); - try (ResultSet rs = q.executeAndGet(globalState)) { + unoptimizedQueryString = "SELECT SUM(count) FROM (" + MariaDBVisitor.asString(select) + ") as asdf"; + QueryAdapter q = new QueryAdapter(unoptimizedQueryString, errors); + try (ResultSet rs = q.executeAndGet(state)) { if (rs == null) { return NOT_FOUND; } else { @@ -122,9 +109,9 @@ private int getOptimizedQuery(MariaDBTable randomTable, MariaDBExpression random select.setWhereClause(randomWhereCondition); select.setSelectType(MariaDBSelectType.ALL); int firstCount = 0; - firstQueryString = MariaDBVisitor.asString(select); - QueryAdapter q = new QueryAdapter(firstQueryString, errors); - try (ResultSet rs = q.executeAndGet(globalState)) { + optimizedQueryString = MariaDBVisitor.asString(select); + QueryAdapter q = new QueryAdapter(optimizedQueryString, errors); + try (ResultSet rs = q.executeAndGet(state)) { if (rs == null) { firstCount = NOT_FOUND; } else { @@ -133,7 +120,7 @@ private int getOptimizedQuery(MariaDBTable randomTable, MariaDBExpression random rs.getStatement().close(); } } catch (Exception e) { - throw new AssertionError(firstQueryString, e); + throw new AssertionError(optimizedQueryString, e); } return firstCount; } diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 29adf56a5..d838ff302 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -1,23 +1,18 @@ package sqlancer.postgres.oracle; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; +import sqlancer.NoRECBase; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.StateToReproduce.PostgresStateToReproduce; import sqlancer.TestOracle; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; @@ -39,39 +34,27 @@ import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class PostgresNoRECOracle implements TestOracle { +public class PostgresNoRECOracle extends NoRECBase implements TestOracle { private final PostgresSchema s; - private final Connection con; - private final PostgresStateToReproduce state; - private String firstQueryString; - private String secondQueryString; - private final StateLogger logger; - private final MainOptions options; - private final Set errors = new HashSet<>(); - private final PostgresGlobalState globalState; public PostgresNoRECOracle(PostgresGlobalState globalState) { + super(globalState); this.s = globalState.getSchema(); - this.con = globalState.getConnection(); - this.state = (PostgresStateToReproduce) globalState.getState(); - this.logger = globalState.getLogger(); - this.options = globalState.getOptions(); - this.globalState = globalState; + PostgresCommon.addCommonExpressionErrors(errors); + PostgresCommon.addCommonFetchErrors(errors); } @Override public void check() throws SQLException { // clear left-over query string from previous test - state.queryString = null; - PostgresCommon.addCommonExpressionErrors(errors); - PostgresCommon.addCommonFetchErrors(errors); + state.getState().queryString = null; PostgresTables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); PostgresExpression randomWhereCondition = getRandomWhereCondition(columns); List tables = randomTables.getTables(); - List joinStatements = getJoinStatements(globalState, columns, tables); + List joinStatements = getJoinStatements(state, columns, tables); List fromTables = tables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) .collect(Collectors.toList()); int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); @@ -81,9 +64,10 @@ public void check() throws SQLException { } if (firstCount != secondCount) { String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, firstQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, secondQueryString, secondCount); - state.queryString = String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount); + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); + state.getState().queryString = String.format("%s\n%s", firstQueryStringWithCount, + secondQueryStringWithCount); String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, firstQueryStringWithCount, secondQueryStringWithCount); throw new AssertionError(assertionMessage); @@ -106,8 +90,7 @@ public static List getJoinStatements(PostgresGlobalState globalSta } private PostgresExpression getRandomWhereCondition(List columns) { - return new PostgresExpressionGenerator(globalState).setColumns(columns).setGlobalState(globalState) - .generateExpression(PostgresDataType.BOOLEAN); + return new PostgresExpressionGenerator(state).setColumns(columns).generateExpression(PostgresDataType.BOOLEAN); } private int getUnoptimizedQueryCount(List fromTables, PostgresExpression randomWhereCondition, @@ -121,17 +104,17 @@ private int getUnoptimizedQueryCount(List fromTables, Postgr select.setSelectType(SelectType.ALL); select.setJoinClauses(joinStatements); int secondCount = 0; - secondQueryString = "SELECT SUM(count) FROM (" + PostgresVisitor.asString(select) + ") as res"; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + PostgresVisitor.asString(select) + ") as res"; if (options.logEachSelect()) { - logger.writeCurrent(secondQueryString); + logger.writeCurrent(unoptimizedQueryString); } errors.add("canceling statement due to statement timeout"); - Query q = new QueryAdapter(secondQueryString, errors); + Query q = new QueryAdapter(unoptimizedQueryString, errors); ResultSet rs; try { - rs = q.executeAndGet(globalState); + rs = q.executeAndGet(state); } catch (Exception e) { - throw new AssertionError(secondQueryString, e); + throw new AssertionError(unoptimizedQueryString, e); } if (rs == null) { return -1; @@ -151,18 +134,17 @@ private int getOptimizedQueryCount(List randomTables, List

implements TestOracle { private static final int NO_VALID_RESULT = -1; private final SQLite3Schema s; - private final SQLite3StateToReproduce state; - private final Set errors = new HashSet<>(); - private final StateLogger logger; - private final MainOptions options; - private final SQLite3GlobalState globalState; private SQLite3ExpressionGenerator gen; - private String firstQueryString; - private String secondQueryString; public SQLite3NoRECOracle(SQLite3GlobalState globalState) { + super(globalState); this.s = globalState.getSchema(); - this.state = (SQLite3StateToReproduce) globalState.getState(); - this.logger = globalState.getLogger(); - this.options = globalState.getOptions(); - this.globalState = globalState; SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addMatchQueryErrors(errors); SQLite3Errors.addQueryErrors(errors); @@ -67,7 +53,7 @@ public SQLite3NoRECOracle(SQLite3GlobalState globalState) { public void check() throws SQLException { SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); - gen = new SQLite3ExpressionGenerator(globalState).setColumns(columns); + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); SQLite3Expression randomWhereCondition = gen.generateExpression(); List tables = randomTables.getTables(); List joinStatements = gen.getRandomJoinClauses(tables); @@ -82,7 +68,7 @@ public void check() throws SQLException { throw new IgnoreMeException(); } if (optimizedCount != unoptimizedCount) { - state.queryString = firstQueryString + ";\n" + secondQueryString + ";"; + state.getState().queryString = optimizedQueryString + ";\n" + unoptimizedQueryString + ";"; throw new AssertionError(optimizedCount + " " + unoptimizedCount); } @@ -94,11 +80,11 @@ private int getUnoptimizedQuery(SQLite3Select select, SQLite3Expression randomWh SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); select.setFetchColumns(Arrays.asList(asText)); select.setWhereClause(null); - secondQueryString = "SELECT SUM(count) FROM (" + SQLite3Visitor.asString(select) + ")"; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + SQLite3Visitor.asString(select) + ")"; if (options.logEachSelect()) { - logger.writeCurrent(secondQueryString); + logger.writeCurrent(unoptimizedQueryString); } - QueryAdapter q = new QueryAdapter(secondQueryString, errors); + QueryAdapter q = new QueryAdapter(unoptimizedQueryString, errors); return extractCounts(q); } @@ -115,17 +101,17 @@ private int getOptimizedQuery(SQLite3Select select, SQLite3Expression randomWher select.setFetchColumns(Arrays.asList(aggr)); } select.setWhereClause(randomWhereCondition); - firstQueryString = SQLite3Visitor.asString(select); + optimizedQueryString = SQLite3Visitor.asString(select); if (options.logEachSelect()) { - logger.writeCurrent(firstQueryString); + logger.writeCurrent(optimizedQueryString); } - QueryAdapter q = new QueryAdapter(firstQueryString, errors); + QueryAdapter q = new QueryAdapter(optimizedQueryString, errors); return useAggregate ? extractCounts(q) : countRows(q); } private int countRows(QueryAdapter q) { int count = 0; - try (ResultSet rs = q.executeAndGet(globalState)) { + try (ResultSet rs = q.executeAndGet(state)) { if (rs == null) { return NO_VALID_RESULT; } else { @@ -142,14 +128,14 @@ private int countRows(QueryAdapter q) { if (e instanceof IgnoreMeException) { throw (IgnoreMeException) e; } - throw new AssertionError(secondQueryString, e); + throw new AssertionError(unoptimizedQueryString, e); } return count; } private int extractCounts(QueryAdapter q) { int count = 0; - try (ResultSet rs = q.executeAndGet(globalState)) { + try (ResultSet rs = q.executeAndGet(state)) { if (rs == null) { return NO_VALID_RESULT; } else { @@ -166,7 +152,7 @@ private int extractCounts(QueryAdapter q) { if (e instanceof IgnoreMeException) { throw (IgnoreMeException) e; } - throw new AssertionError(secondQueryString, e); + throw new AssertionError(unoptimizedQueryString, e); } return count; } From 71c848017fb89c3de8f342372a8b367724cd9e5b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 21 Jul 2020 18:10:40 +0200 Subject: [PATCH 0037/1284] Explain the expression generation mechanism --- docs/DEVELOPMENT.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index c54e44b4f..5b1aacc4d 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -14,6 +14,24 @@ If you do not find an option to import Maven projects, you might need to install The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see `DuckDBNoRECOracle` or `DuckDBQueryPartitioningWhereTester` for TLP). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). +### Typed vs. Untyped Expression Generation + +Each DBMS implementation provides an expression generator used, for example, to generate expressions used in `WHERE` clauses. We found that DBMS can be roughly classified into "permissive" ones, which apply implicit type conversions when needed and "strict" ones, which provide only few implicit conversions and output an error when the type is unexpected. For example, consider the following test case: + +```sql +CREATE TABLE t0(c0 TEXT); +INSERT INTO t0 VALUES ('1'); +SELECT * FROM t0 WHERE c0; +``` + +If the test case is executed using MySQL, which is a permissive DBMS, the `SELECT` fetches a single row, since the content of the `c0` value is interpreted as a boolean. If the test case is executed using PostgreSQL, which is a strict DBMS, the `SELECT` is not accepted as a valid query, and PostgreSQL outputs an error `"argument of WHERE must be type boolean"`. The implementation of the expression generator depends on whether we are dealing with a permissive or a strict DBMS. Since SQLancer's main goal is to find logic bugs, we want to generate as many valid queries as possible. + +For a permissive DBMS, implementing the expression generator is easier, since the expression generator does not need to care about the type of the expression, since the DBMS will apply any necessary conversions implicitly. For MySQL, the main `generateExpression` method thus does not accept any type as an argument (see [MySQLExpressionGenerator](https://github.com/sqlancer/sqlancer/blob/86647df8aa2dd8d167b5c3ce3297290f5b0b2bcd/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java#L54)). This method can be called when a expression is required for, for example, a `WHERE` clause. In principle, this approach can also be used for strict DBMS, by adding errors such as `argument of WHERE must be type boolean` to the list of expected errors. However, using such an "untyped" expression generator for a strict DBMS will result in many semantically invalid queries being generated. + +For a strict DBMS, the better approach is typically to attempt to generate expressions of the expected type. For PostgreSQL, the expression generator thus expects an additional type argument (see [PostgreSQLExpressionGenerator](https://github.com/sqlancer/sqlancer/blob/86647df8aa2dd8d167b5c3ce3297290f5b0b2bcd/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java#L251)). This type is propagated recursively. For example, if we require a predicate for the `WHERE` clause, we pass boolean as a type. The expression generator then calls a method `generateBooleanExpression` that attempts to produce a boolean expression, by, for example, generating a comparison (e.g., `<=`). For the comparison's operands, a random type is then selected and propagated. For example, if an integer type is selected, then `generateExpression` is called with this type once for the left operand, and once for the right operand. Note that this process does not guarantee that the expression will indeed have the expected type. It might happen, for example, that the expression generator attempts to produce an integer value, but that it produces a double value instead, namely when an integer overflow occurs, which, depending on the DBMS, implicitly converts the result to a floating-point value. + + + ## Options SQLancer uses [JCommander](https://jcommander.org/) for handling options. The `MainOptions` class contains options that are expected to be supported by all DBMS-testing implementations. Furthermore, each `*Provider` class provides a method to return an additional set of supported options. From 5a8fdef9a0310d4dd66821a888fd018a42302c53 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Jul 2020 14:23:41 +0200 Subject: [PATCH 0038/1284] Add some guidelines for commit messages and history --- docs/DEVELOPMENT.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 5b1aacc4d..648d8ad0c 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -62,3 +62,12 @@ We found that bugs in SQLancer are quickly found and easy to debug when testing ## Naming Conventions Each class specific to a DBMS is prefixed by the DBMS name. For example, each class specific to SQLite is prefixed by `SQLite3`. The naming convention is [automatically checked](src/check_names.py). + +## Commit History + +Please pay attention to good commit messages (in particular subject lines). As basic guidelines, we recommend a blog post on [How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/) written Chris Beams, which provides 7 useful rules. Implement at least the following of those rules: +1. Capitalize the subject line. For example, write "**R**efactor the handling of indexes" rather than "**r**efactor the handling of indexes". +2. Do not end the subject line with a period. For example, write "Refactor the handling of indexes" rather than "Refactor the handling of indexes.". +3. Use the imperative mood in the subject line. For example, write "Refactor the handling of indexes" rather than "Refactoring" or "Refactor**ed** the handling of indexes". + +Please also pay attention to a clean commit history. Rather than merging with the main branch, use `git rebase` to rebase your commits on the main branch. Sometimes, it might happen that you discover an issue only after having already created a commit, for example, when an issue is found by `mvn verify` in the Travis CI. Do not introduce a separate commit for such issues. If the issue was introduced by the last commit, you can fix the issue, and use `git commit --amend` to change the latest commit. If the change was introduced by one of the previous commits, you can use `git rebase -i` to change the respective commit. If you already have a number of such commits, you can use `git squash` to "collapse" multiple commits into one. For more information, you might want to read [How (and Why!) to Keep Your Git Commit History Clean](https://about.gitlab.com/blog/2018/06/07/keeping-git-commit-history-clean/) written by Kushal Pandya. From ce235f760c3a5405d3be662b96bc8bc959060a5f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Jul 2020 09:54:09 +0200 Subject: [PATCH 0039/1284] Fix a link in the docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 56ec5cf43..761a0bfa8 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ This project should at this stage still be seen as a research prototype. We beli | Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | | Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | -Please find the `.bib` entries [here](docs/DEVELOPMENT.md). +Please find the `.bib` entries [here](docs/PAPERS.md). # Supported DBMS From 3802fa7ca67da63f2bd825ae041308d8d4e4fc06 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Jul 2020 23:33:50 +0200 Subject: [PATCH 0040/1284] Remove an unnecessary enum --- src/sqlancer/MainOptions.java | 12 ------------ src/sqlancer/clickhouse/ClickHouseOptions.java | 11 ++++++----- src/sqlancer/cockroachdb/CockroachDBOptions.java | 3 +-- src/sqlancer/postgres/PostgresOptions.java | 3 +-- src/sqlancer/sqlite3/SQLite3Options.java | 3 +-- src/sqlancer/tidb/TiDBOptions.java | 3 +-- 6 files changed, 10 insertions(+), 25 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index d9999076a..e76595d5d 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -1,6 +1,5 @@ package sqlancer; -import com.beust.jcommander.IStringConverter; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; @@ -115,10 +114,6 @@ public int getNrStatementRetryCount() { return nrStatementRetryCount; } - public enum DBMS { - MariaDB, SQLite3, MySQL, PostgreSQL, TDEngine, CockroachDB, TiDB, ClickHouse - } - public String getUserName() { return userName; } @@ -127,13 +122,6 @@ public String getPassword() { return password; } - public class DBMSConverter implements IStringConverter { - @Override - public DBMS convert(String value) { - return DBMS.valueOf(value); - } - } - public boolean printProgressInformation() { return printProgressInformation; } diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 002ccee77..d380049d0 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -1,7 +1,12 @@ package sqlancer.clickhouse; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; + import sqlancer.MainOptions; import sqlancer.TestOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; @@ -10,14 +15,10 @@ import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPHavingOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; - @Parameters(separators = "=", commandDescription = "ClickHouse") public class ClickHouseOptions extends MainOptions { - @Parameter(names = "--oracle", converter = MainOptions.DBMSConverter.class) + @Parameter(names = "--oracle") public List oracle = Arrays.asList(ClickHouseOracle.TLPWhere); @Parameter(names = { "--test-joins" }, description = "Allow the generation of JOIN clauses", arity = 1) diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index f822f0086..71da218f6 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -5,7 +5,6 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.MainOptions.DBMSConverter; import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; @@ -21,7 +20,7 @@ @Parameters(separators = "=", commandDescription = "Test CockroachDB") public class CockroachDBOptions { - @Parameter(names = "--oracle", converter = DBMSConverter.class) + @Parameter(names = "--oracle") public CockroachDBOracle oracle = CockroachDBOracle.NOREC; public enum CockroachDBOracle { diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 0970360ce..df766d530 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -9,7 +9,6 @@ import com.beust.jcommander.Parameters; import sqlancer.CompositeTestOracle; -import sqlancer.MainOptions.DBMSConverter; import sqlancer.TestOracle; import sqlancer.postgres.oracle.PostgresNoRECOracle; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; @@ -23,7 +22,7 @@ public class PostgresOptions { @Parameter(names = "--bulk-insert") public boolean allowBulkInsert; - @Parameter(names = "--oracle", converter = DBMSConverter.class) + @Parameter(names = "--oracle") public List oracle = Arrays.asList(PostgresOracle.QUERY_PARTITIONING); @Parameter(names = "--test-collations", arity = 1) diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index b066b371a..90c964360 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -8,7 +8,6 @@ import com.beust.jcommander.Parameters; import sqlancer.CompositeTestOracle; -import sqlancer.MainOptions.DBMSConverter; import sqlancer.TestOracle; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.oracle.SQLite3Fuzzer; @@ -70,7 +69,7 @@ public class SQLite3Options { "--test-distinct-in-view" }, description = "DISTINCT in views might cause occasional false positives in NoREC and TLP", arity = 1) public boolean testDistinctInView; - @Parameter(names = "--oracle", converter = DBMSConverter.class) + @Parameter(names = "--oracle") public SQLite3Oracle oracle = SQLite3Oracle.NoREC; @Parameter(names = { diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index e60547860..944491114 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -9,7 +9,6 @@ import com.beust.jcommander.Parameters; import sqlancer.CompositeTestOracle; -import sqlancer.MainOptions.DBMSConverter; import sqlancer.TestOracle; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; @@ -18,7 +17,7 @@ @Parameters public class TiDBOptions { - @Parameter(names = "--oracle", converter = DBMSConverter.class) + @Parameter(names = "--oracle") public List oracle = Arrays.asList(TiDBOracle.QUERY_PARTITIONING); public enum TiDBOracle { From 849b6e052a45cce6d0b23ca3cfa483dad247383c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Jul 2020 00:07:41 +0200 Subject: [PATCH 0041/1284] Only append a single element in a StringBuilder call and enable PMD rule --- configs/pmd-rules.xml | 1 - .../gen/CockroachDBCommentOnGenerator.java | 6 ++-- .../gen/CockroachDBShowGenerator.java | 7 +++-- .../gen/CockroachDBViewGenerator.java | 3 +- .../duckdb/gen/DuckDBViewGenerator.java | 3 +- src/sqlancer/mariadb/MariaDBSchema.java | 10 +++++-- .../mariadb/gen/MariaDBIndexGenerator.java | 3 +- .../mysql/MySQLExpectedValueVisitor.java | 3 +- src/sqlancer/mysql/gen/MySQLAlterTable.java | 3 +- .../mysql/gen/MySQLTableGenerator.java | 30 ++++++++++++------- .../MySQLPivotedQuerySynthesisOracle.java | 3 +- .../PostgresExpectedValueVisitor.java | 3 +- src/sqlancer/postgres/PostgresSchema.java | 3 +- .../gen/PostgresAlterTableGenerator.java | 3 +- .../PostgresPivotedQuerySynthesisOracle.java | 4 ++- src/sqlancer/schema/AbstractSchema.java | 3 +- src/sqlancer/schema/AbstractTable.java | 3 +- .../sqlite3/SQLite3ExpectedValueVisitor.java | 9 ++++-- .../sqlite3/gen/SQLite3ColumnBuilder.java | 4 +-- .../gen/SQLite3MatchStringGenerator.java | 3 +- .../ddl/SQLite3CreateTriggerGenerator.java | 3 +- .../gen/ddl/SQLite3IndexGenerator.java | 7 +++-- .../gen/ddl/SQLite3TableGenerator.java | 4 +-- .../gen/dml/SQLite3InsertGenerator.java | 3 +- .../sqlite3/schema/SQLite3Schema.java | 3 +- src/sqlancer/tidb/gen/TiDBViewGenerator.java | 3 +- 26 files changed, 84 insertions(+), 46 deletions(-) diff --git a/configs/pmd-rules.xml b/configs/pmd-rules.xml index 8098884d4..b9dbe1ce4 100644 --- a/configs/pmd-rules.xml +++ b/configs/pmd-rules.xml @@ -76,7 +76,6 @@ - diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java index 9ee52a41c..954a2de36 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java @@ -28,7 +28,8 @@ public static Query comment(CockroachDBGlobalState globalState) { CockroachDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); switch (Randomly.fromOptions(Option.values())) { case TABLE: - sb.append("TABLE " + randomTable.getName()); + sb.append("TABLE "); + sb.append(randomTable.getName()); break; case INDEX: List indexes = randomTable.getIndexes(); @@ -39,7 +40,8 @@ public static Query comment(CockroachDBGlobalState globalState) { if (index.getIndexName().contains("primary")) { throw new IgnoreMeException(); } - sb.append("INDEX " + index.getIndexName()); + sb.append("INDEX "); + sb.append(index.getIndexName()); break; case COLUMN: sb.append("COLUMN "); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java index 9414e2999..a2ca73ccc 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java @@ -24,8 +24,8 @@ public static Query show(CockroachDBGlobalState globalState) { StringBuilder sb = new StringBuilder(); switch (Randomly.fromOptions(Option.values())) { case EXPERIMENTAL_FINGERPRINTS: - sb.append("SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE " - + globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + sb.append("SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); errors.add("as type bytes: bytea encoded value ends with incomplete escape sequence"); errors.add("invalid bytea escape sequence"); break; @@ -36,7 +36,8 @@ public static Query show(CockroachDBGlobalState globalState) { sb.append("SHOW JOBS"); break; case RANGES: - sb.append("SHOW RANGES FROM TABLE " + globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + sb.append("SHOW RANGES FROM TABLE "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); break; case LOCALITY: sb.append("SHOW LOCALITY"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java index d73b7b9fd..a2cadbeed 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java @@ -24,7 +24,8 @@ public static Query generate(CockroachDBGlobalState globalState) { if (i != 0) { sb.append(", "); } - sb.append("c" + i); + sb.append("c"); + sb.append(i); } sb.append(") AS "); sb.append(CockroachDBRandomQuerySynthesizer.generate(globalState, nrColumns).getQueryString()); diff --git a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java index 194fcc6b3..e2ac4ccaf 100644 --- a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java @@ -25,7 +25,8 @@ public static Query generate(DuckDBGlobalState globalState) { if (i != 0) { sb.append(", "); } - sb.append("c" + i); + sb.append("c"); + sb.append(i); } sb.append(") AS "); sb.append(DuckDBToStringVisitor.asString(DuckDBRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index afec7d6f9..a0f9b715e 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -205,9 +205,12 @@ public MariaDBTable(String tableName, List columns, List databaseTables) { public String toString() { StringBuffer sb = new StringBuffer(); for (MariaDBTable t : getDatabaseTables()) { - sb.append(t + "\n"); + sb.append(t); + sb.append("\n"); } return sb.toString(); } diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 39579e358..d4b396104 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -26,7 +26,8 @@ public static Query generate(MariaDBSchema s) { sb.append("UNIQUE "); } sb.append("INDEX "); - sb.append("i" + SQLite3Common.createColumnName(Randomly.smallNumber())); + sb.append("i"); + sb.append(SQLite3Common.createColumnName(Randomly.smallNumber())); if (Randomly.getBoolean()) { sb.append(" USING "); sb.append(Randomly.fromOptions("BTREE", "HASH")); // , "RTREE") diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index 437c7b654..fd77a9f85 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -31,7 +31,8 @@ private void print(MySQLExpression expr) { sb.append("\t"); } sb.append(v.get()); - sb.append(" -- " + expr.getExpectedValue()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); sb.append("\n"); } diff --git a/src/sqlancer/mysql/gen/MySQLAlterTable.java b/src/sqlancer/mysql/gen/MySQLAlterTable.java index 22b22dee1..a6ef3855e 100644 --- a/src/sqlancer/mysql/gen/MySQLAlterTable.java +++ b/src/sqlancer/mysql/gen/MySQLAlterTable.java @@ -152,7 +152,8 @@ private Query create() { sb.append(Randomly.fromOptions("TO", "AS")); sb.append(" "); } - sb.append("t" + Randomly.smallNumber()); + sb.append("t"); + sb.append(Randomly.smallNumber()); couldAffectSchema = true; break; default: diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 33ec48fa3..775353361 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -50,7 +50,8 @@ private Query create() { if (Randomly.getBoolean()) { sb.append(" IF NOT EXISTS"); } - sb.append(" " + tableName); + sb.append(" "); + sb.append(tableName); if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { sb.append(" LIKE "); sb.append(schema.getRandomTable().getName()); @@ -167,10 +168,12 @@ private void appendTableOptions() { } switch (o) { case AUTO_INCREMENT: - sb.append("AUTO_INCREMENT = " + r.getPositiveInteger()); + sb.append("AUTO_INCREMENT = "); + sb.append(r.getPositiveInteger()); break; case AVG_ROW_LENGTH: - sb.append("AVG_ROW_LENGTH = " + r.getPositiveInteger()); + sb.append("AVG_ROW_LENGTH = "); + sb.append(r.getPositiveInteger()); break; case CHECKSUM: sb.append("CHECKSUM = 1"); @@ -192,7 +195,8 @@ private void appendTableOptions() { // "MERGE": java.sql.SQLException: Table 't0' is read only String fromOptions = Randomly.fromOptions("InnoDB", "MyISAM", "MEMORY", "HEAP", "CSV", "ARCHIVE"); this.engine = MySQLEngine.get(fromOptions); - sb.append("ENGINE = " + fromOptions); + sb.append("ENGINE = "); + sb.append(fromOptions); break; // case ENCRYPTION: // sb.append("ENCRYPTION = '"); @@ -208,22 +212,28 @@ private void appendTableOptions() { sb.append(r.getPositiveInteger()); break; case MAX_ROWS: - sb.append("MAX_ROWS = " + r.getLong(0, Long.MAX_VALUE)); + sb.append("MAX_ROWS = "); + sb.append(r.getLong(0, Long.MAX_VALUE)); break; case MIN_ROWS: - sb.append("MIN_ROWS = " + r.getLong(1, Long.MAX_VALUE)); + sb.append("MIN_ROWS = "); + sb.append(r.getLong(1, Long.MAX_VALUE)); break; case PACK_KEYS: - sb.append("PACK_KEYS = " + Randomly.fromOptions("1", "0", "DEFAULT")); + sb.append("PACK_KEYS = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); break; case STATS_AUTO_RECALC: - sb.append("STATS_AUTO_RECALC = " + Randomly.fromOptions("1", "0", "DEFAULT")); + sb.append("STATS_AUTO_RECALC = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); break; case STATS_PERSISTENT: - sb.append("STATS_PERSISTENT = " + Randomly.fromOptions("1", "0", "DEFAULT")); + sb.append("STATS_PERSISTENT = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); break; case STATS_SAMPLE_PAGES: - sb.append("STATS_SAMPLE_PAGES = " + r.getInteger(1, Short.MAX_VALUE)); + sb.append("STATS_SAMPLE_PAGES = "); + sb.append(r.getInteger(1, Short.MAX_VALUE)); break; default: throw new AssertionError(o); diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 871e13334..2cf9bc991 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -208,7 +208,8 @@ private boolean isContainedIn(String queryString) throws SQLException { if (i++ != 0) { sb.append(" AND "); } - sb.append("result." + c.getTable().getName() + c.getName()); + sb.append("result."); + sb.append(c.getTable().getName() + c.getName()); if (rw.getValues().get(c).isNull()) { sb.append(" IS NULL"); } else { diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index 55287e273..89bf493a6 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -30,7 +30,8 @@ private void print(PostgresExpression expr) { sb.append("\t"); } sb.append(v.get()); - sb.append(" -- " + expr.getExpectedValue()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); sb.append("\n"); } diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index d99c6f223..2b0ae1499 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -358,7 +358,8 @@ public PostgresSchema(List databaseTables, String databaseName) { public String toString() { StringBuffer sb = new StringBuffer(); for (PostgresTable t : getDatabaseTables()) { - sb.append(t + "\n"); + sb.append(t); + sb.append("\n"); } return sb.toString(); } diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 5d19edaa3..2fcdb7414 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -344,7 +344,8 @@ public Query generate() { if (Randomly.getBoolean() || randomTable.getIndexes().isEmpty()) { sb.append(Randomly.fromOptions("DEFAULT", "FULL", "NOTHING")); } else { - sb.append("USING INDEX " + randomTable.getRandomIndex().getIndexName()); + sb.append("USING INDEX "); + sb.append(randomTable.getRandomIndex().getIndexName()); errors.add("cannot be used as replica identity"); errors.add("cannot use non-unique index"); errors.add("cannot use expression index"); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 5d3ac521d..287ca975b 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -167,7 +167,9 @@ private boolean isContainedIn(String queryString, MainOptions options, StateLogg if (i++ != 0) { sb.append(" AND "); } - sb.append("result." + c.getTable().getName() + c.getName()); + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); if (rw.getValues().get(c).isNull()) { sb.append(" IS NULL"); } else { diff --git a/src/sqlancer/schema/AbstractSchema.java b/src/sqlancer/schema/AbstractSchema.java index bb8014a54..df0e61f26 100644 --- a/src/sqlancer/schema/AbstractSchema.java +++ b/src/sqlancer/schema/AbstractSchema.java @@ -19,7 +19,8 @@ public AbstractSchema(List databaseTables) { public String toString() { StringBuffer sb = new StringBuffer(); for (A t : getDatabaseTables()) { - sb.append(t + "\n"); + sb.append(t); + sb.append("\n"); } return sb.toString(); } diff --git a/src/sqlancer/schema/AbstractTable.java b/src/sqlancer/schema/AbstractTable.java index 749d7f878..fe052cc3c 100644 --- a/src/sqlancer/schema/AbstractTable.java +++ b/src/sqlancer/schema/AbstractTable.java @@ -34,7 +34,8 @@ public int compareTo(AbstractTable o) { @Override public String toString() { StringBuffer sb = new StringBuffer(); - sb.append(getName() + "\n"); + sb.append(getName()); + sb.append("\n"); for (C c : columns) { sb.append("\t" + c + "\n"); } diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index fd090b60a..1c148ee86 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -47,9 +47,12 @@ private void print(SQLite3Expression expr) { sb.append("\t"); } sb.append(v.get()); - sb.append(" -- " + expr.getExpectedValue()); - sb.append(" explicit collate: " + expr.getExplicitCollateSequence()); - sb.append(" implicit collate: " + expr.getImplicitCollateSequence()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append(" explicit collate: "); + sb.append(expr.getExplicitCollateSequence()); + sb.append(" implicit collate: "); + sb.append(expr.getImplicitCollateSequence()); sb.append("\n"); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index 7defc9d29..0d6dad577 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -112,8 +112,8 @@ public String createColumn(String columnName, SQLite3GlobalState globalState, Li } } if (allowDefaultValue && Randomly.getBooleanWithSmallProbability()) { - sb.append(" DEFAULT " - + SQLite3Visitor.asString(SQLite3ExpressionGenerator.getRandomLiteralValue(globalState))); + sb.append(" DEFAULT "); + sb.append(SQLite3Visitor.asString(SQLite3ExpressionGenerator.getRandomLiteralValue(globalState))); } if (Randomly.getBooleanWithSmallProbability()) { String randomCollate = SQLite3Common.getRandomCollate(); diff --git a/src/sqlancer/sqlite3/gen/SQLite3MatchStringGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3MatchStringGenerator.java index aa70b4e84..5f68b71ad 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3MatchStringGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3MatchStringGenerator.java @@ -82,7 +82,8 @@ private void appendColspec() { if (i != 0) { sb.append(", "); } - sb.append("c" + Randomly.smallNumber()); + sb.append("c"); + sb.append(Randomly.smallNumber()); } if (braces) { sb.append("}"); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java index fe789c469..fbc58ba76 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java @@ -42,7 +42,8 @@ public static Query create(SQLite3GlobalState globalState) throws SQLException { } sb.append(" TRIGGER"); sb.append(" IF NOT EXISTS "); - sb.append("tr" + Randomly.smallNumber()); + sb.append("tr"); + sb.append(Randomly.smallNumber()); sb.append(" "); if (table.isView()) { sb.append("INSTEAD OF"); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 6277ff653..1f96b00da 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -72,9 +72,10 @@ private String createIndex(SQLite3Table t, List columns) { if (Randomly.getBoolean()) { sb.append(" IF NOT EXISTS"); } - sb.append(" " + SQLite3Common.getFreeIndexName(globalState.getSchema())); - sb.append(" ON"); - sb.append(" " + t.getName()); + sb.append(" "); + sb.append(SQLite3Common.getFreeIndexName(globalState.getSchema())); + sb.append(" ON "); + sb.append(t.getName()); sb.append("("); for (int i = 0; i < columns.size(); i++) { if (i != 0) { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index 72e8fe9e8..f6ee0319c 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -72,8 +72,8 @@ public void start() { if (Randomly.getBoolean()) { sb.append("IF NOT EXISTS "); } - sb.append(tableName + " "); - sb.append("("); + sb.append(tableName); + sb.append(" ("); boolean allowPrimaryKeyInColumn = Randomly.getBoolean(); int nrColumns = 1 + Randomly.smallNumber(); for (int i = 0; i < nrColumns; i++) { diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java index d7bb53e24..af6245fa2 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java @@ -71,7 +71,8 @@ private String insertRow(SQLite3Table table) { sb.append(fromOptions); } boolean defaultValues = false; - sb.append("INTO " + table.getName()); + sb.append("INTO "); + sb.append(table.getName()); List cols = table.getRandomNonEmptyColumnSubset(); if (cols.size() != table.getColumns().size() || Randomly.getBoolean()) { sb.append("("); diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 2b9a6d879..509c6f635 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -338,7 +338,8 @@ public SQLite3Schema(List databaseTables, List indexNames) public String toString() { StringBuffer sb = new StringBuffer(); for (SQLite3Table t : getDatabaseTables()) { - sb.append(t + "\n"); + sb.append(t); + sb.append("\n"); } return sb.toString(); } diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index bbe336314..340673b6a 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -32,7 +32,8 @@ public static Query getQuery(TiDBGlobalState globalState) { if (i != 0) { sb.append(", "); } - sb.append("c" + i); + sb.append("c"); + sb.append(i); } sb.append(") AS "); sb.append(TiDBRandomQuerySynthesizer.generate(globalState, nrColumns).getQueryString()); From 0d916dd4e5921ca8a7a9fbdbd4537e8072015a63 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Wed, 22 Jul 2020 19:43:31 +0300 Subject: [PATCH 0042/1284] implement support for Common Table Expressions --- .../PostgresExpectedValueVisitor.java | 6 +++++ .../postgres/PostgresToStringVisitor.java | 15 +++++++++++- src/sqlancer/postgres/PostgresVisitor.java | 5 ++++ src/sqlancer/postgres/ast/PostgresJoin.java | 17 ++++++++++++++ src/sqlancer/postgres/ast/PostgresSelect.java | 23 +++++++++++++++++++ 5 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index 89bf493a6..c5a021764 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -16,6 +16,7 @@ import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; import sqlancer.postgres.ast.PostgresSimilarTo; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -155,4 +156,9 @@ public void visit(PostgresFromTable from) { print(from); } + @Override + public void visit(PostgresCTE cte) { + print(cte); + } + } diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 66de6c94f..bb87056d5 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -22,6 +22,7 @@ import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.visitor.ToStringVisitor; @@ -75,6 +76,14 @@ public void visit(PostgresFromTable from) { } } + @Override + public void visit(PostgresCTE cte) { + sb.append("("); + visit(cte.getSelect()); + sb.append(") AS "); + sb.append(cte.getName()); + } + @Override public void visit(PostgresSelect s) { sb.append("SELECT "); @@ -126,7 +135,11 @@ public void visit(PostgresSelect s) { throw new AssertionError(j.getType()); } sb.append(" "); - sb.append(j.getTable().getName()); + if (j.joinCTE()) { + visit(j.getCTE()); + } else { + sb.append(j.getTable().getName()); + } if (j.getType() != PostgresJoinType.CROSS) { sb.append(" ON "); visit(j.getOnClause()); diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 06f990fc4..8f0942142 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -20,6 +20,7 @@ import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.gen.PostgresExpressionGenerator; @@ -57,6 +58,8 @@ public interface PostgresVisitor { void visit(PostgresFromTable from); + void visit(PostgresCTE cte); + default void visit(PostgresExpression expression) { if (expression instanceof PostgresConstant) { visit((PostgresConstant) expression); @@ -90,6 +93,8 @@ default void visit(PostgresExpression expression) { visit((PostgresCollate) expression); } else if (expression instanceof PostgresFromTable) { visit((PostgresFromTable) expression); + } else if (expression instanceof PostgresCTE) { + visit((PostgresCTE) expression); } else { throw new AssertionError(expression); } diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index 053541659..7e9a313b5 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -3,6 +3,7 @@ import sqlancer.Randomly; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; public class PostgresJoin implements PostgresExpression { @@ -18,6 +19,7 @@ public static PostgresJoinType getRandom() { private final PostgresTable table; private final PostgresExpression onClause; private final PostgresJoinType type; + private PostgresCTE CTE = null; public PostgresJoin(PostgresTable table, PostgresExpression onClause, PostgresJoinType type) { this.table = table; @@ -25,10 +27,25 @@ public PostgresJoin(PostgresTable table, PostgresExpression onClause, PostgresJo this.type = type; } + public PostgresJoin(PostgresCTE CTE, PostgresExpression onClause, PostgresJoinType type) { + this.CTE = CTE; + this.onClause = onClause; + this.type = type; + this.table = null; + } + public PostgresTable getTable() { return table; } + public PostgresCTE getCTE() { + return CTE; + } + + public boolean joinCTE() { + return (CTE != null); + } + public PostgresExpression getOnClause() { return onClause; } diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index c1e6fa1fd..a9a4b445a 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -56,6 +56,29 @@ public PostgresDataType getExpressionType() { } } + public static class PostgresCTE implements PostgresExpression { + private final PostgresSelect s; + private final String name; + + public PostgresCTE(PostgresSelect s, String name) { + this.s = s; + this.name = name; + } + + public PostgresSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public PostgresDataType getExpressionType() { + return null; + } + } + public enum SelectType { DISTINCT, ALL; From 464fcd66635964f612f7ccd0524c2db0c3347d69 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Wed, 22 Jul 2020 19:51:25 +0300 Subject: [PATCH 0043/1284] create CTE function added to Postgres TLP Base --- .../postgres/oracle/tlp/PostgresTLPBase.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 3b74340a6..dba211119 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -13,12 +13,15 @@ import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresSchema.PostgresTables; +import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.ForClause; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; +import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; import sqlancer.postgres.oracle.PostgresNoRECOracle; @@ -69,4 +72,34 @@ protected ExpressionGenerator getGen() { return gen; } + public static PostgresCTE createCTE(PostgresGlobalState globalState, PostgresTables tables) { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(tables.getColumns()); + PostgresSelect selectCTE = new PostgresSelect(); + selectCTE.setFromList(tables.getTables().stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + if (Randomly.getBoolean()) { + selectCTE.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + selectCTE.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { + selectCTE.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + selectCTE.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + selectCTE.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + selectCTE.setOffsetClause( + PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + selectCTE.setForClause(ForClause.getRandom()); + } + return new PostgresCTE(selectCTE, "cte"); + } + } From 8f172a4cb028d3237fbd5aca7a277ab0497f44b4 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 23 Jul 2020 17:51:01 +0300 Subject: [PATCH 0044/1284] Create tableReference class for clean support for subqueries & tables as table reference options --- .../PostgresExpectedValueVisitor.java | 6 +++ .../postgres/PostgresToStringVisitor.java | 12 +++--- src/sqlancer/postgres/PostgresVisitor.java | 5 +++ src/sqlancer/postgres/ast/PostgresJoin.java | 41 ++++++++++--------- .../postgres/oracle/PostgresNoRECOracle.java | 3 +- 5 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index c5a021764..06ec8a622 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -18,6 +18,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresJoin.PostgresTableReference; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -161,4 +162,9 @@ public void visit(PostgresCTE cte) { print(cte); } + @Override + public void visit(PostgresTableReference ref) { + print(ref); + } + } diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index bb87056d5..2bf7bd6ba 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -15,6 +15,7 @@ import sqlancer.postgres.ast.PostgresInOperation; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; +import sqlancer.postgres.ast.PostgresJoin.PostgresTableReference; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -84,6 +85,11 @@ public void visit(PostgresCTE cte) { sb.append(cte.getName()); } + @Override + public void visit(PostgresTableReference ref) { + visit(ref.getTableReference()); + } + @Override public void visit(PostgresSelect s) { sb.append("SELECT "); @@ -135,11 +141,7 @@ public void visit(PostgresSelect s) { throw new AssertionError(j.getType()); } sb.append(" "); - if (j.joinCTE()) { - visit(j.getCTE()); - } else { - sb.append(j.getTable().getName()); - } + visit(j.getTableReference()); if (j.getType() != PostgresJoinType.CROSS) { sb.append(" ON "); visit(j.getOnClause()); diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 8f0942142..acd7e2e2f 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -23,6 +23,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.ast.PostgresJoin.PostgresTableReference;; public interface PostgresVisitor { @@ -60,6 +61,8 @@ public interface PostgresVisitor { void visit(PostgresCTE cte); + void visit(PostgresTableReference ref); + default void visit(PostgresExpression expression) { if (expression instanceof PostgresConstant) { visit((PostgresConstant) expression); @@ -95,6 +98,8 @@ default void visit(PostgresExpression expression) { visit((PostgresFromTable) expression); } else if (expression instanceof PostgresCTE) { visit((PostgresCTE) expression); + } else if (expression instanceof PostgresTableReference) { + visit((PostgresTableReference) expression); } else { throw new AssertionError(expression); } diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index 7e9a313b5..b2ba415ce 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -4,6 +4,7 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; +import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; public class PostgresJoin implements PostgresExpression { @@ -16,34 +17,36 @@ public static PostgresJoinType getRandom() { } - private final PostgresTable table; + private final PostgresTableReference tableReference; private final PostgresExpression onClause; private final PostgresJoinType type; - private PostgresCTE CTE = null; - public PostgresJoin(PostgresTable table, PostgresExpression onClause, PostgresJoinType type) { - this.table = table; - this.onClause = onClause; - this.type = type; - } + public static class PostgresTableReference implements PostgresExpression { - public PostgresJoin(PostgresCTE CTE, PostgresExpression onClause, PostgresJoinType type) { - this.CTE = CTE; - this.onClause = onClause; - this.type = type; - this.table = null; - } + private final PostgresExpression tableReference; + + public PostgresTableReference(PostgresCTE cte) { + this.tableReference = cte; + } + + public PostgresTableReference(PostgresTable table) { + this.tableReference = new PostgresFromTable(table, Randomly.getBoolean()); + } + + public PostgresExpression getTableReference() { + return tableReference; + } - public PostgresTable getTable() { - return table; } - public PostgresCTE getCTE() { - return CTE; + public PostgresJoin(PostgresTableReference tableReference, PostgresExpression onClause, PostgresJoinType type) { + this.tableReference = tableReference; + this.onClause = onClause; + this.type = type; } - public boolean joinCTE() { - return (CTE != null); + public PostgresTableReference getTableReference() { + return tableReference; } public PostgresExpression getOnClause() { diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index d838ff302..efe2b7aee 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -27,6 +27,7 @@ import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; +import sqlancer.postgres.ast.PostgresJoin.PostgresTableReference; import sqlancer.postgres.ast.PostgresPostfixText; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; @@ -83,7 +84,7 @@ public static List getJoinStatements(PostgresGlobalState globalSta PostgresTable table = Randomly.fromList(tables); tables.remove(table); PostgresJoinType options = PostgresJoinType.getRandom(); - PostgresJoin j = new PostgresJoin(table, joinClause, options); + PostgresJoin j = new PostgresJoin(new PostgresTableReference(table), joinClause, options); joinStatements.add(j); } return joinStatements; From 1c94863a6ffe4bf977e00d64b7561606b6593fe3 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 23 Jul 2020 17:53:53 +0300 Subject: [PATCH 0045/1284] Replace CTE with subquery (correct terminology) --- src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index dba211119..f61acb0a7 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -72,7 +72,7 @@ protected ExpressionGenerator getGen() { return gen; } - public static PostgresCTE createCTE(PostgresGlobalState globalState, PostgresTables tables) { + public static PostgresCTE createSubquery(PostgresGlobalState globalState, PostgresTables tables) { PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(tables.getColumns()); PostgresSelect selectCTE = new PostgresSelect(); selectCTE.setFromList(tables.getTables().stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) From 4f786d7429b8d9e1f2c2954513a757102629ee78 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 23 Jul 2020 21:45:29 +0300 Subject: [PATCH 0046/1284] Remove PostgresTableReference class, use createSubquery function to add subquerie to joinss --- .../PostgresExpectedValueVisitor.java | 12 ++----- .../postgres/PostgresToStringVisitor.java | 14 +++----- src/sqlancer/postgres/PostgresVisitor.java | 13 +++----- src/sqlancer/postgres/ast/PostgresJoin.java | 27 ++------------- src/sqlancer/postgres/ast/PostgresSelect.java | 4 +-- .../gen/PostgresExpressionGenerator.java | 2 +- .../postgres/oracle/PostgresNoRECOracle.java | 13 ++++++-- .../postgres/oracle/tlp/PostgresTLPBase.java | 33 ++++++++++--------- 8 files changed, 45 insertions(+), 73 deletions(-) diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index 06ec8a622..549efdeec 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -16,9 +16,8 @@ import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; -import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; -import sqlancer.postgres.ast.PostgresJoin.PostgresTableReference; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -158,13 +157,8 @@ public void visit(PostgresFromTable from) { } @Override - public void visit(PostgresCTE cte) { - print(cte); - } - - @Override - public void visit(PostgresTableReference ref) { - print(ref); + public void visit(PostgresSubquery subquery) { + print(subquery); } } diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 2bf7bd6ba..2da562f98 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -15,7 +15,6 @@ import sqlancer.postgres.ast.PostgresInOperation; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; -import sqlancer.postgres.ast.PostgresJoin.PostgresTableReference; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -23,7 +22,7 @@ import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; -import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.visitor.ToStringVisitor; @@ -78,16 +77,11 @@ public void visit(PostgresFromTable from) { } @Override - public void visit(PostgresCTE cte) { + public void visit(PostgresSubquery subquery) { sb.append("("); - visit(cte.getSelect()); + visit(subquery.getSelect()); sb.append(") AS "); - sb.append(cte.getName()); - } - - @Override - public void visit(PostgresTableReference ref) { - visit(ref.getTableReference()); + sb.append(subquery.getName()); } @Override diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index acd7e2e2f..0cc1d53bc 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -20,10 +20,9 @@ import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; -import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.gen.PostgresExpressionGenerator; -import sqlancer.postgres.ast.PostgresJoin.PostgresTableReference;; public interface PostgresVisitor { @@ -59,9 +58,7 @@ public interface PostgresVisitor { void visit(PostgresFromTable from); - void visit(PostgresCTE cte); - - void visit(PostgresTableReference ref); + void visit(PostgresSubquery subquery); default void visit(PostgresExpression expression) { if (expression instanceof PostgresConstant) { @@ -96,10 +93,8 @@ default void visit(PostgresExpression expression) { visit((PostgresCollate) expression); } else if (expression instanceof PostgresFromTable) { visit((PostgresFromTable) expression); - } else if (expression instanceof PostgresCTE) { - visit((PostgresCTE) expression); - } else if (expression instanceof PostgresTableReference) { - visit((PostgresTableReference) expression); + } else if (expression instanceof PostgresSubquery) { + visit((PostgresSubquery) expression); } else { throw new AssertionError(expression); } diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index b2ba415ce..57b92ddf3 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -2,9 +2,6 @@ import sqlancer.Randomly; import sqlancer.postgres.PostgresSchema.PostgresDataType; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; -import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; public class PostgresJoin implements PostgresExpression { @@ -17,35 +14,17 @@ public static PostgresJoinType getRandom() { } - private final PostgresTableReference tableReference; + private final PostgresExpression tableReference; private final PostgresExpression onClause; private final PostgresJoinType type; - public static class PostgresTableReference implements PostgresExpression { - - private final PostgresExpression tableReference; - - public PostgresTableReference(PostgresCTE cte) { - this.tableReference = cte; - } - - public PostgresTableReference(PostgresTable table) { - this.tableReference = new PostgresFromTable(table, Randomly.getBoolean()); - } - - public PostgresExpression getTableReference() { - return tableReference; - } - - } - - public PostgresJoin(PostgresTableReference tableReference, PostgresExpression onClause, PostgresJoinType type) { + public PostgresJoin(PostgresExpression tableReference, PostgresExpression onClause, PostgresJoinType type) { this.tableReference = tableReference; this.onClause = onClause; this.type = type; } - public PostgresTableReference getTableReference() { + public PostgresExpression getTableReference() { return tableReference; } diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index a9a4b445a..ef2f6e917 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -56,11 +56,11 @@ public PostgresDataType getExpressionType() { } } - public static class PostgresCTE implements PostgresExpression { + public static class PostgresSubquery implements PostgresExpression { private final PostgresSelect s; private final String name; - public PostgresCTE(PostgresSelect s, String name) { + public PostgresSubquery(PostgresSelect s, String name) { this.s = s; this.name = name; } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index ce9b6bce8..d4584f10b 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -91,7 +91,7 @@ public static PostgresExpression generateExpression(PostgresGlobalState globalSt return new PostgresExpressionGenerator(globalState).generateExpression(0); } - PostgresExpression generateExpression(int depth) { + public PostgresExpression generateExpression(int depth) { return generateExpression(depth, PostgresDataType.getRandomType()); } diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index efe2b7aee..2a7d667f3 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -27,13 +27,14 @@ import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; -import sqlancer.postgres.ast.PostgresJoin.PostgresTableReference; import sqlancer.postgres.ast.PostgresPostfixText; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.oracle.tlp.PostgresTLPBase; public class PostgresNoRECOracle extends NoRECBase implements TestOracle { @@ -84,7 +85,15 @@ public static List getJoinStatements(PostgresGlobalState globalSta PostgresTable table = Randomly.fromList(tables); tables.remove(table); PostgresJoinType options = PostgresJoinType.getRandom(); - PostgresJoin j = new PostgresJoin(new PostgresTableReference(table), joinClause, options); + PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + PostgresSubquery subquery = PostgresTLPBase.createSubquery(globalState, String.format("sub%d", i)); + PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(subquery, joinClause, options); joinStatements.add(j); } return joinStatements; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index f61acb0a7..cd6448809 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.List; +import java.util.ArrayList; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -20,7 +21,7 @@ import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.ForClause; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; -import sqlancer.postgres.ast.PostgresSelect.PostgresCTE; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; @@ -72,34 +73,34 @@ protected ExpressionGenerator getGen() { return gen; } - public static PostgresCTE createSubquery(PostgresGlobalState globalState, PostgresTables tables) { + public static PostgresSubquery createSubquery(PostgresGlobalState globalState, String name) { + List columns = new ArrayList<>(); + PostgresTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(tables.getColumns()); - PostgresSelect selectCTE = new PostgresSelect(); - selectCTE.setFromList(tables.getTables().stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + PostgresSelect select = new PostgresSelect(); + select.setFromList(tables.getTables().stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) .collect(Collectors.toList())); + select.setFetchColumns(columns); if (Randomly.getBoolean()) { - selectCTE.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - selectCTE.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - if (Randomly.getBoolean()) { - selectCTE.setHavingClause(gen.generateHavingClause()); - } + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - selectCTE.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByExpressions(gen.generateOrderBy()); } if (Randomly.getBoolean()) { - selectCTE.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); if (Randomly.getBoolean()) { - selectCTE.setOffsetClause( + select.setOffsetClause( PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); } } if (Randomly.getBooleanWithRatherLowProbability()) { - selectCTE.setForClause(ForClause.getRandom()); + select.setForClause(ForClause.getRandom()); } - return new PostgresCTE(selectCTE, "cte"); + return new PostgresSubquery(select, name); } } From 2dddb490bfe76ea6e64682d1c1e7a14c0d3ca1bc Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 23 Jul 2020 10:02:45 +0300 Subject: [PATCH 0047/1284] Rewrite ClickHouseTableGenerator --- src/sqlancer/clickhouse/ClickHouseErrors.java | 14 ++ .../clickhouse/ClickHouseProvider.java | 4 +- src/sqlancer/clickhouse/ClickHouseSchema.java | 25 +++- .../gen/ClickHouseColumnBuilder.java | 96 +++++++++++++ .../clickhouse/gen/ClickHouseCommon.java | 8 ++ .../gen/ClickHouseExpressionGenerator.java | 21 ++- .../gen/ClickHouseInsertGenerator.java | 3 +- .../gen/ClickHouseTableGenerator.java | 128 ++++++++++++------ ...ickHouseBinaryComparisonOperationTest.java | 4 +- 9 files changed, 253 insertions(+), 50 deletions(-) create mode 100644 src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 20665b139..4aeafd6d5 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -34,6 +34,7 @@ public static void addExpectedExpressionErrors(Set errors) { errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database errors.add("is not under aggregate function"); errors.add("Invalid type for filter in"); + errors.add("argument of function"); } public static void addExpressionHavingErrors(Set errors) { @@ -47,4 +48,17 @@ public static void addQueryErrors(Set errors) { public static void addGroupingErrors(Set errors) { errors.add("Memory limit"); } + + public static void addTableManipulationErrors(Set errors) { + errors.add("Memory limit"); + errors.add("Directory for table data"); + errors.add("Directory not empty"); + errors.add("Partition key cannot contain constants"); + errors.add("Cannot convert string"); + errors.add("argument of function"); + errors.add("Attempt to read after eof: while converting"); + errors.add("Sorting key cannot contain constants"); + errors.add("Sampling expression must be present in the primary key"); + } + } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 945785c0f..d6f7e388c 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -19,6 +19,7 @@ import sqlancer.StatementExecutor; import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseInsertGenerator; import sqlancer.clickhouse.gen.ClickHouseTableGenerator; @@ -87,7 +88,8 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws SQLExcept for (int i = 0; i < Randomly.fromOptions(1); i++) { boolean success; do { - Query qt = new ClickHouseTableGenerator().getQuery(globalState); + String tableName = ClickHouseCommon.createTableName(i); + Query qt = ClickHouseTableGenerator.createTableStatement(tableName, globalState); success = globalState.executeStatement(qt); } while (!success); } diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index f0c7523f5..81423300d 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -59,10 +59,27 @@ public String toString() { public static class ClickHouseColumn extends AbstractTableColumn { - public ClickHouseColumn(String name, ClickHouseLancerDataType columnType) { + private final boolean isAlias; + private final boolean isMaterialized; + + public ClickHouseColumn(String name, ClickHouseLancerDataType columnType, boolean isAlias, + boolean isMaterialized) { super(name, null, columnType); + this.isAlias = isAlias; + this.isMaterialized = isMaterialized; + } + + public static ClickHouseSchema.ClickHouseColumn createDummy(String name) { + return new ClickHouseSchema.ClickHouseColumn(name, ClickHouseLancerDataType.getRandom(), false, false); } + public boolean isAlias() { + return isAlias; + } + + public boolean isMaterialized() { + return isMaterialized; + } } public static ClickHouseConstant getConstant(ResultSet randomRowValues, int columnIndex, @@ -302,7 +319,11 @@ private static List getTableColumns(Connection con, String tab while (rs.next()) { String columnName = rs.getString("name"); String dataType = rs.getString("type"); - ClickHouseColumn c = new ClickHouseColumn(columnName, getColumnType(dataType)); + String defaultType = rs.getString("default_type"); + boolean isAlias = "ALIAS".compareTo(defaultType) == 0; + boolean isMaterialized = "MATERIALIZED".compareTo(defaultType) == 0; + ClickHouseColumn c = new ClickHouseColumn(columnName, getColumnType(dataType), isAlias, + isMaterialized); columns.add(c); } } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java new file mode 100644 index 000000000..c12213f97 --- /dev/null +++ b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java @@ -0,0 +1,96 @@ +package sqlancer.clickhouse.gen; + +import sqlancer.Randomly; +import sqlancer.clickhouse.ClickHouseProvider; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseVisitor; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class ClickHouseColumnBuilder { + + private final StringBuilder sb = new StringBuilder(); + + private static boolean allowAlias = true; + private static boolean allowMaterialized = true; + private static boolean allowDefaultValue = true; + private static boolean allowCodec = true; + + private enum Constraints { + DEFAULT, MATERIALIZED, ALIAS, CODEC // TTL + } + + public String createColumn(String columnName, ClickHouseProvider.ClickHouseGlobalState globalState, + List columns) { + sb.append(columnName); + sb.append(" "); + List constraints = new ArrayList<>(); + ClickHouseSchema.ClickHouseLancerDataType dataType = ClickHouseSchema.ClickHouseLancerDataType.getRandom(); + if (Randomly.getBooleanWithSmallProbability()) { + constraints = Randomly.subset(Constraints.values()); + if (!allowAlias || columns.isEmpty() || columns.size() == 1) { + constraints.remove(Constraints.ALIAS); + } + if (!allowMaterialized) { + constraints.remove(Constraints.MATERIALIZED); + } + if (!allowDefaultValue) { + constraints.remove(Constraints.DEFAULT); + } + if (constraints.contains(Constraints.MATERIALIZED)) { + constraints.remove(Constraints.ALIAS); + constraints.remove(Constraints.DEFAULT); + } else if (constraints.contains(Constraints.ALIAS)) { + constraints.remove(Constraints.DEFAULT); + } + } + + if (!constraints.contains(Constraints.ALIAS)) { + sb.append(dataType); + } + + for (Constraints c : constraints) { + switch (c) { + case MATERIALIZED: + if (allowMaterialized) { + sb.append(" MATERIALIZED ("); + sb.append( + ClickHouseVisitor + .asString(new ClickHouseExpressionGenerator(globalState) + .setColumns( + columns.stream().filter(p -> !p.getName().contentEquals(columnName)) + .collect(Collectors.toList())) + .generateExpression(dataType))); + sb.append(")"); + } + break; + case DEFAULT: + if (allowDefaultValue) { + sb.append(" DEFAULT "); + sb.append(new ClickHouseExpressionGenerator(globalState).generateConstant(dataType)); + } + break; + case ALIAS: + if (allowAlias) { + sb.append(" ALIAS "); + sb.append(Randomly.fromList(columns.stream().filter(p -> !p.getName().contentEquals(columnName)) + .collect(Collectors.toList())).getName()); + } + break; + case CODEC: + if (allowCodec) { + sb.append(" CODEC ("); + sb.append(Randomly.fromOptions("NONE", "ZSTD", "LZ4HC")); + sb.append(")"); + } + break; + default: + throw new AssertionError(); + } + } + return sb.toString(); + } + +} diff --git a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java index 467f8591c..cff33a94d 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java @@ -16,6 +16,14 @@ public static String createColumnName(int nr) { return String.format("c%d", nr); } + public static String createTableName(int nr) { + return String.format("t%d", nr); + } + + public static String createConstraintName(int nr) { + return String.format("x%d", nr); + } + public static List getTableRefs(List tables, ClickHouseSchema s) { List tableRefs = new ArrayList<>(); diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index f5abe276c..f279b8cc1 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -45,20 +45,26 @@ protected ClickHouseExpression generateExpression(ClickHouseLancerDataType type, return generateLeafNode(type); } Expression expr = Randomly.fromOptions(Expression.values()); + ClickHouseLancerDataType leftLeafType = ClickHouseLancerDataType.getRandom(); + ClickHouseLancerDataType rightLeafType = ClickHouseLancerDataType.getRandom(); + if (Randomly.getBoolean()) { + rightLeafType = leftLeafType; + } + switch (expr) { case UNARY_PREFIX: - return new ClickHouseUnaryPrefixOperation(generateExpression(type, depth + 1), + return new ClickHouseUnaryPrefixOperation(generateExpression(leftLeafType, depth + 1), ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.getRandom()); case UNARY_POSTFIX: - return new ClickHouseUnaryPostfixOperation(generateExpression(type, depth + 1), + return new ClickHouseUnaryPostfixOperation(generateExpression(leftLeafType, depth + 1), ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.getRandom(), false); case BINARY_COMPARISON: - return new ClickHouseBinaryComparisonOperation(generateExpression(type, depth + 1), - generateExpression(type, depth + 1), + return new ClickHouseBinaryComparisonOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(rightLeafType, depth + 1), ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.getRandomOperator()); case BINARY_LOGICAL: - return new ClickHouseBinaryLogicalOperation(generateExpression(type, depth + 1), - generateExpression(type, depth + 1), + return new ClickHouseBinaryLogicalOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(rightLeafType, depth + 1), ClickHouseBinaryLogicalOperation.ClickHouseBinaryLogicalOperator.getRandom()); default: throw new AssertionError(expr); @@ -67,6 +73,9 @@ protected ClickHouseExpression generateExpression(ClickHouseLancerDataType type, @Override protected ClickHouseExpression generateColumn(ClickHouseLancerDataType type) { + if (columns.isEmpty()) { + return generateConstant(type); + } List filteredColumns = columns.stream() .filter(c -> c.getType().getType().name().equals(type.getType().name())).collect(Collectors.toList()); ClickHouseColumn column = filteredColumns.isEmpty() ? Randomly.fromList(columns) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 2ecccbef0..9d56b3c52 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -36,7 +36,8 @@ public static Query getQuery(ClickHouseGlobalState globalState) throws SQLExcept private Query get() { ClickHouseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyColumnSubset(); + List columns = table.getRandomNonEmptyColumnSubset().stream() + .filter(c -> !c.isAlias() && !c.isMaterialized()).collect(Collectors.toList()); sb.append("INSERT INTO "); sb.append(table.getName()); sb.append("("); diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index aac5f6413..b516ef450 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -1,67 +1,119 @@ package sqlancer.clickhouse.gen; -import java.util.HashSet; -import java.util.Set; - +import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import sqlancer.clickhouse.ClickHouseErrors; +import sqlancer.clickhouse.ClickHouseProvider; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseToStringVisitor; +import sqlancer.clickhouse.ast.ClickHouseExpression; public class ClickHouseTableGenerator { - StringBuilder sb = new StringBuilder("CREATE TABLE "); - Set errors = new HashSet<>(); + private enum ClickHouseEngine { + // TinyLog, StripeLog, + Log, Memory, MergeTree + } - public ClickHouseTableGenerator() { - errors.add("Memory limit"); - errors.add("Directory for table data"); - errors.add("Directory not empty"); + private final StringBuilder sb = new StringBuilder(); + private final String tableName; + private int columnId; + private final List columnNames = new ArrayList<>(); + private final List columns = new ArrayList<>(); + private final ClickHouseProvider.ClickHouseGlobalState globalState; + + public ClickHouseTableGenerator(String tableName, ClickHouseProvider.ClickHouseGlobalState globalState) { + this.tableName = tableName; + this.globalState = globalState; } - private enum ClickHouseEngine { - TinyLog, StripeLog, Log, Memory, MergeTree + public static Query createTableStatement(String tableName, ClickHouseProvider.ClickHouseGlobalState globalState) { + ClickHouseTableGenerator chTableGenerator = new ClickHouseTableGenerator(tableName, globalState); + chTableGenerator.start(); + Set errors = new HashSet<>(); + ClickHouseErrors.addTableManipulationErrors(errors); + return new QueryAdapter(chTableGenerator.sb.toString(), errors, true); } - public Query getQuery(ClickHouseGlobalState globalState) { + public void start() { ClickHouseEngine engine = Randomly.fromOptions(ClickHouseEngine.values()); - sb.append(globalState.getSchema().getFreeTableName()); - sb.append("("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState).allowAggregates(false); + sb.append("CREATE "); + sb.append("TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(tableName); + sb.append(" ("); + int nrColumns = 1 + Randomly.smallNumber(); + for (int i = 0; i < nrColumns; i++) { + columns.add(ClickHouseSchema.ClickHouseColumn.createDummy(ClickHouseCommon.createColumnName(i))); + } + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } - sb.append("c"); - sb.append(i); - sb.append(" "); - if (Randomly.getBoolean()) { - sb.append(ClickHouseLancerDataType.getRandom()); - } else { - // sb.append("Nullable("); - sb.append(ClickHouseLancerDataType.getRandom()); - // sb.append(")"); + String columnName = ClickHouseCommon.createColumnName(columnId); + ClickHouseColumnBuilder columnBuilder = new ClickHouseColumnBuilder(); + sb.append(columnBuilder.createColumn(columnName, globalState, columns)); + columnNames.add(columnName); + columnId++; + } + if (Randomly.getBooleanWithSmallProbability()) { + for (int i = 0; i < Randomly.smallNumber(); i++) { + addColumnsConstraint(gen); } - potentiallyAppendCodec(); } sb.append(") ENGINE = "); sb.append(engine); sb.append("("); - sb.append(")"); + sb.append(") "); if (engine == ClickHouseEngine.MergeTree) { - sb.append(" ORDER BY tuple()"); + if (Randomly.getBoolean()) { + sb.append(" ORDER BY "); + ClickHouseExpression expr = gen + .generateExpression(ClickHouseSchema.ClickHouseLancerDataType.getRandom()); + sb.append(ClickHouseToStringVisitor.asString(expr)); + } else { + sb.append(" ORDER BY tuple() "); + } + + if (Randomly.getBoolean()) { + sb.append(" PARTITION BY "); + ClickHouseExpression expr = gen + .generateExpression(ClickHouseSchema.ClickHouseLancerDataType.getRandom()); + sb.append(ClickHouseToStringVisitor.asString(expr)); + } + if (Randomly.getBoolean()) { + sb.append(" SAMPLE BY "); + ClickHouseExpression expr = gen + .generateExpression(ClickHouseSchema.ClickHouseLancerDataType.getRandom()); + sb.append(ClickHouseToStringVisitor.asString(expr)); + } + // TODO: PRIMARY KEY } - sb.append(";"); - return new QueryAdapter(sb.toString(), errors, true); + } - private void potentiallyAppendCodec() { - if (Randomly.getBoolean()) { - sb.append(" CODEC("); - errors.add(" in memory is not of fixed size"); - sb.append(Randomly.fromOptions("NONE", "ZSTD", "LZ4HC")); - sb.append(")"); + private void addColumnsConstraint(ClickHouseExpressionGenerator gen) { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(" CONSTRAINT "); + sb.append(ClickHouseCommon.createConstraintName(i)); + sb.append(" CHECK "); + ClickHouseExpression expr = gen + .generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + sb.append(ClickHouseToStringVisitor.asString(expr)); } } - } diff --git a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java index 3977b0bdf..a5fafd55a 100644 --- a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java +++ b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java @@ -6,7 +6,7 @@ import java.util.Arrays; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class ClickHouseBinaryComparisonOperationTest { @@ -125,4 +125,4 @@ void getExpectedValueInt32EqualsFloat64() { assertEquals(positiveConst.applyEquals(float64OneConst).asInt(), 0); assertEquals(positiveConst.applyEquals(float64ZeroConst).asInt(), 0); } -} \ No newline at end of file +} From 263943dd62add5d3af34fdfb401045624dbc7fce Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Jul 2020 20:53:28 +0200 Subject: [PATCH 0048/1284] Make the statements field in the StateToReproduce class private --- src/sqlancer/Main.java | 2 +- src/sqlancer/StateToReproduce.java | 31 +++++++++++++++++-- .../clickhouse/ClickHouseProvider.java | 5 ++- .../cockroachdb/CockroachDBProvider.java | 8 ++--- src/sqlancer/mariadb/MariaDBProvider.java | 8 ++--- src/sqlancer/mysql/MySQLProvider.java | 6 ++-- src/sqlancer/postgres/PostgresProvider.java | 8 ++--- src/sqlancer/tidb/TiDBProvider.java | 8 ++--- 8 files changed, 50 insertions(+), 26 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index f0fb2df61..36940ef3a 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -254,7 +254,7 @@ public static class QueryManager { } public boolean execute(Query q) throws SQLException { - globalState.getState().statements.add(q); + globalState.getState().logStatement(q); boolean success = q.execute(globalState); Main.nrSuccessfulActions.addAndGet(1); return success; diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index b27717589..f640eae2d 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -1,6 +1,7 @@ package sqlancer; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -19,7 +20,7 @@ public class StateToReproduce { - public final List statements = new ArrayList<>(); + private final List statements = new ArrayList<>(); /** * The string printed at the bottom of the error log file, which contains the queries that caused the test to fail @@ -57,8 +58,34 @@ public String getDatabaseVersion() { return databaseVersion; } + /** + * Logs the statement string without executing the corresponding statement. + * + * @param queryString + * the query string to be logged + */ + public void logStatement(String queryString) { + if (queryString == null) { + throw new IllegalArgumentException(); + } + logStatement(new QueryAdapter(queryString)); + } + + /** + * Logs the statement without executing it. + * + * @param query + * the query to be logged + */ + public void logStatement(Query query) { + if (query == null) { + throw new IllegalArgumentException(); + } + statements.add(query); + } + public List getStatements() { - return statements; + return Collections.unmodifiableList(statements); } public String getQueryString() { diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index d6f7e388c..9adca52ba 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -13,7 +13,6 @@ import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StatementExecutor; @@ -124,9 +123,9 @@ public Connection createDatabase(ClickHouseGlobalState globalState) throws SQLEx Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); String dropDatabaseCommand = "DROP DATABASE IF EXISTS " + databaseName; - globalState.getState().statements.add(new QueryAdapter(dropDatabaseCommand)); + globalState.getState().logStatement(dropDatabaseCommand); String createDatabaseCommand = "CREATE DATABASE IF NOT EXISTS " + databaseName; - globalState.getState().statements.add(new QueryAdapter(createDatabaseCommand)); + globalState.getState().logStatement(createDatabaseCommand); try (Statement s = con.createStatement()) { s.execute(dropDatabaseCommand); Thread.sleep(1000); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 47ccd25fd..f10e6068d 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -259,11 +259,11 @@ public Connection createDatabase(CockroachDBGlobalState globalState) throws SQLE String url = "jdbc:postgresql://localhost:26257/test"; Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - globalState.getState().statements.add(new QueryAdapter("USE test")); - globalState.getState().statements.add(new QueryAdapter("DROP DATABASE IF EXISTS " + databaseName + " CASCADE")); + globalState.getState().logStatement("USE test"); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); String createDatabaseCommand = "CREATE DATABASE " + databaseName; - globalState.getState().statements.add(new QueryAdapter(createDatabaseCommand)); - globalState.getState().statements.add(new QueryAdapter("USE " + databaseName)); + globalState.getState().logStatement(createDatabaseCommand); + globalState.getState().logStatement("USE " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); } diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 7193ad633..8643d5aa1 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -12,7 +12,6 @@ import sqlancer.MainOptions; import sqlancer.ProviderAdapter; import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.TestOracle; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; @@ -171,10 +170,9 @@ protected void updateSchema() throws SQLException { @Override public Connection createDatabase(MariaDBGlobalState globalState) throws SQLException { - globalState.getState().statements - .add(new QueryAdapter("DROP DATABASE IF EXISTS " + globalState.getDatabaseName())); - globalState.getState().statements.add(new QueryAdapter("CREATE DATABASE " + globalState.getDatabaseName())); - globalState.getState().statements.add(new QueryAdapter("USE " + globalState.getDatabaseName())); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + globalState.getDatabaseName()); + globalState.getState().logStatement("CREATE DATABASE " + globalState.getDatabaseName()); + globalState.getState().logStatement("USE " + globalState.getDatabaseName()); // /?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true String url = "jdbc:mariadb://localhost:3306"; Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 7ad3e5967..23a7eeae5 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -176,9 +176,9 @@ public static int getNrRows(Connection con, MySQLTable table) throws SQLExceptio @Override public Connection createDatabase(MySQLGlobalState globalState) throws SQLException { String databaseName = globalState.getDatabaseName(); - globalState.getState().statements.add(new QueryAdapter("DROP DATABASE IF EXISTS " + databaseName)); - globalState.getState().statements.add(new QueryAdapter("CREATE DATABASE " + databaseName)); - globalState.getState().statements.add(new QueryAdapter("USE " + databaseName)); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); String url = "jdbc:mysql://localhost:3306/?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true"; Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 4cf1dfba8..1d66b6450 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -222,11 +222,11 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - globalState.getState().statements.add(new QueryAdapter("\\c test;")); - globalState.getState().statements.add(new QueryAdapter("DROP DATABASE IF EXISTS " + databaseName)); + globalState.getState().logStatement("\\c test;"); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); String createDatabaseCommand = getCreateDatabaseCommand(databaseName, con, globalState); - globalState.getState().statements.add(new QueryAdapter(createDatabaseCommand)); - globalState.getState().statements.add(new QueryAdapter("\\c " + databaseName)); + globalState.getState().logStatement(createDatabaseCommand); + globalState.getState().logStatement("\\c " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 225ef1a13..0cc308dd1 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -153,11 +153,11 @@ public Connection createDatabase(TiDBGlobalState globalState) throws SQLExceptio String url = "jdbc:mysql://127.0.0.1:4000/"; Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - globalState.getState().statements.add(new QueryAdapter("USE test")); - globalState.getState().statements.add(new QueryAdapter("DROP DATABASE IF EXISTS " + databaseName)); + globalState.getState().logStatement("USE test"); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); String createDatabaseCommand = "CREATE DATABASE " + databaseName; - globalState.getState().statements.add(new QueryAdapter(createDatabaseCommand)); - globalState.getState().statements.add(new QueryAdapter("USE " + databaseName)); + globalState.getState().logStatement(createDatabaseCommand); + globalState.getState().logStatement("USE " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); } From fa42b8bef5c989a1c88c204300d9b16e25ee66a9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Jul 2020 23:11:36 +0200 Subject: [PATCH 0049/1284] Unify and check the import order --- configs/checkstyle.xml | 9 ++++++++- src/sqlancer/clickhouse/ClickHouseSchema.java | 1 - src/sqlancer/clickhouse/ast/ClickHouseAggregate.java | 8 ++++---- src/sqlancer/clickhouse/ast/ClickHouseCast.java | 4 ++-- .../clickhouse/ast/ClickHouseCastOperation.java | 1 - src/sqlancer/clickhouse/ast/ClickHouseConstant.java | 4 ++-- .../ast/ClickHouseUnaryPrefixOperation.java | 2 +- .../clickhouse/gen/ClickHouseColumnBuilder.java | 8 ++++---- src/sqlancer/clickhouse/gen/ClickHouseCommon.java | 6 +++--- .../clickhouse/gen/ClickHouseExpressionGenerator.java | 2 +- .../clickhouse/gen/ClickHouseTableGenerator.java | 9 ++++----- .../oracle/tlp/ClickHouseTLPAggregateOracle.java | 10 +++++----- .../oracle/tlp/ClickHouseTLPDistinctOracle.java | 8 ++++---- .../oracle/tlp/ClickHouseTLPGroupByOracle.java | 10 +++++----- .../oracle/tlp/ClickHouseTLPHavingOracle.java | 10 +++++----- .../oracle/tlp/PostgresTLPAggregateOracle.java | 11 +++++++---- src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 6 +++--- 17 files changed, 58 insertions(+), 51 deletions(-) diff --git a/configs/checkstyle.xml b/configs/checkstyle.xml index 10d21514e..7cbd93534 100644 --- a/configs/checkstyle.xml +++ b/configs/checkstyle.xml @@ -169,9 +169,16 @@ - + + + + + + + + diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index 81423300d..f42775470 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -13,7 +13,6 @@ import java.util.stream.Collectors; import ru.yandex.clickhouse.domain.ClickHouseDataType; - import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.StateToReproduce; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java index 40803d782..172554b77 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java @@ -1,13 +1,13 @@ package sqlancer.clickhouse.ast; -import sqlancer.Randomly; -import sqlancer.clickhouse.ClickHouseSchema; -import ru.yandex.clickhouse.domain.ClickHouseDataType; - import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import ru.yandex.clickhouse.domain.ClickHouseDataType; +import sqlancer.Randomly; +import sqlancer.clickhouse.ClickHouseSchema; + public class ClickHouseAggregate extends ClickHouseExpression { private final ClickHouseAggregate.ClickHouseAggregateFunction func; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCast.java b/src/sqlancer/clickhouse/ast/ClickHouseCast.java index 9fe2392cf..d0d1bc0b9 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCast.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCast.java @@ -1,13 +1,13 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; - import java.math.BigDecimal; import java.sql.Connection; import java.sql.Statement; import java.util.Optional; import java.util.regex.Pattern; +import ru.yandex.clickhouse.domain.ClickHouseDataType; + public final class ClickHouseCast extends ClickHouseExpression { private static final double MAX_INT_FOR_WHICH_CONVERSION_TO_INT_IS_TRIED = Math.pow(2, 51 - 1) - 1; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java index f47d2dca4..490a77d37 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java @@ -1,7 +1,6 @@ package sqlancer.clickhouse.ast; import ru.yandex.clickhouse.domain.ClickHouseDataType; - import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; public class ClickHouseCastOperation extends ClickHouseExpression { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java index 54a3da2a9..b70b0841f 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java @@ -1,10 +1,10 @@ package sqlancer.clickhouse.ast; +import java.math.BigInteger; + import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.IgnoreMeException; -import java.math.BigInteger; - public abstract class ClickHouseConstant extends ClickHouseExpression { public static class ClickHouseNullConstant extends ClickHouseConstant { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java index 4f90b0760..41f91ef56 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java @@ -1,9 +1,9 @@ package sqlancer.clickhouse.ast; +import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Randomly; import sqlancer.ast.BinaryOperatorNode.Operator; import sqlancer.visitor.UnaryOperation; -import ru.yandex.clickhouse.domain.ClickHouseDataType; public class ClickHouseUnaryPrefixOperation extends ClickHouseExpression implements UnaryOperation { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java index c12213f97..cf78388cb 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java @@ -1,14 +1,14 @@ package sqlancer.clickhouse.gen; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - public class ClickHouseColumnBuilder { private final StringBuilder sb = new StringBuilder(); diff --git a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java index cff33a94d..612e807d5 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java @@ -1,12 +1,12 @@ package sqlancer.clickhouse.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseTableReference; -import java.util.ArrayList; -import java.util.List; - public final class ClickHouseCommon { private ClickHouseCommon() { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index f279b8cc1..9943f455f 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -17,8 +17,8 @@ import sqlancer.clickhouse.ast.ClickHouseConstant; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; -import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator; +import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; import sqlancer.gen.TypedExpressionGenerator; diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index b516ef450..3eb0e1537 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -1,15 +1,14 @@ package sqlancer.clickhouse.gen; -import ru.yandex.clickhouse.domain.ClickHouseDataType; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.Randomly; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import ru.yandex.clickhouse.domain.ClickHouseDataType; +import sqlancer.Query; +import sqlancer.QueryAdapter; +import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseSchema; diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 001cdc547..f101e1842 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -1,5 +1,10 @@ package sqlancer.clickhouse.oracle.tlp; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; @@ -17,11 +22,6 @@ import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; - public class ClickHouseTLPAggregateOracle implements TestOracle { private final ClickHouseProvider.ClickHouseGlobalState state; diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java index 72523fdf1..3165306ae 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java @@ -1,14 +1,14 @@ package sqlancer.clickhouse.oracle.tlp; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + import sqlancer.ComparatorHelper; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseSelect; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - public class ClickHouseTLPDistinctOracle extends ClickHouseTLPBase { public ClickHouseTLPDistinctOracle(ClickHouseProvider.ClickHouseGlobalState state) { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index 5f4abdc42..bb3629604 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -1,5 +1,10 @@ package sqlancer.clickhouse.oracle.tlp; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider; @@ -7,11 +12,6 @@ import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - public class ClickHouseTLPGroupByOracle extends ClickHouseTLPBase { public ClickHouseTLPGroupByOracle(ClickHouseProvider.ClickHouseGlobalState state) { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index 40393fd59..f6268ab30 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -1,5 +1,10 @@ package sqlancer.clickhouse.oracle.tlp; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; @@ -14,11 +19,6 @@ import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import java.sql.SQLException; -import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; - public class ClickHouseTLPHavingOracle extends ClickHouseTLPBase { public ClickHouseTLPHavingOracle(ClickHouseProvider.ClickHouseGlobalState state) { diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index b1edd66cd..c35c9992a 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -1,12 +1,12 @@ package sqlancer.postgres.oracle.tlp; +import java.io.IOException; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.io.IOException; import org.postgresql.util.PSQLException; import sqlancer.ComparatorHelper; @@ -139,13 +139,16 @@ private List mapped(PostgresAggregate aggregate) { case MIN: return aliasArgs(Arrays.asList(aggregate)); // case AVG: - //// List arg = Arrays.asList(new PostgresCast(aggregate.getExpr().get(0), + //// List arg = Arrays.asList(new + // PostgresCast(aggregate.getExpr().get(0), // PostgresDataType.DECIMAL.get())); - // PostgresAggregate sum = new PostgresAggregate(PostgresAggregateFunction.SUM, aggregate.getExpr()); + // PostgresAggregate sum = new PostgresAggregate(PostgresAggregateFunction.SUM, + // aggregate.getExpr()); // PostgresCast count = new PostgresCast( // new PostgresAggregate(PostgresAggregateFunction.COUNT, aggregate.getExpr()), // PostgresDataType.DECIMAL.get()); - //// PostgresBinaryArithmeticOperation avg = new PostgresBinaryArithmeticOperation(sum, count, + //// PostgresBinaryArithmeticOperation avg = new + // PostgresBinaryArithmeticOperation(sum, count, // PostgresBinaryArithmeticOperator.DIV); // return aliasArgs(Arrays.asList(sum, count)); default: diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index cd6448809..68f994cd7 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -1,9 +1,9 @@ package sqlancer.postgres.oracle.tlp; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.ArrayList; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -12,17 +12,17 @@ import sqlancer.gen.ExpressionGenerator; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresSchema.PostgresTables; -import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresColumnValue; +import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.ForClause; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; -import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; import sqlancer.postgres.oracle.PostgresNoRECOracle; From dc08003c326ccc469883564c374afbc9d7b038cf Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2020 08:22:02 +0000 Subject: [PATCH 0050/1284] Bump duckdb_jdbc from 0.1.9 to 0.2.0 Bumps [duckdb_jdbc](https://github.com/cwida/duckdb) from 0.1.9 to 0.2.0. - [Release notes](https://github.com/cwida/duckdb/releases) - [Commits](https://github.com/cwida/duckdb/compare/v0.1.9...v0.2.0) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9e8dda10a..0553988e4 100644 --- a/pom.xml +++ b/pom.xml @@ -211,7 +211,7 @@ org.duckdb duckdb_jdbc - 0.1.9 + 0.2.0 org.junit.jupiter From a9b575dad7097be5d0af1d9a52060a50fb69d457 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2020 09:36:43 +0000 Subject: [PATCH 0051/1284] Bump mariadb-java-client from 2.6.1 to 2.6.2 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.6.1 to 2.6.2. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/2.6.1...2.6.2) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0553988e4..6c8b3a119 100644 --- a/pom.xml +++ b/pom.xml @@ -206,7 +206,7 @@ org.mariadb.jdbc mariadb-java-client - 2.6.1 + 2.6.2 org.duckdb From 8185632150c18788e7dc22d3ea54c3debae75548 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Fri, 24 Jul 2020 18:00:14 +0300 Subject: [PATCH 0052/1284] Support SELECT * --- src/sqlancer/postgres/PostgresSchema.java | 4 ++++ src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 2b0ae1499..2a76d939f 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -50,6 +50,10 @@ public PostgresColumn(String name, PostgresDataType columnType) { super(name, null, columnType); } + public static PostgresColumn createDummy(String name) { + return new PostgresColumn(name, PostgresDataType.INT); + } + } public static class PostgresTables extends AbstractTables { diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 68f994cd7..facdbfc02 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -12,6 +12,7 @@ import sqlancer.gen.ExpressionGenerator; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresSchema.PostgresTables; @@ -65,7 +66,15 @@ public void check() throws SQLException { } List generateFetchColumns() { - return Arrays.asList(new PostgresColumnValue(targetTables.getColumns().get(0), null)); + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new PostgresColumnValue(PostgresColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(targetTables.getColumns()); + for (PostgresColumn c : targetColumns) { + fetchColumns.add(new PostgresColumnValue(c, null)); + } + return fetchColumns; } @Override From e21c0796bfff3a8f12d708e65ac83d3ca8e49ca2 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Sat, 25 Jul 2020 16:53:49 +0300 Subject: [PATCH 0053/1284] Allow option to pass in connection URL as parameter --- src/sqlancer/postgres/PostgresOptions.java | 3 ++ src/sqlancer/postgres/PostgresProvider.java | 51 ++++++++++++++++++--- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index df766d530..3475cc0cd 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -28,6 +28,9 @@ public class PostgresOptions { @Parameter(names = "--test-collations", arity = 1) public boolean testCollations = true; + @Parameter(names = "--connection-url") + public String connectionURL = "postgresql://localhost:5432/test"; + public enum PostgresOracle { NOREC { @Override diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 1d66b6450..4816f16e5 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -2,6 +2,8 @@ import java.io.FileWriter; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -218,15 +220,46 @@ protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLEx @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { - String url = "jdbc:postgresql://localhost:5432/test"; + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String entryPath = "/test"; + String entryURL = globalState.getDmbsSpecificOptions().connectionURL; + // trim URL to exclude "jdbc:" + if (entryURL.startsWith("jdbc:")) { + entryURL = entryURL.substring(5); + } + try { + URI uri = new URI(entryURL); + String userInfoURI = uri.getUserInfo(); + String pathURI = uri.getPath(); + if (userInfoURI != null) { + // username and password specified in URL take precedence + if (userInfoURI.contains(":")) { + String[] userInfo = userInfoURI.split(":", 2); + username = userInfo[0]; + password = userInfo[1]; + } else { + username = userInfoURI; + password = null; + } + int userInfoIndex = entryURL.indexOf(userInfoURI); + String preUserInfo = entryURL.substring(0, userInfoIndex); + String postUserInfo = entryURL.substring(userInfoIndex + userInfoURI.length() + 1); + entryURL = preUserInfo + postUserInfo; + } + if (pathURI != null) { + entryPath = pathURI; + } + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + String entryDatabaseName = entryPath.substring(1); String databaseName = globalState.getDatabaseName(); - Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); - globalState.getState().logStatement("\\c test;"); + Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); String createDatabaseCommand = getCreateDatabaseCommand(databaseName, con, globalState); globalState.getState().logStatement(createDatabaseCommand); - globalState.getState().logStatement("\\c " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); } @@ -234,8 +267,12 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce s.execute(createDatabaseCommand); } con.close(); - con = DriverManager.getConnection("jdbc:postgresql://localhost:5432/" + databaseName, - globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); + int databaseIndex = entryURL.indexOf(entryPath) + 1; + String preDatabaseName = entryURL.substring(0, databaseIndex); + String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); + String testURL = preDatabaseName + databaseName + postDatabaseName; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + con = DriverManager.getConnection("jdbc:" + testURL, username, password); return con; } From 858dc537eefd9c5a91b510f0b2c51bc007d17372 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 28 Jul 2020 13:43:44 +0300 Subject: [PATCH 0054/1284] Fix in comparing table columns --- src/sqlancer/schema/AbstractTableColumn.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/schema/AbstractTableColumn.java b/src/sqlancer/schema/AbstractTableColumn.java index 07c5b2fee..978ef38a0 100644 --- a/src/sqlancer/schema/AbstractTableColumn.java +++ b/src/sqlancer/schema/AbstractTableColumn.java @@ -52,7 +52,10 @@ public boolean equals(Object obj) { } else { @SuppressWarnings("unchecked") AbstractTableColumn c = (AbstractTableColumn) obj; - return table.getName().contentEquals(getName()) && getName().equals(c.getName()); + if (c.getTable() == null) { + return getName().equals(c.getName()); + } + return table.getName().contentEquals(c.getTable().getName()) && getName().equals(c.getName()); } } From 79b4b179173926d061cc1882db968d8e951ac0cf Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2020 08:35:16 +0000 Subject: [PATCH 0055/1284] Bump sqlite-jdbc from 3.32.3.1 to 3.32.3.2 Bumps [sqlite-jdbc](https://github.com/xerial/sqlite-jdbc) from 3.32.3.1 to 3.32.3.2. - [Release notes](https://github.com/xerial/sqlite-jdbc/releases) - [Changelog](https://github.com/xerial/sqlite-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/xerial/sqlite-jdbc/compare/3.32.3.1...3.32.3.2) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6c8b3a119..660ed886a 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ org.xerial sqlite-jdbc - 3.32.3.1 + 3.32.3.2 mysql From f6428ce51469ea345c41844845b1a24b3cd4674b Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 28 Jul 2020 18:48:07 +0300 Subject: [PATCH 0056/1284] Diversify query execution options, protect from SQL injection using prepared statements --- src/sqlancer/GlobalState.java | 23 +++++++++++- src/sqlancer/Main.java | 14 ++++++- src/sqlancer/Query.java | 4 +- src/sqlancer/QueryAdapter.java | 37 ++++++++++++------- src/sqlancer/QueryResultCheckAdapter.java | 2 +- .../gen/PostgresQueryCatalogGenerator.java | 2 +- 6 files changed, 60 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 0f93ca3a2..4e123616c 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,6 +1,7 @@ package sqlancer; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; import sqlancer.Main.QueryManager; @@ -89,7 +90,7 @@ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } - public boolean executeStatement(Query q) throws SQLException { + public ExecutionTimer executePrologue(Query q) throws SQLException { boolean logExecutionTime = getOptions().logExecutionTime(); ExecutionTimer timer = null; if (logExecutionTime) { @@ -105,7 +106,11 @@ public boolean executeStatement(Query q) throws SQLException { getLogger().writeCurrent(q.getQueryString()); } } - boolean success = manager.execute(q); + return timer; + } + + public void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws SQLException { + boolean logExecutionTime = getOptions().logExecutionTime(); if (success && getOptions().printSucceedingStatements()) { System.out.println(q.getQueryString()); } @@ -115,9 +120,23 @@ public boolean executeStatement(Query q) throws SQLException { if (q.couldAffectSchema()) { updateSchema(); } + } + + public boolean executeStatement(Query q, String... fills) throws SQLException { + ExecutionTimer timer = executePrologue(q); + boolean success = manager.execute(q, fills); + executeEpilogue(q, success, timer); return success; } + public ResultSet executeStatementAndGet(Query q, String... fills) throws SQLException { + ExecutionTimer timer = executePrologue(q); + ResultSet result = manager.executeAndGet(q, fills); + boolean success = result != null; + executeEpilogue(q, success, timer); + return result; + } + public S getSchema() { if (schema == null) { try { diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 36940ef3a..ec197be30 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -8,6 +8,7 @@ import java.io.Writer; import java.nio.file.Files; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.text.DateFormat; @@ -253,13 +254,22 @@ public static class QueryManager { this.globalState = globalState; } - public boolean execute(Query q) throws SQLException { + public boolean execute(Query q, String... fills) throws SQLException { globalState.getState().logStatement(q); - boolean success = q.execute(globalState); + boolean success; + success = q.execute(globalState, fills); Main.nrSuccessfulActions.addAndGet(1); return success; } + public ResultSet executeAndGet(Query q, String... fills) throws SQLException { + globalState.getState().logStatement(q); + ResultSet result; + result = q.executeAndGet(globalState, fills); + Main.nrSuccessfulActions.addAndGet(1); + return result; + } + public void incrementSelectQueryCount() { Main.nrQueries.addAndGet(1); } diff --git a/src/sqlancer/Query.java b/src/sqlancer/Query.java index abb3a5718..c4f8aff55 100644 --- a/src/sqlancer/Query.java +++ b/src/sqlancer/Query.java @@ -23,7 +23,7 @@ public abstract class Query { * * @throws SQLException */ - public abstract boolean execute(GlobalState globalState) throws SQLException; + public abstract boolean execute(GlobalState globalState, String... fills) throws SQLException; public abstract Collection getExpectedErrors(); @@ -32,7 +32,7 @@ public String toString() { return getQueryString(); } - public ResultSet executeAndGet(GlobalState globalState) throws SQLException { + public ResultSet executeAndGet(GlobalState globalState, String... fills) throws SQLException { throw new AssertionError(); } diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index 0853979aa..3db9d32ca 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -1,5 +1,6 @@ package sqlancer; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -57,8 +58,17 @@ public String getQueryString() { } @Override - public boolean execute(GlobalState globalState) throws SQLException { - try (Statement s = globalState.getConnection().createStatement()) { + public boolean execute(GlobalState globalState, String... fills) throws SQLException { + Statement s; + if (fills.length > 0) { + s = globalState.getConnection().prepareStatement(getQueryString()); + for (int i = 0; i < fills.length; i++) { + ((PreparedStatement) s).setString(i, fills[i]); + } + } else { + s = globalState.getConnection().createStatement(); + } + try { s.execute(query); Main.nrSuccessfulActions.addAndGet(1); return true; @@ -83,8 +93,16 @@ public void checkException(Exception e) throws AssertionError { } @Override - public ResultSet executeAndGet(GlobalState globalState) throws SQLException { - Statement s = globalState.getConnection().createStatement(); + public ResultSet executeAndGet(GlobalState globalState, String... fills) throws SQLException { + Statement s; + if (fills.length > 0) { + s = globalState.getConnection().prepareStatement(getQueryString()); + for (int i = 0; i < fills.length; i++) { + ((PreparedStatement) s).setString(i, fills[i]); + } + } else { + s = globalState.getConnection().createStatement(); + } ResultSet result = null; try { result = s.executeQuery(query); @@ -92,17 +110,8 @@ public ResultSet executeAndGet(GlobalState globalState) throws SQLExceptio return result; } catch (Exception e) { s.close(); - boolean isExcluded = false; Main.nrUnsuccessfulActions.addAndGet(1); - for (String expectedError : expectedErrors) { - if (e.getMessage().contains(expectedError)) { - isExcluded = true; - break; - } - } - if (!isExcluded) { - throw e; - } + checkException(e); } return null; } diff --git a/src/sqlancer/QueryResultCheckAdapter.java b/src/sqlancer/QueryResultCheckAdapter.java index e0c131ed8..ba0dacfa8 100644 --- a/src/sqlancer/QueryResultCheckAdapter.java +++ b/src/sqlancer/QueryResultCheckAdapter.java @@ -15,7 +15,7 @@ public QueryResultCheckAdapter(String query, Consumer rsChecker) { } @Override - public boolean execute(GlobalState globalState) throws SQLException { + public boolean execute(GlobalState globalState, String... fills) throws SQLException { try (Statement s = globalState.getConnection().createStatement()) { ResultSet rs = s.executeQuery(getQueryString()); rsChecker.accept(rs); diff --git a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java index 6c2a60646..9f4f79cc3 100644 --- a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java @@ -33,7 +33,7 @@ public static Query query() { "pg_ts_template", "pg_type", "pg_user_mapping")); return new QueryAdapter(sb.toString()) { @Override - public boolean execute(GlobalState globalState) throws SQLException { + public boolean execute(GlobalState globalState, String... fills) throws SQLException { try (Statement s = globalState.getConnection().createStatement()) { try (ResultSet rs = s.executeQuery(getQueryString())) { // CHECKSTYLE:OFF From b1e08d7a8b683d416f0faf9423d9f257050a53ae Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 30 Jul 2020 09:14:52 +0200 Subject: [PATCH 0057/1284] [SQLite3] Ignore newly-appearing SQLite3 VACUUM error --- src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java index a2fc69566..a6a2af842 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java @@ -21,7 +21,8 @@ public static Query executeVacuum(SQLite3GlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("temp", "main")); } - return new QueryAdapter(sb.toString(), Arrays.asList("cannot VACUUM from within a transaction")); + return new QueryAdapter(sb.toString(), + Arrays.asList("cannot VACUUM from within a transaction", "cannot VACUUM - SQL statements in progress")); } } From ddadb7d492dab0939b88533228008d48eeb4fd0a Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 30 Jul 2020 13:27:16 +0300 Subject: [PATCH 0058/1284] Check for null second result in PostgresTLPAggregateOracle --- .../postgres/oracle/tlp/PostgresTLPAggregateOracle.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index c35c9992a..0da4bf239 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -68,10 +68,10 @@ public void check() throws SQLException { String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); - if (firstResult == null && secondResult != null - || firstResult != null && (!firstResult.contentEquals(secondResult) - && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { - if (secondResult.contains("Inf")) { + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { throw new IgnoreMeException(); // FIXME: average computation } String assertionMessage = String.format("the results mismatch!\n%s\n%s", firstQueryString, From 74d9514bbf5d8221b785341bc4ecd031f93ddea1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 2 Aug 2020 12:29:06 +0200 Subject: [PATCH 0059/1284] [CockroachDB] Use the composite test oracle rather than a custom implementation --- .../cockroachdb/CockroachDBOptions.java | 13 ++++++-- .../oracle/tlp/CockroachDBTLPOracle.java | 32 ------------------- 2 files changed, 11 insertions(+), 34 deletions(-) delete mode 100644 src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPOracle.java diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 71da218f6..9c6141e53 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -1,10 +1,13 @@ package sqlancer.cockroachdb; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import sqlancer.CompositeTestOracle; import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; @@ -14,7 +17,6 @@ import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPJoinOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPWhereOracle; @Parameters(separators = "=", commandDescription = "Test CockroachDB") @@ -77,7 +79,14 @@ public TestOracle create(CockroachDBGlobalState globalState) throws SQLException QUERY_PARTITIONING { @Override public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPOracle(globalState); + List oracles = new ArrayList<>(); + oracles.add(new CockroachDBTLPAggregateOracle(globalState)); + oracles.add(new CockroachDBTLPHavingOracle(globalState)); + oracles.add(new CockroachDBTLPWhereOracle(globalState)); + oracles.add(new CockroachDBTLPGroupByOracle(globalState)); + oracles.add(new CockroachDBTLPExtendedWhereOracle(globalState)); + oracles.add(new CockroachDBTLPDistinctOracle(globalState)); + return new CompositeTestOracle(oracles); } }; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPOracle.java deleted file mode 100644 index 41d6b9269..000000000 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPOracle.java +++ /dev/null @@ -1,32 +0,0 @@ -package sqlancer.cockroachdb.oracle.tlp; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.TestOracle; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; - -public class CockroachDBTLPOracle implements TestOracle { - - private final TestOracle[] oracles; - private int i; - - public CockroachDBTLPOracle(CockroachDBGlobalState state) { - List oracles = new ArrayList<>(); - oracles.add(new CockroachDBTLPAggregateOracle(state)); - oracles.add(new CockroachDBTLPHavingOracle(state)); - oracles.add(new CockroachDBTLPWhereOracle(state)); - oracles.add(new CockroachDBTLPGroupByOracle(state)); - oracles.add(new CockroachDBTLPExtendedWhereOracle(state)); - oracles.add(new CockroachDBTLPDistinctOracle(state)); - this.oracles = oracles.toArray(new TestOracle[4]); - } - - @Override - public void check() throws SQLException { - oracles[i].check(); - i = (i + 1) % oracles.length; - } - -} From df54b2e61566ad1d8308d86da02735f4e26cb26a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 2 Aug 2020 15:19:34 +0200 Subject: [PATCH 0060/1284] Update the bibliography after the conditionally-accepted OOPSLA paper --- docs/PAPERS.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/PAPERS.md b/docs/PAPERS.md index 0ed6390c7..dcd0162ed 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -17,7 +17,7 @@ This paper describes PQS, a testing approach to detect various kinds of logic bu ## Detecting Optimization Bugs in Database Engines via Non-Optimizing Reference Engine Construction -This paper describes NoREC, a metamorphic testing approach to detect optimization bugs, that is, logic bugs that affect the query optimizer. A preprint is available [here](https://www.manuelrigger.at/preprints/NoREC.pdf). +This paper describes NoREC, a metamorphic testing approach to detect optimization bugs, that is, logic bugs that affect the query optimizer. A preprint is available [here](https://arxiv.org/abs/2007.08292). ``` @inproceedings{Rigger2020NoREC, @@ -26,18 +26,22 @@ This paper describes NoREC, a metamorphic testing approach to detect optimizatio booktitle = {Proceedings of the 2020 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering}, series={ESEC/FSE 2020}, location={Sacramento, California, United States}, - year={2020} + year={2020}, + doi={10.1145/3368089.3409710} } ``` ## Ternary Logic Partitioning: Detecting Logic Bugs in Database Management Systems -This paper describes TLP, a metamorphic testing approach that can detect various kinds of logic bugs and is applicable also test features such as aggregate functions. A preprint is available [here](https://www.manuelrigger.at/preprints/TLP.pdf). +This paper describes TLP, a metamorphic testing approach that can detect various kinds of logic bugs and is applicable also to test features such as aggregate functions. A preprint is available [here](https://www.manuelrigger.at/preprints/TLP.pdf). ``` -@inproceedings{Rigger2020TLP, +@article{Rigger2020TLP, author={Manuel Rigger and Zhendong Su}, - title={Ternary Logic Partitioning: Detecting Logic Bugs in Database Management Systems}, - year={2020} + title={Finding Bugs in Database Systems via Query Partitioning}, + journal = {Proc. ACM Program. Lang.}, + number = {OOPSLA}, + year={2020}, + note={Conditionally accepted} } ``` From aad5d66472e027b1745dddf594de5627ea769ed5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Jul 2020 23:34:39 +0200 Subject: [PATCH 0061/1284] Implement a local state class and replace the queryString in the state --- src/sqlancer/ComparatorHelper.java | 4 +- src/sqlancer/Main.java | 3 - src/sqlancer/ProviderAdapter.java | 15 +++-- src/sqlancer/StateToReproduce.java | 62 ++++++++++++++++--- .../tlp/ClickHouseTLPAggregateOracle.java | 4 +- .../oracle/CockroachDBNoRECOracle.java | 2 +- .../tlp/CockroachDBTLPAggregateOracle.java | 4 +- ...uckDBQueryPartitioningAggregateTester.java | 4 +- .../mariadb/oracle/MariaDBNoRECOracle.java | 2 +- .../MySQLPivotedQuerySynthesisOracle.java | 2 +- .../postgres/oracle/PostgresNoRECOracle.java | 6 +- .../PostgresPivotedQuerySynthesisOracle.java | 8 +-- .../tlp/PostgresTLPAggregateOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPBase.java | 2 - .../sqlite3/oracle/SQLite3NoRECOracle.java | 2 +- .../SQLite3PivotedQuerySynthesisOracle.java | 2 +- .../oracle/tlp/SQLite3TLPAggregateOracle.java | 4 +- 17 files changed, 84 insertions(+), 44 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 93448a9c4..07986bd54 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -89,7 +89,7 @@ public static void assumeResultSetsAreEqual(List resultSet, List String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); String secondQueryString = String.format(queryFormatString, combinedString.stream().collect(Collectors.joining(";")), secondResultSet.size()); - state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); throw new AssertionError(assertionMessage); @@ -108,7 +108,7 @@ public static void assumeResultSetsAreEqual(List resultSet, List String secondQueryString = String.format(queryFormatString, combinedString.stream().collect(Collectors.joining(";")), secondResultSetMisses); // update the SELECT queries to be logged at the bottom of the error log file - state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", firstQueryString, secondQueryString); throw new AssertionError(assertionMessage); diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index ec197be30..62c999a3c 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -233,9 +233,6 @@ private void printState(FileWriter writer, StateToReproduce state) { } sb.append('\n'); } - if (state.getQueryString() != null) { - sb.append(state.getQueryString() + ";\n"); - } try { writer.write(sb.toString()); } catch (IOException e) { diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 0f4c5bdf5..f132627bd 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -3,6 +3,8 @@ import java.io.FileWriter; import java.sql.SQLException; +import sqlancer.StateToReproduce.OracleRunReproductionState; + public abstract class ProviderAdapter, O> implements DatabaseProvider { private final Class globalClass; @@ -41,11 +43,16 @@ public void generateAndTestDatabase(G globalState) throws SQLException { TestOracle oracle = getTestOracle(globalState); for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { - try { - oracle.check(); - globalState.getManager().incrementSelectQueryCount(); - } catch (IgnoreMeException e) { + try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { + assert localState != null; + try { + oracle.check(); + globalState.getManager().incrementSelectQueryCount(); + } catch (IgnoreMeException e) { + } + assert localState != null; + localState.executedWithoutError(); } } } finally { diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index f640eae2d..67e2e62b3 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -1,5 +1,6 @@ package sqlancer; +import java.io.Closeable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -22,12 +23,6 @@ public class StateToReproduce { private final List statements = new ArrayList<>(); - /** - * The string printed at the bottom of the error log file, which contains the queries that caused the test to fail - * and information about their results. - */ - public String queryString; - private final String databaseName; public String databaseVersion; @@ -42,6 +37,8 @@ public class StateToReproduce { public String queryTargetedColumnsString; + public OracleRunReproductionState localState; + public StateToReproduce(String databaseName) { this.databaseName = databaseName; } @@ -88,14 +85,20 @@ public List getStatements() { return Collections.unmodifiableList(statements); } - public String getQueryString() { - return queryString; - } - public long getSeedValue() { return seedValue; } + /** + * Returns a local state in which a test oracle can save useful information about a single run. If the local state + * is closed without indicating access to it, the local statements will be added to the global state. + * + * @return + */ + public OracleRunReproductionState getLocalState() { + return localState; + } + public static class MySQLStateToReproduce extends StateToReproduce { public Map randomRowValues; @@ -181,4 +184,43 @@ public ClickHouseExpression getWhereClause() { } + /** + * State information that is logged if the test oracle finds a bug or if an exception is thrown. + */ + public class OracleRunReproductionState implements Closeable { + + private final List statements = new ArrayList<>(); + + public boolean success; + + public OracleRunReproductionState() { + StateToReproduce.this.localState = this; + } + + public void executedWithoutError() { + this.success = true; + } + + public void log(Query q) { + statements.add(q); + } + + public void log(String s) { + statements.add(new QueryAdapter(s)); + } + + @Override + public void close() { + if (!success) { + StateToReproduce.this.statements.addAll(statements); + } + + } + + } + + public OracleRunReproductionState createLocalState() { + return new OracleRunReproductionState(); + } + } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index f101e1842..2526d59ef 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -89,8 +89,8 @@ public void check() throws SQLException { // TODO throw new IgnoreMeException(); } - state.getState().queryString = "--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " - + secondResult; + state.getState().getLocalState() + .log("--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " + secondResult); if ((firstResult == null && secondResult != null || firstResult != null && !firstResult.contentEquals(secondResult)) && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 6ce80fb73..adac39470 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -60,7 +60,7 @@ public void check() throws SQLException { throw new IgnoreMeException(); } if (optimizableCount != nonOptimizableCount) { - state.getState().queryString = optimizedQueryString + ";\n" + unoptimizedQueryString + ";"; + state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); throw new AssertionError(CockroachDBVisitor.asString(whereCondition)); } } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index c6b531ccd..e4f86b53b 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -84,8 +84,8 @@ public void check() throws SQLException { metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, from); secondResult = getAggregateResult(metamorphicQuery); - state.getState().queryString = "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult - + "\n-- " + secondResult; + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); if (firstResult == null && secondResult != null || firstResult != null && (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index e46cff858..4771f3117 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -64,8 +64,8 @@ public void check() throws SQLException { metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); secondResult = getAggregateResult(metamorphicQuery); - state.getState().queryString = "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult - + "\n-- " + secondResult; + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); if (firstResult == null && secondResult != null || firstResult != null && (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index 4f7b9b413..cf45121a9 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -63,7 +63,7 @@ public void check() throws SQLException { throw new IgnoreMeException(); } if (optimizedCount != unoptimizedCount) { - state.getState().queryString = optimizedQueryString + ";\n" + unoptimizedQueryString + ";"; + state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); throw new AssertionError(optimizedCount + " " + unoptimizedCount); } } diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 2cf9bc991..37b205dc5 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -219,7 +219,7 @@ private boolean isContainedIn(String queryString) throws SQLException { } String resultingQueryString = sb.toString(); - state.queryString = resultingQueryString; + state.getLocalState().log(resultingQueryString); if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(resultingQueryString); } diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 2a7d667f3..751f69c00 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -49,8 +49,6 @@ public PostgresNoRECOracle(PostgresGlobalState globalState) { @Override public void check() throws SQLException { - // clear left-over query string from previous test - state.getState().queryString = null; PostgresTables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); PostgresExpression randomWhereCondition = getRandomWhereCondition(columns); @@ -68,8 +66,8 @@ public void check() throws SQLException { String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().queryString = String.format("%s\n%s", firstQueryStringWithCount, - secondQueryStringWithCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, firstQueryStringWithCount, secondQueryStringWithCount); throw new AssertionError(assertionMessage); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 287ca975b..aff969f45 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -49,12 +49,10 @@ public PostgresPivotedQuerySynthesisOracle(PostgresGlobalState globalState) thro @Override public void check() throws SQLException { - // clear left-over query string from previous test - state.queryString = null; String queryString = getQueryThatContainsAtLeastOneRow(state); - state.queryString = queryString; + state.getLocalState().log(queryString); if (options.logEachSelect()) { - logger.writeCurrent(state.queryString); + logger.writeCurrent(queryString); } boolean isContainedIn = isContainedIn(queryString, options, logger); @@ -179,7 +177,7 @@ private boolean isContainedIn(String queryString, MainOptions options, StateLogg } String resultingQueryString = sb.toString(); // log both SELECT queries at the bottom of the error log file - state.queryString = String.format("-- %s;\n-- %s;", queryString, resultingQueryString); + state.getLocalState().log(String.format("-- %s;\n-- %s;", queryString, resultingQueryString)); if (options.logEachSelect()) { logger.writeCurrent(resultingQueryString); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index 0da4bf239..fbf2b9897 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -67,7 +67,7 @@ public void check() throws SQLException { String queryFormatString = "-- %s;\n-- result: %s"; String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); - state.getState().queryString = String.format("%s\n%s", firstQueryString, secondQueryString); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); if (firstResult == null && secondResult != null || firstResult != null && secondResult == null || firstResult != null && !firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index facdbfc02..36a353a2e 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -44,8 +44,6 @@ public PostgresTLPBase(PostgresGlobalState state) { @Override public void check() throws SQLException { - // clear left-over query string from previous test - state.getState().queryString = null; s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new PostgresExpressionGenerator(state).setColumns(targetTables.getColumns()); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 1cea27a2c..6752a70ea 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -68,7 +68,7 @@ public void check() throws SQLException { throw new IgnoreMeException(); } if (optimizedCount != unoptimizedCount) { - state.getState().queryString = optimizedQueryString + ";\n" + unoptimizedQueryString + ";"; + state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); throw new AssertionError(optimizedCount + " " + unoptimizedCount); } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 3a479ebd6..fb0c65b8f 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -232,7 +232,7 @@ private boolean isContainedIn(Query query) throws SQLException { sb.append(query.getQueryString()); sb.append(")"); String resultingQueryString = sb.toString(); - state.queryString = resultingQueryString; + state.getLocalState().log(resultingQueryString); Query finalQuery = new QueryAdapter(resultingQueryString, query.getExpectedErrors()); try (ResultSet result = createStatement.executeQuery(finalQuery.getQueryString())) { boolean isContainedIn = !result.isClosed(); diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 6a693d3d1..be6eeb039 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -95,8 +95,8 @@ public void check() throws SQLException { // TODO throw new IgnoreMeException(); } - state.getState().queryString = "--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " - + secondResult; + state.getState().getLocalState() + .log("--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " + secondResult); if ((firstResult == null && secondResult != null || firstResult != null && !firstResult.contentEquals(secondResult)) && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { From 0a3a09c5c74b89ec49f28569a411c8140a98a915 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 2 Aug 2020 22:44:33 +0200 Subject: [PATCH 0062/1284] Describe the testing of SQLancer --- docs/DEVELOPMENT.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 648d8ad0c..cdd3ad432 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -57,7 +57,23 @@ We use [Travis-CI](https://travis-ci.com/) to automatically check PRs. ## Testing -We found that bugs in SQLancer are quickly found and easy to debug when testing the DBMS. However, it would still be preferable to automatically check that SQLancer still executes as expected. To this end, we would like to add smoke testing for each DBMS to test that the respective testing implementation is not obviously broken, see [here](https://github.com/sqlancer/sqlancer/issues/3). +As part of the Travis-CI gate, we use smoke testing by running SQLancer on each supported DBMS for some minutes, to test that nothing is obviously broken. For DBMS for which all bugs have been fixed, we verify that SQLancer cannot find any further bugs (i.e., the return code is zero). + +In addition, we use [unit tests](https://github.com/sqlancer/sqlancer/tree/master/test/sqlancer) to test SQLancer's core functionality, such as random string and number generation as well as option passing. When fixing a bug, add a unit test, if it is easily possible. + +You can run the tests using the following command: + +``` +mvn test +``` + +Note that per default, the smoke testing is performed only for embedded DBMS (i.e., DuckDB and SQLite). To run smoke tests also for the other DBMS, you need to set environment variables. For example, you can run the MySQL smoke testing (and no other tests) using the following command: + +``` +MYSQL_AVAILABLE=true mvn -Dtest=TestMySQL test +``` + +For up-to-date testing commands, check out the `.travis.yml` file. ## Naming Conventions From 1abc586ddaa1fa0508baaab83f72d6e5e7ab9d5c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 2 Aug 2020 22:55:25 +0200 Subject: [PATCH 0063/1284] Describe the reviewing process --- docs/DEVELOPMENT.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index cdd3ad432..81177e352 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -30,8 +30,6 @@ For a permissive DBMS, implementing the expression generator is easier, since th For a strict DBMS, the better approach is typically to attempt to generate expressions of the expected type. For PostgreSQL, the expression generator thus expects an additional type argument (see [PostgreSQLExpressionGenerator](https://github.com/sqlancer/sqlancer/blob/86647df8aa2dd8d167b5c3ce3297290f5b0b2bcd/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java#L251)). This type is propagated recursively. For example, if we require a predicate for the `WHERE` clause, we pass boolean as a type. The expression generator then calls a method `generateBooleanExpression` that attempts to produce a boolean expression, by, for example, generating a comparison (e.g., `<=`). For the comparison's operands, a random type is then selected and propagated. For example, if an integer type is selected, then `generateExpression` is called with this type once for the left operand, and once for the right operand. Note that this process does not guarantee that the expression will indeed have the expected type. It might happen, for example, that the expression generator attempts to produce an integer value, but that it produces a double value instead, namely when an integer overflow occurs, which, depending on the DBMS, implicitly converts the result to a floating-point value. - - ## Options SQLancer uses [JCommander](https://jcommander.org/) for handling options. The `MainOptions` class contains options that are expected to be supported by all DBMS-testing implementations. Furthermore, each `*Provider` class provides a method to return an additional set of supported options. @@ -75,6 +73,10 @@ MYSQL_AVAILABLE=true mvn -Dtest=TestMySQL test For up-to-date testing commands, check out the `.travis.yml` file. +## Reviewing + +Reviewing is an effective way of improving code quality. Everyone is welcome to review any PRs. Currently, all PRs are reviewed at least by the main contributor, @mrigger. Contributions by @mrigger are currently not (necessarily) reviewed, which is not ideal. If you are willing to regularly and timely review PRs, indicate so in the SQLancer Slack workspace. + ## Naming Conventions Each class specific to a DBMS is prefixed by the DBMS name. For example, each class specific to SQLite is prefixed by `SQLite3`. The naming convention is [automatically checked](src/check_names.py). From 48fb0029b500c229ee6be2b25100dac4384bdd5d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 2 Aug 2020 23:01:33 +0200 Subject: [PATCH 0064/1284] Use the GitHub naming convention and file location for the contribution guidelines --- docs/DEVELOPMENT.md => CONTRIBUTING.md | 0 README.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/DEVELOPMENT.md => CONTRIBUTING.md (100%) diff --git a/docs/DEVELOPMENT.md b/CONTRIBUTING.md similarity index 100% rename from docs/DEVELOPMENT.md rename to CONTRIBUTING.md diff --git a/README.md b/README.md index 761a0bfa8..399123444 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_inv # Additional Documentation -* [Contributing to SQLancer](docs/DEVELOPMENT.md) +* [Contributing to SQLancer](CONTRIBUTING.md) * [Papers and .bib entries](docs/PAPERS.md) # Additional Resources From 6639739615e8fc6c1dc462b94d329fca2a79aefb Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 2 Aug 2020 23:21:01 +0200 Subject: [PATCH 0065/1284] Fix the reporting of the issued queries count when using the composite test oracle --- src/sqlancer/CompositeTestOracle.java | 8 +++++++- src/sqlancer/clickhouse/ClickHouseProvider.java | 2 +- src/sqlancer/cockroachdb/CockroachDBOptions.java | 2 +- src/sqlancer/duckdb/DuckDBOptions.java | 2 +- src/sqlancer/duckdb/DuckDBProvider.java | 2 +- src/sqlancer/postgres/PostgresOptions.java | 2 +- src/sqlancer/postgres/PostgresProvider.java | 2 +- src/sqlancer/sqlite3/SQLite3Options.java | 2 +- src/sqlancer/tidb/TiDBOptions.java | 2 +- src/sqlancer/tidb/TiDBProvider.java | 2 +- 10 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/CompositeTestOracle.java b/src/sqlancer/CompositeTestOracle.java index 9851ee24c..e83194823 100644 --- a/src/sqlancer/CompositeTestOracle.java +++ b/src/sqlancer/CompositeTestOracle.java @@ -6,9 +6,11 @@ public class CompositeTestOracle implements TestOracle { private final TestOracle[] oracles; + private final GlobalState globalState; private int i; - public CompositeTestOracle(List oracles) { + public CompositeTestOracle(List oracles, GlobalState globalState) { + this.globalState = globalState; this.oracles = oracles.toArray(new TestOracle[oracles.size()]); } @@ -16,6 +18,10 @@ public CompositeTestOracle(List oracles) { public void check() throws SQLException { try { oracles[i].check(); + boolean lastOracleIndex = i == oracles.length - 1; + if (!lastOracleIndex) { + globalState.getManager().incrementSelectQueryCount(); + } } finally { i = (i + 1) % oracles.length; } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 9adca52ba..37e998ddb 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -111,7 +111,7 @@ protected TestOracle getTestOracle(ClickHouseGlobalState globalState) throws SQL throw new AssertionError(e1); } }).collect(Collectors.toList()); - return new CompositeTestOracle(oracles); + return new CompositeTestOracle(oracles, globalState); } @Override diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 9c6141e53..75e92a776 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -86,7 +86,7 @@ public TestOracle create(CockroachDBGlobalState globalState) throws SQLException oracles.add(new CockroachDBTLPGroupByOracle(globalState)); oracles.add(new CockroachDBTLPExtendedWhereOracle(globalState)); oracles.add(new CockroachDBTLPDistinctOracle(globalState)); - return new CompositeTestOracle(oracles); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index bd33a81d4..1c633cbc4 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -140,7 +140,7 @@ public TestOracle create(DuckDBGlobalState globalState) throws SQLException { oracles.add(new DuckDBQueryPartitioningAggregateTester(globalState)); oracles.add(new DuckDBQueryPartitioningDistinctTester(globalState)); oracles.add(new DuckDBQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index e2383beb1..6ff8b35b8 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -127,7 +127,7 @@ protected TestOracle getTestOracle(DuckDBGlobalState globalState) throws SQLExce } catch (SQLException e1) { throw new AssertionError(e1); } - }).collect(Collectors.toList())); + }).collect(Collectors.toList()), globalState); } @Override diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 3475cc0cd..e817e3f62 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -59,7 +59,7 @@ public TestOracle create(PostgresGlobalState globalState) throws SQLException { oracles.add(new PostgresTLPWhereOracle(globalState)); oracles.add(new PostgresTLPHavingOracle(globalState)); oracles.add(new PostgresTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 4816f16e5..64dfde48a 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -215,7 +215,7 @@ protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLEx throw new AssertionError(e1); } }).collect(Collectors.toList()); - return new CompositeTestOracle(oracles); + return new CompositeTestOracle(oracles, globalState); } @Override diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 90c964360..5d86d8baa 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -146,7 +146,7 @@ public TestOracle create(SQLite3GlobalState globalState) throws SQLException { oracles.add(new SQLite3TLPGroupByOracle(globalState)); oracles.add(new SQLite3TLPHavingOracle(globalState)); oracles.add(new SQLite3TLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 944491114..b24102f6a 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -39,7 +39,7 @@ public TestOracle create(TiDBGlobalState globalState) throws SQLException { List oracles = new ArrayList<>(); oracles.add(new TiDBTLPWhereOracle(globalState)); oracles.add(new TiDBTLPHavingOracle(globalState)); - return new CompositeTestOracle(oracles); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 0cc308dd1..cec4caffd 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -144,7 +144,7 @@ protected TestOracle getTestOracle(TiDBGlobalState globalState) throws SQLExcept throw new AssertionError(e1); } }).collect(Collectors.toList()); - return new CompositeTestOracle(oracles); + return new CompositeTestOracle(oracles, globalState); } @Override From 3b105fb657286d9f1143b6f2cd45eb584e821901 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 2 Aug 2020 23:28:18 +0200 Subject: [PATCH 0066/1284] Only start the progress monitor once Fixes https://github.com/sqlancer/sqlancer/issues/90. --- src/sqlancer/Main.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 62c999a3c..c14ec75d1 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -44,6 +44,7 @@ public final class Main { public static volatile AtomicLong nrSuccessfulActions = new AtomicLong(); public static volatile AtomicLong nrUnsuccessfulActions = new AtomicLong(); static int threadsShutdown; + static boolean progressMonitorStarted; static { System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "ERROR"); @@ -547,7 +548,16 @@ private boolean run(MainOptions options, ExecutorService execService, return providers; } - private static void startProgressMonitor() { + private static synchronized void startProgressMonitor() { + if (progressMonitorStarted) { + /* + * it might be already started if, for example, the main method is called multiple times in a test (see + * https://github.com/sqlancer/sqlancer/issues/90). + */ + return; + } else { + progressMonitorStarted = true; + } final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); scheduler.scheduleAtFixedRate(new Runnable() { From fd69d9ee30bafb97026d7cfe89023b841ff8ddd3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 4 Aug 2020 13:24:44 +0200 Subject: [PATCH 0067/1284] [SQLite3] Fix new SQLite error --- src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java | 4 +++- src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java | 2 +- .../sqlite3/gen/SQLite3TransactionGenerator.java | 10 ++++++---- .../gen/SQLite3VirtualFTSTableCommandGenerator.java | 1 + .../sqlite3/gen/dml/SQLite3InsertGenerator.java | 2 +- .../sqlite3/gen/dml/SQLite3UpdateGenerator.java | 1 + src/sqlancer/sqlite3/schema/SQLite3Schema.java | 4 ++++ 7 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java index 35133a6c4..2933214b1 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java @@ -1,5 +1,7 @@ package sqlancer.sqlite3.gen; +import java.util.Arrays; + import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -35,7 +37,7 @@ public static Query generateAnalyze(SQLite3GlobalState globalState) { throw new AssertionError(); } } - return new QueryAdapter(sb.toString()); + return new QueryAdapter(sb.toString(), Arrays.asList("The database file is locked")); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java index fa2c5d019..1c0a73115 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java @@ -242,7 +242,7 @@ public QueryAdapter insert(SQLite3GlobalState globalState) { } sb.append(";"); String pragmaString = sb.toString(); - // errors.add("cannot change"); + errors.add("The database file is locked"); return new QueryAdapter(pragmaString, errors); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java index e7ee11d1d..78787b31d 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java @@ -18,8 +18,8 @@ public static Query generateCommit(SQLite3GlobalState globalState) { if (Randomly.getBoolean()) { sb.append(" TRANSACTION"); } - return new QueryAdapter(sb.toString(), - Arrays.asList("no transaction is active", "FOREIGN KEY constraint failed"), true); + return new QueryAdapter(sb.toString(), Arrays.asList("no transaction is active", "The database file is locked", + "FOREIGN KEY constraint failed"), true); } public static Query generateBeginTransaction(SQLite3GlobalState globalState) { @@ -29,12 +29,14 @@ public static Query generateBeginTransaction(SQLite3GlobalState globalState) { sb.append(Randomly.fromOptions("DEFERRED", "IMMEDIATE", "EXCLUSIVE")); } sb.append(" TRANSACTION;"); - return new QueryAdapter(sb.toString(), Arrays.asList("cannot start a transaction within a transaction")); + return new QueryAdapter(sb.toString(), + Arrays.asList("cannot start a transaction within a transaction", "The database file is locked")); } public static Query generateRollbackTransaction(SQLite3GlobalState globalState) { // TODO: could be extended by savepoint - return new QueryAdapter("ROLLBACK TRANSACTION;", Arrays.asList("no transaction is active"), true); + return new QueryAdapter("ROLLBACK TRANSACTION;", + Arrays.asList("no transaction is active", "The database file is locked"), true); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java index b20f9ca04..5c5cf8817 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java @@ -93,6 +93,7 @@ private Query generate() { default: throw new AssertionError(); } + errors.add("The database file is locked"); return new QueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java index af6245fa2..38e2ca594 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java @@ -58,7 +58,7 @@ private String insertRow(SQLite3Table table) { errors.add("A table in the database is locked"); errors.add("cannot INSERT into generated column"); // TODO: filter out generated columns - + errors.add("The database file is locked"); errors.add("load_extension() prohibited in triggers and views"); SQLite3Errors.addInsertNowErrors(errors); SQLite3Errors.addExpectedExpressionErrors(errors); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 76e1b4a36..8661ee99f 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -113,6 +113,7 @@ private Query update(SQLite3Table table) { // TODO not update generated columns? errors.add("cannot INSERT into generated column"); errors.add("A table in the database is locked"); // https://www.sqlite.org/src/tktview?name=56a74875be + errors.add("The database file is locked"); SQLite3Errors.addInsertNowErrors(errors); SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addDeleteErrors(errors); diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 509c6f635..3ab0b917f 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -435,6 +435,10 @@ public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throw } indexNames.add(name); } + } catch (SQLException e) { + if (!e.getMessage().contains("The database file is locked")) { + throw new AssertionError(e); + } } } From d7b91551d9d8ee7f556349a15313b1c3a6efdee6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 5 Aug 2020 16:39:09 +0200 Subject: [PATCH 0068/1284] [SQLite3] Add an additional expected error --- src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 42982a1ce..d686e7fbc 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -40,7 +40,7 @@ public static Query deleteContent(SQLite3GlobalState globalState, SQLite3Table t "[SQLITE_ERROR] SQL error or missing database (no such table:", "no such column", "too many levels of trigger recursion", "cannot UPDATE generated column", "cannot INSERT into generated column", "A table in the database is locked", - "load_extension() prohibited in triggers and views")); + "load_extension() prohibited in triggers and views", "The database file is locked")); SQLite3Errors.addDeleteErrors(errors); return new QueryAdapter(sb.toString(), errors, true); } From 650e3e19720dc620e831654b41273b0a48648ff5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 6 Aug 2020 15:00:48 +0200 Subject: [PATCH 0069/1284] [MariaDB] Select another mirror --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 76418fdb7..d04ade287 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: jdk : oraclejdk11 before_install: - sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 - - sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://ftp.utexas.edu/mariadb/repo/10.3/ubuntu bionic main' + - sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main' - sudo apt update - sudo apt install mariadb-server - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" From 5cd03c436934e878434d167e6ed6792194555bb8 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 6 Aug 2020 18:21:06 +0300 Subject: [PATCH 0070/1284] prepared statement execution cherry-pick --- src/sqlancer/QueryAdapter.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index 3db9d32ca..e7c72a47f 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -63,13 +63,17 @@ public boolean execute(GlobalState globalState, String... fills) throws SQ if (fills.length > 0) { s = globalState.getConnection().prepareStatement(getQueryString()); for (int i = 0; i < fills.length; i++) { - ((PreparedStatement) s).setString(i, fills[i]); + ((PreparedStatement) s).setString(i + 1, fills[i]); } } else { s = globalState.getConnection().createStatement(); } try { - s.execute(query); + if (fills.length > 0) { + ((PreparedStatement) s).execute(); + } else { + s.execute(query); + } Main.nrSuccessfulActions.addAndGet(1); return true; } catch (Exception e) { @@ -98,14 +102,18 @@ public ResultSet executeAndGet(GlobalState globalState, String... fills) t if (fills.length > 0) { s = globalState.getConnection().prepareStatement(getQueryString()); for (int i = 0; i < fills.length; i++) { - ((PreparedStatement) s).setString(i, fills[i]); + ((PreparedStatement) s).setString(i + 1, fills[i]); } } else { s = globalState.getConnection().createStatement(); } ResultSet result = null; try { - result = s.executeQuery(query); + if (fills.length > 0) { + result = ((PreparedStatement) s).executeQuery(); + } else { + result = s.executeQuery(query); + } Main.nrSuccessfulActions.addAndGet(1); return result; } catch (Exception e) { From e5ccd4af3b94e84eb521ecaa13634ebe703f0b30 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 6 Aug 2020 17:53:23 +0200 Subject: [PATCH 0071/1284] Introduce a wrapped result set This allows registering a callback to correctly log the time for queries that iterate through a result set. --- src/sqlancer/ComparatorHelper.java | 5 +- src/sqlancer/GlobalState.java | 15 ++++-- src/sqlancer/Main.java | 5 +- src/sqlancer/Query.java | 5 +- src/sqlancer/QueryAdapter.java | 7 ++- src/sqlancer/SQLancerResultSet.java | 53 +++++++++++++++++++ .../tlp/ClickHouseTLPAggregateOracle.java | 6 +-- .../oracle/CockroachDBNoRECOracle.java | 4 +- .../tlp/CockroachDBTLPAggregateOracle.java | 4 +- .../duckdb/test/DuckDBNoRECOracle.java | 3 +- ...uckDBQueryPartitioningAggregateTester.java | 4 +- .../mariadb/oracle/MariaDBNoRECOracle.java | 9 ++-- .../postgres/oracle/PostgresNoRECOracle.java | 3 +- .../tlp/PostgresTLPAggregateOracle.java | 4 +- .../gen/ddl/SQLite3IndexGenerator.java | 2 + .../sqlite3/oracle/SQLite3NoRECOracle.java | 8 ++- .../oracle/tlp/SQLite3TLPAggregateOracle.java | 6 +-- .../sqlite3/schema/SQLite3Schema.java | 7 ++- 18 files changed, 103 insertions(+), 47 deletions(-) create mode 100644 src/sqlancer/SQLancerResultSet.java diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 07986bd54..d65713e23 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -1,7 +1,6 @@ package sqlancer; import java.io.IOException; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashSet; @@ -46,7 +45,7 @@ public static List getResultSetFirstColumnAsString(String queryString, S } QueryAdapter q = new QueryAdapter(queryString, errors); List resultSet = new ArrayList<>(); - ResultSet result = null; + SQLancerResultSet result = null; try { result = q.executeAndGet(state); if (result == null) { @@ -55,7 +54,6 @@ public static List getResultSetFirstColumnAsString(String queryString, S while (result.next()) { resultSet.add(result.getString(1)); } - result.getStatement().close(); } catch (Exception e) { if (e instanceof IgnoreMeException) { throw e; @@ -75,7 +73,6 @@ public static List getResultSetFirstColumnAsString(String queryString, S throw new AssertionError(queryString, e); } finally { if (result != null && !result.isClosed()) { - result.getStatement().close(); result.close(); } } diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 4e123616c..f2e4e99ad 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,7 +1,6 @@ package sqlancer; import java.sql.Connection; -import java.sql.ResultSet; import java.sql.SQLException; import sqlancer.Main.QueryManager; @@ -129,11 +128,19 @@ public boolean executeStatement(Query q, String... fills) throws SQLException { return success; } - public ResultSet executeStatementAndGet(Query q, String... fills) throws SQLException { + public SQLancerResultSet executeStatementAndGet(Query q, String... fills) throws SQLException { ExecutionTimer timer = executePrologue(q); - ResultSet result = manager.executeAndGet(q, fills); + SQLancerResultSet result = manager.executeAndGet(q, fills); boolean success = result != null; - executeEpilogue(q, success, timer); + if (success) { + result.registerEpilogue(() -> { + try { + executeEpilogue(q, success, timer); + } catch (SQLException e) { + throw new AssertionError(e); + } + }); + } return result; } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index c14ec75d1..23ad2e774 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -8,7 +8,6 @@ import java.io.Writer; import java.nio.file.Files; import java.sql.Connection; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.text.DateFormat; @@ -260,9 +259,9 @@ public boolean execute(Query q, String... fills) throws SQLException { return success; } - public ResultSet executeAndGet(Query q, String... fills) throws SQLException { + public SQLancerResultSet executeAndGet(Query q, String... fills) throws SQLException { globalState.getState().logStatement(q); - ResultSet result; + SQLancerResultSet result; result = q.executeAndGet(globalState, fills); Main.nrSuccessfulActions.addAndGet(1); return result; diff --git a/src/sqlancer/Query.java b/src/sqlancer/Query.java index c4f8aff55..fbac75548 100644 --- a/src/sqlancer/Query.java +++ b/src/sqlancer/Query.java @@ -1,6 +1,5 @@ package sqlancer; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; @@ -32,7 +31,7 @@ public String toString() { return getQueryString(); } - public ResultSet executeAndGet(GlobalState globalState, String... fills) throws SQLException { + public SQLancerResultSet executeAndGet(GlobalState globalState, String... fills) throws SQLException { throw new AssertionError(); } @@ -41,7 +40,7 @@ public boolean executeLogged(GlobalState globalState) throws SQLException return execute(globalState); } - public ResultSet executeAndGetLogged(GlobalState globalState) throws SQLException { + public SQLancerResultSet executeAndGetLogged(GlobalState globalState) throws SQLException { logQueryString(globalState); return executeAndGet(globalState); } diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index e7c72a47f..83d270228 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -97,7 +97,7 @@ public void checkException(Exception e) throws AssertionError { } @Override - public ResultSet executeAndGet(GlobalState globalState, String... fills) throws SQLException { + public SQLancerResultSet executeAndGet(GlobalState globalState, String... fills) throws SQLException { Statement s; if (fills.length > 0) { s = globalState.getConnection().prepareStatement(getQueryString()); @@ -115,7 +115,10 @@ public ResultSet executeAndGet(GlobalState globalState, String... fills) t result = s.executeQuery(query); } Main.nrSuccessfulActions.addAndGet(1); - return result; + if (result == null) { + return null; + } + return new SQLancerResultSet(result); } catch (Exception e) { s.close(); Main.nrUnsuccessfulActions.addAndGet(1); diff --git a/src/sqlancer/SQLancerResultSet.java b/src/sqlancer/SQLancerResultSet.java new file mode 100644 index 000000000..896b4f7ce --- /dev/null +++ b/src/sqlancer/SQLancerResultSet.java @@ -0,0 +1,53 @@ +package sqlancer; + +import java.io.Closeable; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class SQLancerResultSet implements Closeable { + + ResultSet rs; + private Runnable runnableEpilogue; + + public SQLancerResultSet(ResultSet rs) { + this.rs = rs; + } + + @Override + public void close() { + try { + if (runnableEpilogue != null) { + runnableEpilogue.run(); + } + rs.getStatement().close(); + rs.close(); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + public boolean next() throws SQLException { + return rs.next(); + } + + public int getInt(int i) throws SQLException { + return rs.getInt(i); + } + + public String getString(int i) throws SQLException { + return rs.getString(i); + } + + public boolean isClosed() throws SQLException { + return rs.isClosed(); + } + + public long getLong(int i) throws SQLException { + return rs.getLong(i); + } + + public void registerEpilogue(Runnable runnableEpilogue) { + this.runnableEpilogue = runnableEpilogue; + } + +} diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 2526d59ef..0fef32786 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -1,6 +1,5 @@ package sqlancer.clickhouse.oracle.tlp; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -10,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.SQLancerResultSet; import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseSchema; @@ -69,7 +69,7 @@ public void check() throws SQLException { String firstResult; String secondResult; QueryAdapter q = new QueryAdapter(originalQuery); - try (ResultSet result = q.executeAndGet(state)) { + try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } @@ -80,7 +80,7 @@ public void check() throws SQLException { } QueryAdapter q2 = new QueryAdapter(metamorphicText); - try (ResultSet result = q2.executeAndGet(state)) { + try (SQLancerResultSet result = q2.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index adac39470..14f8e814e 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -1,6 +1,5 @@ package sqlancer.cockroachdb.oracle; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -14,6 +13,7 @@ import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.SQLancerResultSet; import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; @@ -137,7 +137,7 @@ private int getNonOptimizedResult(CockroachDBExpression whereCondition, List globalState, Query q) throws AssertionError { int count = 0; - try (ResultSet rs = q.executeAndGet(globalState)) { + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs == null) { return -1; } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index e4f86b53b..45bb06930 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -1,6 +1,5 @@ package sqlancer.cockroachdb.oracle.tlp; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -15,6 +14,7 @@ import sqlancer.IgnoreMeException; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.SQLancerResultSet; import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; @@ -118,7 +118,7 @@ private String createMetamorphicUnionQuery(CockroachDBSelect select, CockroachDB private String getAggregateResult(String queryString) throws SQLException { String resultString; QueryAdapter q = new QueryAdapter(queryString, errors); - try (ResultSet result = q.executeAndGet(state)) { + try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 148551b97..f4fd87336 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -13,6 +13,7 @@ import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.SQLancerResultSet; import sqlancer.TestOracle; import sqlancer.ast.newast.ColumnReferenceNode; import sqlancer.ast.newast.NewPostfixTextNode; @@ -83,7 +84,7 @@ private int getSecondQuery(List> tableList, Node fromTables, Postgr } errors.add("canceling statement due to statement timeout"); Query q = new QueryAdapter(unoptimizedQueryString, errors); - ResultSet rs; + SQLancerResultSet rs; try { rs = q.executeAndGet(state); } catch (Exception e) { diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index fbf2b9897..f7a350284 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -1,7 +1,6 @@ package sqlancer.postgres.oracle.tlp; import java.io.IOException; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -13,6 +12,7 @@ import sqlancer.IgnoreMeException; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.SQLancerResultSet; import sqlancer.TestOracle; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -112,7 +112,7 @@ private String getAggregateResult(String queryString) throws SQLException { } String resultString; QueryAdapter q = new QueryAdapter(queryString, errors); - try (ResultSet result = q.executeAndGet(state)) { + try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 1f96b00da..b05525265 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -71,6 +71,8 @@ private String createIndex(SQLite3Table t, List columns) { sb.append(" INDEX"); if (Randomly.getBoolean()) { sb.append(" IF NOT EXISTS"); + } else { + errors.add("already exists"); } sb.append(" "); sb.append(SQLite3Common.getFreeIndexName(globalState.getSchema())); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 6752a70ea..1b0e06602 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -1,6 +1,5 @@ package sqlancer.sqlite3.oracle; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; import java.util.Collections; @@ -10,6 +9,7 @@ import sqlancer.NoRECBase; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.SQLancerResultSet; import sqlancer.TestOracle; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -111,7 +111,7 @@ private int getOptimizedQuery(SQLite3Select select, SQLite3Expression randomWher private int countRows(QueryAdapter q) { int count = 0; - try (ResultSet rs = q.executeAndGet(state)) { + try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { return NO_VALID_RESULT; } else { @@ -122,7 +122,6 @@ private int countRows(QueryAdapter q) { } catch (SQLException e) { count = NO_VALID_RESULT; } - rs.getStatement().close(); } } catch (Exception e) { if (e instanceof IgnoreMeException) { @@ -135,7 +134,7 @@ private int countRows(QueryAdapter q) { private int extractCounts(QueryAdapter q) { int count = 0; - try (ResultSet rs = q.executeAndGet(state)) { + try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { return NO_VALID_RESULT; } else { @@ -146,7 +145,6 @@ private int extractCounts(QueryAdapter q) { } catch (SQLException e) { count = NO_VALID_RESULT; } - rs.getStatement().close(); } } catch (Exception e) { if (e instanceof IgnoreMeException) { diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index be6eeb039..df61d4d3c 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -1,6 +1,5 @@ package sqlancer.sqlite3.oracle.tlp; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -10,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.SQLancerResultSet; import sqlancer.TestOracle; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -75,7 +75,7 @@ public void check() throws SQLException { String firstResult; String secondResult; QueryAdapter q = new QueryAdapter(originalQuery, errors); - try (ResultSet result = q.executeAndGet(state)) { + try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } @@ -86,7 +86,7 @@ public void check() throws SQLException { } QueryAdapter q2 = new QueryAdapter(metamorphicText, errors); - try (ResultSet result = q2.executeAndGet(state)) { + try (SQLancerResultSet result = q2.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 3ab0b917f..32bce6522 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -17,6 +17,7 @@ import sqlancer.IgnoreMeException; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.SQLancerResultSet; import sqlancer.StateToReproduce.SQLite3StateToReproduce; import sqlancer.schema.AbstractTable; import sqlancer.schema.AbstractTableColumn; @@ -354,14 +355,12 @@ public static int getNrRows(SQLite3GlobalState globalState, String table) throws "misuse of window function")); SQLite3Errors.addExpectedExpressionErrors(errors); QueryAdapter q = new QueryAdapter(string, errors); - try (ResultSet query = q.executeAndGet(globalState)) { + try (SQLancerResultSet query = q.executeAndGet(globalState)) { if (query == null) { throw new IgnoreMeException(); } query.next(); - int int1 = query.getInt(1); - query.getStatement().close(); - return int1; + return query.getInt(1); } } From 3194abd94c054fba9f7dac0adb642d4e20a95e24 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 10 Aug 2020 12:50:18 +0300 Subject: [PATCH 0072/1284] Separate template from query to log filled-in statements --- src/sqlancer/QueryAdapter.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index 83d270228..e0735463b 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -61,9 +61,9 @@ public String getQueryString() { public boolean execute(GlobalState globalState, String... fills) throws SQLException { Statement s; if (fills.length > 0) { - s = globalState.getConnection().prepareStatement(getQueryString()); - for (int i = 0; i < fills.length; i++) { - ((PreparedStatement) s).setString(i + 1, fills[i]); + s = globalState.getConnection().prepareStatement(fills[0]); + for (int i = 1; i < fills.length; i++) { + ((PreparedStatement) s).setString(i, fills[i]); } } else { s = globalState.getConnection().createStatement(); @@ -100,9 +100,9 @@ public void checkException(Exception e) throws AssertionError { public SQLancerResultSet executeAndGet(GlobalState globalState, String... fills) throws SQLException { Statement s; if (fills.length > 0) { - s = globalState.getConnection().prepareStatement(getQueryString()); - for (int i = 0; i < fills.length; i++) { - ((PreparedStatement) s).setString(i + 1, fills[i]); + s = globalState.getConnection().prepareStatement(fills[0]); + for (int i = 1; i < fills.length; i++) { + ((PreparedStatement) s).setString(i, fills[i]); } } else { s = globalState.getConnection().createStatement(); From 6055cc5551d0b00befa7aef9543c28ec63771e62 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 10 Aug 2020 12:57:15 +0300 Subject: [PATCH 0073/1284] Make reproducible states deterministic by reading from database in order --- src/sqlancer/postgres/PostgresSchema.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 2a76d939f..6e1dfcd28 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -264,7 +264,7 @@ public static PostgresSchema fromConnection(Connection con, String databaseName) List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery( - "SELECT table_name, table_schema, table_type, is_insertable_into FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { + "SELECT table_name, table_schema, table_type, is_insertable_into FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { while (rs.next()) { String tableName = rs.getString("table_name"); String tableTypeSchema = rs.getString("table_schema"); @@ -297,7 +297,7 @@ public static PostgresSchema fromConnection(Connection con, String databaseName) private static List getStatistics(Connection con) throws SQLException { List statistics = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery("SELECT stxname FROM pg_statistic_ext;")) { + try (ResultSet rs = s.executeQuery("SELECT stxname FROM pg_statistic_ext ORDER BY stxname;")) { while (rs.next()) { statistics.add(new PostgresStatisticsObject(rs.getString("stxname"))); } @@ -321,8 +321,8 @@ private static PostgresTable.TableType getTableType(String tableTypeStr) throws private static List getIndexes(Connection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s - .executeQuery(String.format("SELECT indexname FROM pg_indexes WHERE tablename='%s';", tableName))) { + try (ResultSet rs = s.executeQuery(String + .format("SELECT indexname FROM pg_indexes WHERE tablename='%s' ORDER BY indexname;", tableName))) { while (rs.next()) { String indexName = rs.getString("indexname"); if (indexName.length() != 2) { @@ -341,7 +341,7 @@ private static List getTableColumns(Connection con, String table try (Statement s = con.createStatement()) { try (ResultSet rs = s .executeQuery("select column_name, data_type from INFORMATION_SCHEMA.COLUMNS where table_name = '" - + tableName + "'")) { + + tableName + "' ORDER BY column_name")) { while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); From 40da7d0ac62ff06eaf341342f55a3428983cd87d Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Fri, 24 Jul 2020 21:43:22 +0300 Subject: [PATCH 0074/1284] Create Citus classes --- src/sqlancer/citus/CitusGlobalState.java | 26 ++++ src/sqlancer/citus/CitusOptions.java | 12 ++ src/sqlancer/citus/CitusProvider.java | 147 ++++++++++++++++++ src/sqlancer/citus/CitusSchema.java | 95 +++++++++++ src/sqlancer/citus/gen/CitusCommon.java | 58 +++++++ src/sqlancer/postgres/PostgresProvider.java | 49 +++--- src/sqlancer/postgres/gen/PostgresCommon.java | 2 +- 7 files changed, 367 insertions(+), 22 deletions(-) create mode 100644 src/sqlancer/citus/CitusGlobalState.java create mode 100644 src/sqlancer/citus/CitusOptions.java create mode 100644 src/sqlancer/citus/CitusProvider.java create mode 100644 src/sqlancer/citus/CitusSchema.java create mode 100644 src/sqlancer/citus/gen/CitusCommon.java diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java new file mode 100644 index 000000000..a1d1f2a57 --- /dev/null +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -0,0 +1,26 @@ +package sqlancer.citus; + +import java.sql.SQLException; + +import sqlancer.postgres.PostgresGlobalState; + +public class CitusGlobalState extends PostgresGlobalState { + + private boolean repartition; + + public void setRepartition(boolean repartition) { + this.repartition = repartition; + } + + public boolean getRepartition() { + return repartition; + } + + @Override + protected void updateSchema() throws SQLException { + setSchema(CitusSchema.fromConnection(getConnection(), getDatabaseName())); + } + +} + + \ No newline at end of file diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java new file mode 100644 index 000000000..bc9e2b8f7 --- /dev/null +++ b/src/sqlancer/citus/CitusOptions.java @@ -0,0 +1,12 @@ +package sqlancer.citus; + +import com.beust.jcommander.Parameter; + +import sqlancer.postgres.PostgresOptions; + +public class CitusOptions extends PostgresOptions { + + @Parameter(names= "--repartition") + public boolean repartition = true; + +} \ No newline at end of file diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java new file mode 100644 index 000000000..a878a9ce4 --- /dev/null +++ b/src/sqlancer/citus/CitusProvider.java @@ -0,0 +1,147 @@ +package sqlancer.citus; + +import java.sql.SQLException; +import java.util.HashSet; +import java.util.Set; + +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresProvider; + +public class CitusProvider extends PostgresProvider { + + protected final Set errors = new HashSet<>(); + + public CitusProvider() { + super(); + CitusCommon.addCitusErrors(errors); + } + + // FIXME: static or not? + private class WorkerNode{ + + private final String name; + private final int port; + + public WorkerNode(String node_name, int node_port) { + this.name = node_name; + this.port = node_port; + } + + public String get_name() { + return this.name; + } + + public int get_port() { + return this.port; + } + + } + + // FIXME: static or not? + private final void distributeTable(List columns, String tableName, PostgresGlobalState globalState, Connection con) throws SQLException { + if (columns.size() != 0) { + PostgresColumn columnToDistribute = Randomly.fromList(columns); + QueryAdapter query = new QueryAdapter("SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + "');", errors); + String template = "SELECT create_distributed_table(?, ?);"; + List fills = Arrays.asList(tableName, columnToDistribute.getName()); + globalState.fillAndExecuteStatement(query, template, fills); + // distribution column cannot take NULL value + // TODO: find a way to protect from SQL injection without '' around string input + query = new QueryAdapter("ALTER TABLE " + tableName + " ALTER COLUMN " + columnToDistribute.getName() + " SET NOT NULL;", errors); + globalState.executeStatement(query); + } + } + + private final List getTableConstraints(String tableName, PostgresGlobalState globalState, Connection con) throws SQLException { + List constraints = new ArrayList<>(); + QueryAdapter query = new QueryAdapter("SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + tableName + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"); + String template = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; + List fills = new ArrayList<>(); + fills.add(tableName); + ResultSet rs = query.fillAndExecuteAndGet(globalState, template, fills); + while (rs.next()) { + constraints.add(rs.getString("constraint_type")); + } + return constraints; + } + + // FIXME: static or not? + private final void createDistributedTable(String tableName, PostgresGlobalState globalState, Connection con) throws SQLException { + List columns = new ArrayList<>(); + List tableConstraints = getTableConstraints(tableName, globalState, con); + if (tableConstraints.size() == 0) { + QueryAdapter query = new QueryAdapter("SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + tableName + "';"); + String template = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;"; + List fills = Arrays.asList(tableName); + ResultSet rs = query.fillAndExecuteAndGet(globalState, template, fills); + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + // data types money & bit varying have no default operator class for specified partition method + if (! (dataType.equals("money") || dataType.equals("bit varying"))) { + PostgresColumn c = new PostgresColumn(columnName, getColumnType(dataType)); + columns.add(c); + } + } + } else { + // TODO: multiple constraints? + HashMap> columnConstraints = new HashMap<>(); + QueryAdapter query = new QueryAdapter("SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + tableName + "';"); + // TODO: decide whether to log + // globalState.getState().statements.add(query); + String template = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;"; + List fills = Arrays.asList(tableName); + ResultSet rs = query.fillAndExecuteAndGet(globalState, template, fills); + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + String constraintType = rs.getString("constraint_type"); + // data types money & bit varying have no default operator class for specified partition method + if (! (dataType.equals("money") || dataType.equals("bit varying"))) { + PostgresColumn c = new PostgresColumn(columnName, getColumnType(dataType)); + if (columnConstraints.containsKey(c)) { + columnConstraints.get(c).add(constraintType); + } else { + columnConstraints.put(c, new ArrayList<>(Arrays.asList(constraintType))); + } + } + } + for (PostgresColumn c : columnConstraints.keySet()) { + // TODO: check if table and column constraint sets are equal? but then it's O(N) instead of O(1) + if (tableConstraints.size() == columnConstraints.get(c).size()) { + columns.add(c); + } + } + // TODO: figure out how to use EXCLUDE + } + distributeTable(columns, tableName, globalState, con); + } + + @Override + public void generateDatabase(PostgresGlobalState globalState) throws SQLException { + // TODO: function reading? add to Postgres implementation? + createTables(globalState); + for (PostgresTable table : globalState.getSchema().getDatabaseTables()) { + // TODO: random 0-1 range double + if (Randomly.getBooleanWithRatherLowProbability()) { + // create local table + } else if (Randomly.getBooleanWithRatherLowProbability()) { + // create reference table + query = new QueryAdapter("SELECT create_reference_table('" + table.getName() + "');", errors); + String template = "SELECT create_reference_table(?);"; + List fills = Arrays.asList(table.getName()); + globalState.fillAndExecuteStatement(query, template, fills); + } else { + // create distributed table + createDistributedTable(table.getName(), globalState, globalState.getConnection()); + } + } + globalState.updateSchema(); + prepareTables(globalState); + if (globalState.getRepartition()) { + // allow repartition joins + globalState.executeStatement(new QueryAdapter("SET citus.enable_repartition_joins to ON;\n", errors)); + } + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java new file mode 100644 index 000000000..691ff5c16 --- /dev/null +++ b/src/sqlancer/citus/CitusSchema.java @@ -0,0 +1,95 @@ +package sqlancer.citus; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.postgres.PostgresSchema; + +public class CitusSchema extends PostgresSchema { + + public CitusSchema(List databaseTables, String databaseName) { + super(databaseTables, databaseName); + } + + public static class CitusTable extends PostgresTable { + + private PostgresColumn distributionColumn = null; + private Integer colocationId = null; + + public CitusTable(String tableName, List columns, List indexes, + TableType tableType, List statistics, boolean isView, boolean isInsertable) { + super(tableName, columns, indexes, tableType, statistics, isView, isInsertable); + } + + public void setDistributionColumn(PostgresColumn distributionColumn) { + this.distributionColumn = distributionColumn; + } + + public void setColocationId(Integer colocationId) { + this.colocationId = colocationId; + } + + public PostgresColumn getDistributionColumn() { + return this.distributionColumn; + } + + public Integer getColocationId() { + return this.colocationId; + } + + } + + // TODO: Override? + public static CitusSchema fromConnection(Connection con, String databaseName) throws SQLException { + Exception ex = null; + try { + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + "SELECT table_name, table_schema, table_type, is_insertable_into, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + String tableTypeSchema = rs.getString("table_schema"); + boolean isInsertable = rs.getBoolean("is_insertable_into"); + String distributionColumnName = rs.getString("dist_col_name"); + Integer colocationId = rs.getInt("colocationid"); + if (rs.wasNull()) { + colocationId = null; + } + // TODO: also check insertable + // TODO: insert into view? + boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || + // tableTypeStr.contains("LOCAL TEMPORARY") && + // !isInsertable; + PostgresTable.TableType tableType = getTableType(tableTypeSchema); + List databaseColumns = getTableColumns(con, tableName); + List indexes = getIndexes(con, tableName); + List statistics = getStatistics(con); + PostgresTable t = new PostgresTable(tableName, databaseColumns, indexes, tableType, statistics, + isView, isInsertable); + if (distributionColumnName != null && !distributionColumnName.equals("")) { + PostgresColumn distributionColumn = databaseColumns.stream().filter(c -> c.getName().equals(distributionColumnName)).collect(Collectors.toList()).get(0); + t.setDistributionColumn(distributionColumn); + } + if (colocationId != null) { + t.setColocationId(colocationId); + } + for (PostgresColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + return new CitusSchema(databaseTables, databaseName); + } catch (SQLIntegrityConstraintViolationException e) { + ex = e; + } + throw new AssertionError(ex); + } +} \ No newline at end of file diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java new file mode 100644 index 000000000..5317af682 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -0,0 +1,58 @@ +package sqlancer.citus.gen; + +import java.util.Set; + +import sqlancer.postgres.gen.PostgresCommon; + +public class CitusCommon extends PostgresCommon { + + private CitusCommon() { + } + + public static void addCitusErrors(Set errors) { + errors.add("recursive CTEs are not supported in distributed queries"); + errors.add("could not run distributed query with GROUPING SETS, CUBE, or ROLLUP"); + errors.add("Subqueries in HAVING cannot refer to outer query"); + errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); + errors.add("functions used in UPDATE queries on distributed tables must not be VOLATILE"); + errors.add("STABLE functions used in UPDATE queries cannot be called with column references"); + // should be fixed now + // errors.add("failed to evaluate partition key in insert"); + errors.add("functions used in the WHERE clause of modification queries on distributed tables must not be VOLATILE"); + errors.add("cannot perform an INSERT with NULL in the partition column"); + errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); + errors.add("cannot execute ALTER TABLE command involving partition column"); + errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); + // TODO: remove once fixed + // errors.add("unrecognized node type: 127"); + // TODO: SQLancer error + errors.add("not a foreign key or check constraint"); + errors.add("cannot perform an INSERT without a partition column value"); + // TODO: remove once fixed + errors.add("failed to find conversion function from unknown to text"); + // ERROR: cannot create foreign key constraint + // Detail: SET NULL or SET DEFAULT is not supported in ON DELETE operation when distribution key is included in the foreign key constraint + errors.add("cannot create foreign key constraint"); + // ERROR: cannot pushdown the subquery + // Detail: Complex subqueries and CTEs cannot be in the outer part of the outer join + errors.add("cannot pushdown the subquery"); + // TODO: remove after PostgreSQL 13 upgrade + errors.add("unrecognized configuration parameter \"enable_hashagg_disk\""); + // TODO: remove after PostgreSQL 13 upgrade + errors.add("unrecognized configuration parameter \"enable_groupingsets_hash_disk\""); + errors.add("is not a regular, foreign or partitioned table"); + errors.add("must be a distributed table or a reference table"); + // Citus restrictions on SELECT queries + errors.add("complex joins are only supported when all distributed tables are co-located and joined on their distribution columns"); + errors.add("complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator"); + errors.add("cannot perform distributed planning on this query"); + // errors.add("the query contains a join that requires repartitioning"); + // SQLancer error + errors.add("non-integer constant in GROUP BY"); + // SQLancer error + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + // SQLancer error + errors.add("GROUP BY position"); + } + +} \ No newline at end of file diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 64dfde48a..23e79956c 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -52,7 +52,7 @@ // EXISTS // IN -public final class PostgresProvider extends ProviderAdapter { +public class PostgresProvider extends ProviderAdapter { public static boolean generateOnlyKnown; @@ -184,26 +184,8 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { @Override public void generateDatabase(PostgresGlobalState globalState) throws SQLException { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { - try { - String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), - generateOnlyKnown, globalState); - globalState.executeStatement(createTable); - } catch (IgnoreMeException e) { - - } - } - - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - PostgresProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - globalState.executeStatement(new QueryAdapter("COMMIT", true)); - globalState.executeStatement(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); + createTables(globalState); + prepareTables(globalState); } @Override @@ -276,6 +258,31 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce return con; } + private void createTables(PostgresGlobalState globalState) throws SQLException { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { + try { + String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); + Query createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), + generateOnlyKnown, globalState); + globalState.executeStatement(createTable); + } catch (IgnoreMeException e) { + + } + } + } + + private void prepareTables(PostgresGlobalState globalState) throws SQLException { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + PostgresProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new QueryAdapter("COMMIT", true)); + globalState.executeStatement(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); + } + private String getCreateDatabaseCommand(String databaseName, Connection con, PostgresGlobalState state) { StringBuilder sb = new StringBuilder(); sb.append("CREATE DATABASE " + databaseName + " "); diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index f45180467..7d7940dfe 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -17,7 +17,7 @@ import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; -public final class PostgresCommon { +public class PostgresCommon { private PostgresCommon() { } From 5a71fef6135cbf3b9c60700f20850b8b578e1894 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 27 Jul 2020 11:38:56 +0300 Subject: [PATCH 0075/1284] Citus integration into database creation & preparation --- src/sqlancer/citus/CitusGlobalState.java | 1 + src/sqlancer/citus/CitusProvider.java | 143 +++++++++++++++--- src/sqlancer/citus/CitusSchema.java | 12 +- src/sqlancer/citus/gen/CitusCommon.java | 44 +++--- src/sqlancer/postgres/PostgresProvider.java | 6 +- src/sqlancer/postgres/PostgresSchema.java | 10 +- src/sqlancer/postgres/gen/PostgresCommon.java | 2 +- 7 files changed, 162 insertions(+), 56 deletions(-) diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java index a1d1f2a57..52b519dc8 100644 --- a/src/sqlancer/citus/CitusGlobalState.java +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -18,6 +18,7 @@ public boolean getRepartition() { @Override protected void updateSchema() throws SQLException { + // FIXME: Will casting lose CitusSchema information? setSchema(CitusSchema.fromConnection(getConnection(), getDatabaseName())); } diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index a878a9ce4..ce38736bb 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -1,34 +1,50 @@ package sqlancer.citus; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Set; +import sqlancer.QueryAdapter; +import sqlancer.Randomly; +import sqlancer.citus.CitusSchema.CitusTable; import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTable; public class CitusProvider extends PostgresProvider { - + // FIXME: protected final Set errors = new HashSet<>(); public CitusProvider() { + // how to change the extension of the super class? super(); CitusCommon.addCitusErrors(errors); } - // FIXME: static or not? private class WorkerNode{ - private final String name; + private final String host; private final int port; - public WorkerNode(String node_name, int node_port) { - this.name = node_name; + public WorkerNode(String node_host, int node_port) { + this.host = node_host; this.port = node_port; } - public String get_name() { - return this.name; + public String get_host() { + return this.host; } public int get_port() { @@ -37,8 +53,7 @@ public int get_port() { } - // FIXME: static or not? - private final void distributeTable(List columns, String tableName, PostgresGlobalState globalState, Connection con) throws SQLException { + private static void distributeTable(List columns, String tableName, CitusGlobalState globalState, Connection con) throws SQLException { if (columns.size() != 0) { PostgresColumn columnToDistribute = Randomly.fromList(columns); QueryAdapter query = new QueryAdapter("SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + "');", errors); @@ -52,7 +67,7 @@ private final void distributeTable(List columns, String tableNam } } - private final List getTableConstraints(String tableName, PostgresGlobalState globalState, Connection con) throws SQLException { + private static List getTableConstraints(String tableName, CitusGlobalState globalState, Connection con) throws SQLException { List constraints = new ArrayList<>(); QueryAdapter query = new QueryAdapter("SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + tableName + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"); String template = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; @@ -65,8 +80,7 @@ private final List getTableConstraints(String tableName, PostgresGlobalS return constraints; } - // FIXME: static or not? - private final void createDistributedTable(String tableName, PostgresGlobalState globalState, Connection con) throws SQLException { + private static void createDistributedTable(String tableName, CitusGlobalState globalState, Connection con) throws SQLException { List columns = new ArrayList<>(); List tableConstraints = getTableConstraints(tableName, globalState, con); if (tableConstraints.size() == 0) { @@ -79,7 +93,7 @@ private final void createDistributedTable(String tableName, PostgresGlobalState String dataType = rs.getString("data_type"); // data types money & bit varying have no default operator class for specified partition method if (! (dataType.equals("money") || dataType.equals("bit varying"))) { - PostgresColumn c = new PostgresColumn(columnName, getColumnType(dataType)); + PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); columns.add(c); } } @@ -98,7 +112,7 @@ private final void createDistributedTable(String tableName, PostgresGlobalState String constraintType = rs.getString("constraint_type"); // data types money & bit varying have no default operator class for specified partition method if (! (dataType.equals("money") || dataType.equals("bit varying"))) { - PostgresColumn c = new PostgresColumn(columnName, getColumnType(dataType)); + PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); if (columnConstraints.containsKey(c)) { columnConstraints.get(c).add(constraintType); } else { @@ -117,8 +131,7 @@ private final void createDistributedTable(String tableName, PostgresGlobalState distributeTable(columns, tableName, globalState, con); } - @Override - public void generateDatabase(PostgresGlobalState globalState) throws SQLException { + public void generateDatabase(CitusGlobalState globalState) throws SQLException { // TODO: function reading? add to Postgres implementation? createTables(globalState); for (PostgresTable table : globalState.getSchema().getDatabaseTables()) { @@ -127,7 +140,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio // create local table } else if (Randomly.getBooleanWithRatherLowProbability()) { // create reference table - query = new QueryAdapter("SELECT create_reference_table('" + table.getName() + "');", errors); + QueryAdapter query = new QueryAdapter("SELECT create_reference_table('" + table.getName() + "');", errors); String template = "SELECT create_reference_table(?);"; List fills = Arrays.asList(table.getName()); globalState.fillAndExecuteStatement(query, template, fills); @@ -144,4 +157,100 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio } } + //FIXME: pass in Postgres or CitusGlobalState? + @Override + public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { + // returns connection to coordinator node, test database + Connection con = super.createDatabase(globalState); + + // add citus extension to coordinator node, test database + globalState.getState().logStatement(new QueryAdapter("CREATE EXTENSION citus;")); + try (Statement s = con.createStatement()) { + s.execute("CREATE EXTENSION citus;"); + } + con.close(); + + // reconnect to coordinator node, entry database + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + con = DriverManager.getConnection("jdbc:" + entryURL, username, password); + + // read info about worker nodes + globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); + List workerNodes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet rs = s.executeQuery("SELECT * FROM master_get_active_worker_nodes();"); + while (rs.next()) { + String node_host = rs.getString("node_host"); + int node_port = rs.getInt("node_port"); + WorkerNode w = new WorkerNode(node_host, node_port); + workerNodes.add(w); + } + } + con.close(); + + for (WorkerNode w : workerNodes) { + // connect to worker node, entry database + int hostIndex = entryURL.indexOf(host); + String preHost = entryURL.substring(0, hostIndex); + String postHost = entryURL.substring(databaseIndex - 1); + String entryWorkerURL = preHost + w.get_host() + ":" + w.get_port() + postHost; + // TODO: better way of logging this + globalState.getState().logStatement("\\q"); + globalState.getState().logStatement(entryWorkerURL); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + con = DriverManager.getConnection(entryWorkerURL, username, password); + + // create test database at worker node + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + + // connect to worker node, test database + int databaseIndexWorker = entryWorkerURL.indexOf(entryPath) + 1; + String preDatabaseNameWorker = entryWorkerURL.substring(0, databaseIndex); + String postDatabaseNameWorker = entryWorkerURL.substring(databaseIndex + entryDatabaseName.length()); + String testWorkerURL = preDatabaseNameWorker + databaseName + postDatabaseNameWorker; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + con = DriverManager.getConnection("jdbc:" + testWorkerURL, username, password); + + // add citus extension to worker node, test database + globalState.getState().logStatement("CREATE EXTENSION citus;"); + try (Statement s = con.createStatement()) { + s.execute("CREATE EXTENSION citus;"); + } + con.close(); + } + + // reconnect to coordinator node, test database + // TODO: better way of logging this + globalState.getState().logStatement("\\q"); + globalState.getState().logStatement(testURL); + con = DriverManager.getConnection(testURL, username, password); + + // add worker nodes to coordinator node for test database + for (WorkerNode w : workerNodes) { + // TODO: protect from sql injection - is it necessary though since these are read from the system? + String addWorkers = "SELECT * from master_add_node('" + w.get_host() + "', " + w.get_port() + ");"; + globalState.getState().logStatement(addWorkers); + try (Statement s = con.createStatement()) { + s.execute(addWorkers); + } + } + con.close(); + // reconnect to coordinator node, test database + con = DriverManager.getConnection(testURL, username, password); + return con; + } + + @Override + public String getDBMSName() { + return "citus"; + } + } \ No newline at end of file diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index 691ff5c16..ebbf223a9 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -7,15 +7,17 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import sqlancer.postgres.PostgresSchema; public class CitusSchema extends PostgresSchema { public CitusSchema(List databaseTables, String databaseName) { - super(databaseTables, databaseName); + // FIXME: Will casting to PostgresTable lose CitusTable features? + super(new ArrayList<>(databaseTables), databaseName); } - + public static class CitusTable extends PostgresTable { private PostgresColumn distributionColumn = null; @@ -44,11 +46,10 @@ public Integer getColocationId() { } - // TODO: Override? public static CitusSchema fromConnection(Connection con, String databaseName) throws SQLException { Exception ex = null; try { - List databaseTables = new ArrayList<>(); + List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery( "SELECT table_name, table_schema, table_type, is_insertable_into, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { @@ -70,7 +71,7 @@ public static CitusSchema fromConnection(Connection con, String databaseName) th List databaseColumns = getTableColumns(con, tableName); List indexes = getIndexes(con, tableName); List statistics = getStatistics(con); - PostgresTable t = new PostgresTable(tableName, databaseColumns, indexes, tableType, statistics, + CitusTable t = new CitusTable(tableName, databaseColumns, indexes, tableType, statistics, isView, isInsertable); if (distributionColumnName != null && !distributionColumnName.equals("")) { PostgresColumn distributionColumn = databaseColumns.stream().filter(c -> c.getName().equals(distributionColumnName)).collect(Collectors.toList()).get(0); @@ -92,4 +93,5 @@ public static CitusSchema fromConnection(Connection con, String databaseName) th } throw new AssertionError(ex); } + } \ No newline at end of file diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 5317af682..4de76c230 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -6,9 +6,6 @@ public class CitusCommon extends PostgresCommon { - private CitusCommon() { - } - public static void addCitusErrors(Set errors) { errors.add("recursive CTEs are not supported in distributed queries"); errors.add("could not run distributed query with GROUPING SETS, CUBE, or ROLLUP"); @@ -16,43 +13,40 @@ public static void addCitusErrors(Set errors) { errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); errors.add("functions used in UPDATE queries on distributed tables must not be VOLATILE"); errors.add("STABLE functions used in UPDATE queries cannot be called with column references"); - // should be fixed now - // errors.add("failed to evaluate partition key in insert"); errors.add("functions used in the WHERE clause of modification queries on distributed tables must not be VOLATILE"); - errors.add("cannot perform an INSERT with NULL in the partition column"); errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); errors.add("cannot execute ALTER TABLE command involving partition column"); errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); - // TODO: remove once fixed - // errors.add("unrecognized node type: 127"); - // TODO: SQLancer error - errors.add("not a foreign key or check constraint"); - errors.add("cannot perform an INSERT without a partition column value"); - // TODO: remove once fixed - errors.add("failed to find conversion function from unknown to text"); + errors.add("is not a regular, foreign or partitioned table"); + errors.add("must be a distributed table or a reference table"); // ERROR: cannot create foreign key constraint // Detail: SET NULL or SET DEFAULT is not supported in ON DELETE operation when distribution key is included in the foreign key constraint errors.add("cannot create foreign key constraint"); - // ERROR: cannot pushdown the subquery - // Detail: Complex subqueries and CTEs cannot be in the outer part of the outer join - errors.add("cannot pushdown the subquery"); - // TODO: remove after PostgreSQL 13 upgrade - errors.add("unrecognized configuration parameter \"enable_hashagg_disk\""); - // TODO: remove after PostgreSQL 13 upgrade - errors.add("unrecognized configuration parameter \"enable_groupingsets_hash_disk\""); - errors.add("is not a regular, foreign or partitioned table"); - errors.add("must be a distributed table or a reference table"); + // Citus restrictions on SELECT queries errors.add("complex joins are only supported when all distributed tables are co-located and joined on their distribution columns"); errors.add("complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator"); errors.add("cannot perform distributed planning on this query"); + errors.add("cannot pushdown the subquery"); + // Check for whether repartition joins are enabled is made during query generation // errors.add("the query contains a join that requires repartitioning"); - // SQLancer error + + // SQLancer errors errors.add("non-integer constant in GROUP BY"); - // SQLancer error errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - // SQLancer error errors.add("GROUP BY position"); + errors.add("not a foreign key or check constraint"); + + // current Citus errors to be removed once fixed + errors.add("unrecognized node type: 127"); + errors.add("failed to find conversion function from unknown to text"); + errors.add("failed to evaluate partition key in insert"); + errors.add("cannot perform an INSERT without a partition column value"); + errors.add("cannot perform an INSERT with NULL in the partition column"); + + // current errors to be removed once upgraded to PostgreSQL 13? + errors.add("unrecognized configuration parameter \"enable_hashagg_disk\""); + errors.add("unrecognized configuration parameter \"enable_groupingsets_hash_disk\""); } } \ No newline at end of file diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 23e79956c..99457e76c 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -258,8 +258,8 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce return con; } - private void createTables(PostgresGlobalState globalState) throws SQLException { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { + protected void createTables(PostgresGlobalState globalState) throws SQLException { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(4, 5, 6)) { try { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); Query createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), @@ -271,7 +271,7 @@ private void createTables(PostgresGlobalState globalState) throws SQLException { } } - private void prepareTables(PostgresGlobalState globalState) throws SQLException { + protected void prepareTables(PostgresGlobalState globalState) throws SQLException { StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), PostgresProvider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 6e1dfcd28..99efe2f98 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -107,7 +107,7 @@ public PostgresRowValue getRandomRowValue(Connection con, PostgresStateToReprodu } - private static PostgresDataType getColumnType(String typeString) { + public static PostgresDataType getColumnType(String typeString) { switch (typeString) { case "smallint": case "integer": @@ -294,7 +294,7 @@ public static PostgresSchema fromConnection(Connection con, String databaseName) throw new AssertionError(ex); } - private static List getStatistics(Connection con) throws SQLException { + protected static List getStatistics(Connection con) throws SQLException { List statistics = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SELECT stxname FROM pg_statistic_ext ORDER BY stxname;")) { @@ -306,7 +306,7 @@ private static List getStatistics(Connection con) thro return statistics; } - private static PostgresTable.TableType getTableType(String tableTypeStr) throws AssertionError { + protected static PostgresTable.TableType getTableType(String tableTypeStr) throws AssertionError { PostgresTable.TableType tableType; if (tableTypeStr.contentEquals("public")) { tableType = TableType.STANDARD; @@ -318,7 +318,7 @@ private static PostgresTable.TableType getTableType(String tableTypeStr) throws return tableType; } - private static List getIndexes(Connection con, String tableName) throws SQLException { + protected static List getIndexes(Connection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String @@ -336,7 +336,7 @@ private static List getIndexes(Connection con, String tableName) return indexes; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + protected static List getTableColumns(Connection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 7d7940dfe..97a9af3b8 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -19,7 +19,7 @@ public class PostgresCommon { - private PostgresCommon() { + protected PostgresCommon() { } public static void addCommonFetchErrors(Set errors) { From d57676abfa9df242f99578e02b3e10365660f4cd Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 27 Jul 2020 13:17:15 +0300 Subject: [PATCH 0076/1284] Add CitusProvider option to Main --- src/sqlancer/Main.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 23ad2e774..d265a288d 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -26,6 +26,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.JCommander.Builder; +import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.cockroachdb.CockroachDBProvider; import sqlancer.duckdb.DuckDBProvider; @@ -544,6 +545,7 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new PostgresProvider()); providers.add(new ClickHouseProvider()); providers.add(new DuckDBProvider()); + providers.add(new CitusProvider()); return providers; } From a97ccb236706c9e0da5f10c4f17ad149349ca5fa Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 27 Jul 2020 18:08:39 +0300 Subject: [PATCH 0077/1284] Generic Postgres classes --- src/sqlancer/Main.java | 6 +++++- src/sqlancer/citus/CitusGlobalState.java | 3 ++- src/sqlancer/citus/CitusProvider.java | 14 +++++++------- src/sqlancer/postgres/PostgresGlobalState.java | 4 ++-- src/sqlancer/postgres/PostgresProvider.java | 7 ++++--- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index d265a288d..8fd012815 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -32,6 +32,9 @@ import sqlancer.duckdb.DuckDBProvider; import sqlancer.mariadb.MariaDBProvider; import sqlancer.mysql.MySQLProvider; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; @@ -535,6 +538,7 @@ private boolean run(MainOptions options, ExecutorService execService, return threadsShutdown == 0 ? 0 : options.getErrorExitCode(); } + @SuppressWarnings("unchecked") static List> getDBMSProviders() { List> providers = new ArrayList<>(); providers.add(new SQLite3Provider()); @@ -542,7 +546,7 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new MySQLProvider()); providers.add(new MariaDBProvider()); providers.add(new TiDBProvider()); - providers.add(new PostgresProvider()); + providers.add(new PostgresProvider, PostgresOptions>((Class>)(Object)PostgresGlobalState.class, PostgresOptions.class)); providers.add(new ClickHouseProvider()); providers.add(new DuckDBProvider()); providers.add(new CitusProvider()); diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java index 52b519dc8..73e4053e7 100644 --- a/src/sqlancer/citus/CitusGlobalState.java +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -4,7 +4,7 @@ import sqlancer.postgres.PostgresGlobalState; -public class CitusGlobalState extends PostgresGlobalState { +public class CitusGlobalState extends PostgresGlobalState { private boolean repartition; @@ -16,6 +16,7 @@ public boolean getRepartition() { return repartition; } + // TODO: What if this doesn't exist - why can't the function in PostgresGlobalState call S.fromConnection()? @Override protected void updateSchema() throws SQLException { // FIXME: Will casting lose CitusSchema information? diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index ce38736bb..fcc2b8525 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -23,17 +23,17 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; -public class CitusProvider extends PostgresProvider { +public class CitusProvider extends PostgresProvider { // FIXME: - protected final Set errors = new HashSet<>(); + private static final Set errors = new HashSet<>(); public CitusProvider() { // how to change the extension of the super class? - super(); + super(CitusGlobalState.class, CitusOptions.class); CitusCommon.addCitusErrors(errors); } - private class WorkerNode{ +/* private class WorkerNode{ private final String host; private final int port; @@ -155,10 +155,10 @@ public void generateDatabase(CitusGlobalState globalState) throws SQLException { // allow repartition joins globalState.executeStatement(new QueryAdapter("SET citus.enable_repartition_joins to ON;\n", errors)); } - } + } */ //FIXME: pass in Postgres or CitusGlobalState? - @Override + /* @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { // returns connection to coordinator node, test database Connection con = super.createDatabase(globalState); @@ -247,7 +247,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce con = DriverManager.getConnection(testURL, username, password); return con; } - + */ @Override public String getDBMSName() { return "citus"; diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 58e5be0a4..1415c51ee 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -10,7 +10,7 @@ import sqlancer.GlobalState; import sqlancer.Randomly; -public class PostgresGlobalState extends GlobalState { +public class PostgresGlobalState extends GlobalState { private List operators; private List collates; @@ -91,7 +91,7 @@ public String getRandomOpclass() { @Override protected void updateSchema() throws SQLException { - setSchema(PostgresSchema.fromConnection(getConnection(), getDatabaseName())); + setSchema((S) S.fromConnection(getConnection(), getDatabaseName())); } } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 99457e76c..1fe910e18 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -14,6 +14,7 @@ import sqlancer.AbstractAction; import sqlancer.CompositeTestOracle; +import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Query; @@ -52,14 +53,14 @@ // EXISTS // IN -public class PostgresProvider extends ProviderAdapter { +public class PostgresProvider, O extends PostgresOptions> extends ProviderAdapter { public static boolean generateOnlyKnown; private PostgresGlobalState globalState; - public PostgresProvider() { - super(PostgresGlobalState.class, PostgresOptions.class); + public PostgresProvider(Class globalClass, Class optionsClass) { + super(globalClass, optionsClass); } public enum Action implements AbstractAction { From 5cd7cbb8858938584396c3a4e18567f418a63fd2 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 27 Jul 2020 18:44:32 +0300 Subject: [PATCH 0078/1284] Undo generic Postgres classes, instead alter parameter of parameterized types declared --- src/sqlancer/GlobalState.java | 1 + src/sqlancer/Main.java | 3 +- src/sqlancer/Query.java | 9 + src/sqlancer/QueryAdapter.java | 38 ++++ src/sqlancer/citus/CitusGlobalState.java | 4 +- src/sqlancer/citus/CitusProvider.java | 184 +++++++++--------- .../postgres/PostgresGlobalState.java | 4 +- src/sqlancer/postgres/PostgresProvider.java | 33 ++-- 8 files changed, 168 insertions(+), 108 deletions(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index f2e4e99ad..90f2dc730 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -2,6 +2,7 @@ import java.sql.Connection; import java.sql.SQLException; +import java.util.List; import sqlancer.Main.QueryManager; import sqlancer.Main.StateLogger; diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 8fd012815..9547d4e40 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -34,7 +34,6 @@ import sqlancer.mysql.MySQLProvider; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresOptions; -import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; @@ -546,7 +545,7 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new MySQLProvider()); providers.add(new MariaDBProvider()); providers.add(new TiDBProvider()); - providers.add(new PostgresProvider, PostgresOptions>((Class>)(Object)PostgresGlobalState.class, PostgresOptions.class)); + providers.add(new PostgresProvider(PostgresGlobalState.class, PostgresOptions.class)); providers.add(new ClickHouseProvider()); providers.add(new DuckDBProvider()); providers.add(new CitusProvider()); diff --git a/src/sqlancer/Query.java b/src/sqlancer/Query.java index fbac75548..2169f294e 100644 --- a/src/sqlancer/Query.java +++ b/src/sqlancer/Query.java @@ -2,6 +2,7 @@ import java.sql.SQLException; import java.util.Collection; +import java.util.List; public abstract class Query { @@ -24,6 +25,9 @@ public abstract class Query { */ public abstract boolean execute(GlobalState globalState, String... fills) throws SQLException; + public abstract boolean fillAndExecute(GlobalState globalState, String template, List fills) + throws SQLException; + public abstract Collection getExpectedErrors(); @Override @@ -35,6 +39,11 @@ public SQLancerResultSet executeAndGet(GlobalState globalState, String... throw new AssertionError(); } + public ResultSet fillAndExecuteAndGet(GlobalState globalState, String template, List fills) + throws SQLException { + throw new AssertionError(); + } + public boolean executeLogged(GlobalState globalState) throws SQLException { logQueryString(globalState); return execute(globalState); diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index e0735463b..7a92cdc15 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -6,6 +6,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; +import java.util.List; public class QueryAdapter extends Query { @@ -137,4 +138,41 @@ public Collection getExpectedErrors() { return expectedErrors; } + @Override + public boolean fillAndExecute(GlobalState globalState, String template, List fills) + throws SQLException { + try (PreparedStatement s = globalState.getConnection().prepareStatement(template)) { + for (int i = 1; i < fills.size() + 1; i++) { + s.setString(i, fills.get(i - 1)); + } + s.execute(); + Main.nrSuccessfulActions.addAndGet(1); + return true; + } catch (Exception e) { + Main.nrUnsuccessfulActions.addAndGet(1); + checkException(e); + return false; + } + } + + @Override + public ResultSet fillAndExecuteAndGet(GlobalState globalState, String template, List fills) + throws SQLException { + ResultSet result = null; + PreparedStatement s = globalState.getConnection().prepareStatement(template); + try { + for (int i = 1; i < fills.size() + 1; i++) { + s.setString(i, fills.get(i - 1)); + } + result = s.executeQuery(); + Main.nrSuccessfulActions.addAndGet(1); + return result; + } catch (Exception e) { + s.close(); + Main.nrUnsuccessfulActions.addAndGet(1); + checkException(e); + } + return null; + } + } diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java index 73e4053e7..a1d1f2a57 100644 --- a/src/sqlancer/citus/CitusGlobalState.java +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -4,7 +4,7 @@ import sqlancer.postgres.PostgresGlobalState; -public class CitusGlobalState extends PostgresGlobalState { +public class CitusGlobalState extends PostgresGlobalState { private boolean repartition; @@ -16,10 +16,8 @@ public boolean getRepartition() { return repartition; } - // TODO: What if this doesn't exist - why can't the function in PostgresGlobalState call S.fromConnection()? @Override protected void updateSchema() throws SQLException { - // FIXME: Will casting lose CitusSchema information? setSchema(CitusSchema.fromConnection(getConnection(), getDatabaseName())); } diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index fcc2b8525..e16fbd3b0 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -14,26 +14,26 @@ import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.citus.CitusSchema.CitusTable; import sqlancer.citus.gen.CitusCommon; import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; import sqlancer.postgres.PostgresProvider; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; -public class CitusProvider extends PostgresProvider { - // FIXME: +public class CitusProvider extends PostgresProvider { + private static final Set errors = new HashSet<>(); + + @SuppressWarnings("unchecked") public CitusProvider() { - // how to change the extension of the super class? - super(CitusGlobalState.class, CitusOptions.class); + super((Class)(Object) CitusGlobalState.class, (Class)(Object) CitusOptions.class); CitusCommon.addCitusErrors(errors); } -/* private class WorkerNode{ + private class WorkerNode{ private final String host; private final int port; @@ -131,7 +131,8 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl distributeTable(columns, tableName, globalState, con); } - public void generateDatabase(CitusGlobalState globalState) throws SQLException { + @Override + public void generateDatabase(PostgresGlobalState globalState) throws SQLException { // TODO: function reading? add to Postgres implementation? createTables(globalState); for (PostgresTable table : globalState.getSchema().getDatabaseTables()) { @@ -146,108 +147,111 @@ public void generateDatabase(CitusGlobalState globalState) throws SQLException { globalState.fillAndExecuteStatement(query, template, fills); } else { // create distributed table - createDistributedTable(table.getName(), globalState, globalState.getConnection()); + createDistributedTable(table.getName(), (CitusGlobalState) globalState, globalState.getConnection()); } } - globalState.updateSchema(); + ((CitusGlobalState) globalState).updateSchema(); prepareTables(globalState); - if (globalState.getRepartition()) { + if (((CitusGlobalState) globalState).getRepartition()) { // allow repartition joins globalState.executeStatement(new QueryAdapter("SET citus.enable_repartition_joins to ON;\n", errors)); } - } */ + } //FIXME: pass in Postgres or CitusGlobalState? - /* @Override + @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { - // returns connection to coordinator node, test database - Connection con = super.createDatabase(globalState); - - // add citus extension to coordinator node, test database - globalState.getState().logStatement(new QueryAdapter("CREATE EXTENSION citus;")); - try (Statement s = con.createStatement()) { - s.execute("CREATE EXTENSION citus;"); - } - con.close(); - - // reconnect to coordinator node, entry database - globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - con = DriverManager.getConnection("jdbc:" + entryURL, username, password); - - // read info about worker nodes - globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); - List workerNodes = new ArrayList<>(); - try (Statement s = con.createStatement()) { - ResultSet rs = s.executeQuery("SELECT * FROM master_get_active_worker_nodes();"); - while (rs.next()) { - String node_host = rs.getString("node_host"); - int node_port = rs.getInt("node_port"); - WorkerNode w = new WorkerNode(node_host, node_port); - workerNodes.add(w); - } - } - con.close(); - - for (WorkerNode w : workerNodes) { - // connect to worker node, entry database - int hostIndex = entryURL.indexOf(host); - String preHost = entryURL.substring(0, hostIndex); - String postHost = entryURL.substring(databaseIndex - 1); - String entryWorkerURL = preHost + w.get_host() + ":" + w.get_port() + postHost; - // TODO: better way of logging this - globalState.getState().logStatement("\\q"); - globalState.getState().logStatement(entryWorkerURL); - globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - con = DriverManager.getConnection(entryWorkerURL, username, password); - - // create test database at worker node - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - globalState.getState().logStatement(createDatabaseCommand); + synchronized(CitusProvider.class) { + // returns connection to coordinator node, test database + Connection con = super.createDatabase(globalState); + String entryDatabaseName = entryPath.substring(1); + int databaseIndex = entryURL.indexOf(entryPath) + 1; + // add citus extension to coordinator node, test database + globalState.getState().logStatement(new QueryAdapter("CREATE EXTENSION citus;")); try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); - } - try (Statement s = con.createStatement()) { - s.execute(createDatabaseCommand); + s.execute("CREATE EXTENSION citus;"); } con.close(); - - // connect to worker node, test database - int databaseIndexWorker = entryWorkerURL.indexOf(entryPath) + 1; - String preDatabaseNameWorker = entryWorkerURL.substring(0, databaseIndex); - String postDatabaseNameWorker = entryWorkerURL.substring(databaseIndex + entryDatabaseName.length()); - String testWorkerURL = preDatabaseNameWorker + databaseName + postDatabaseNameWorker; - globalState.getState().logStatement(String.format("\\c %s;", databaseName)); - con = DriverManager.getConnection("jdbc:" + testWorkerURL, username, password); - // add citus extension to worker node, test database - globalState.getState().logStatement("CREATE EXTENSION citus;"); + // reconnect to coordinator node, entry database + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + con = DriverManager.getConnection("jdbc:" + entryURL, username, password); + + // read info about worker nodes + globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); + List workerNodes = new ArrayList<>(); try (Statement s = con.createStatement()) { - s.execute("CREATE EXTENSION citus;"); + ResultSet rs = s.executeQuery("SELECT * FROM master_get_active_worker_nodes();"); + while (rs.next()) { + String node_host = rs.getString("node_name"); + int node_port = rs.getInt("node_port"); + WorkerNode w = new WorkerNode(node_host, node_port); + workerNodes.add(w); + } } con.close(); - } - - // reconnect to coordinator node, test database - // TODO: better way of logging this - globalState.getState().logStatement("\\q"); - globalState.getState().logStatement(testURL); - con = DriverManager.getConnection(testURL, username, password); + + for (WorkerNode w : workerNodes) { + // connect to worker node, entry database + int hostIndex = entryURL.indexOf(host); + String preHost = entryURL.substring(0, hostIndex); + String postHost = entryURL.substring(databaseIndex - 1); + String entryWorkerURL = preHost + w.get_host() + ":" + w.get_port() + postHost; + // TODO: better way of logging this + globalState.getState().logStatement("\\q"); + globalState.getState().logStatement(entryWorkerURL); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + con = DriverManager.getConnection("jdbc:" + entryWorkerURL, username, password); + + // create test database at worker node + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + + // connect to worker node, test database + int databaseIndexWorker = entryWorkerURL.indexOf(entryPath) + 1; + String preDatabaseNameWorker = entryWorkerURL.substring(0, databaseIndexWorker); + String postDatabaseNameWorker = entryWorkerURL.substring(databaseIndexWorker + entryDatabaseName.length()); + String testWorkerURL = preDatabaseNameWorker + databaseName + postDatabaseNameWorker; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + con = DriverManager.getConnection("jdbc:" + testWorkerURL, username, password); + + // add citus extension to worker node, test database + globalState.getState().logStatement("CREATE EXTENSION citus;"); + try (Statement s = con.createStatement()) { + s.execute("CREATE EXTENSION citus;"); + } + con.close(); + } - // add worker nodes to coordinator node for test database - for (WorkerNode w : workerNodes) { - // TODO: protect from sql injection - is it necessary though since these are read from the system? - String addWorkers = "SELECT * from master_add_node('" + w.get_host() + "', " + w.get_port() + ");"; - globalState.getState().logStatement(addWorkers); - try (Statement s = con.createStatement()) { - s.execute(addWorkers); + // reconnect to coordinator node, test database + // TODO: better way of logging this + globalState.getState().logStatement("\\q"); + globalState.getState().logStatement(testURL); + con = DriverManager.getConnection("jdbc:" + testURL, username, password); + + // add worker nodes to coordinator node for test database + for (WorkerNode w : workerNodes) { + // TODO: protect from sql injection - is it necessary though since these are read from the system? + String addWorkers = "SELECT * from master_add_node('" + w.get_host() + "', " + w.get_port() + ");"; + globalState.getState().logStatement(addWorkers); + try (Statement s = con.createStatement()) { + s.execute(addWorkers); + } } + con.close(); + // reconnect to coordinator node, test database + con = DriverManager.getConnection("jdbc:" + testURL, username, password); + return con; } - con.close(); - // reconnect to coordinator node, test database - con = DriverManager.getConnection(testURL, username, password); - return con; } - */ + @Override public String getDBMSName() { return "citus"; diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 1415c51ee..58e5be0a4 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -10,7 +10,7 @@ import sqlancer.GlobalState; import sqlancer.Randomly; -public class PostgresGlobalState extends GlobalState { +public class PostgresGlobalState extends GlobalState { private List operators; private List collates; @@ -91,7 +91,7 @@ public String getRandomOpclass() { @Override protected void updateSchema() throws SQLException { - setSchema((S) S.fromConnection(getConnection(), getDatabaseName())); + setSchema(PostgresSchema.fromConnection(getConnection(), getDatabaseName())); } } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 1fe910e18..5ee60f5d7 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -14,7 +14,6 @@ import sqlancer.AbstractAction; import sqlancer.CompositeTestOracle; -import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Query; @@ -53,14 +52,24 @@ // EXISTS // IN -public class PostgresProvider, O extends PostgresOptions> extends ProviderAdapter { +public class PostgresProvider extends ProviderAdapter { public static boolean generateOnlyKnown; private PostgresGlobalState globalState; - public PostgresProvider(Class globalClass, Class optionsClass) { - super(globalClass, optionsClass); + protected String entryURL; + protected String username; + protected String password; + protected String entryPath; + protected String host; + protected int port; + protected String testURL; + protected String databaseName; + protected String createDatabaseCommand; + + public PostgresProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); } public enum Action implements AbstractAction { @@ -203,10 +212,10 @@ protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLEx @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { - String username = globalState.getOptions().getUserName(); - String password = globalState.getOptions().getPassword(); - String entryPath = "/test"; - String entryURL = globalState.getDmbsSpecificOptions().connectionURL; + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + entryPath = "/test"; + entryURL = globalState.getDmbsSpecificOptions().connectionURL; // trim URL to exclude "jdbc:" if (entryURL.startsWith("jdbc:")) { entryURL = entryURL.substring(5); @@ -233,15 +242,17 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce if (pathURI != null) { entryPath = pathURI; } + host = uri.getHost(); + port = uri.getPort(); } catch (URISyntaxException e) { throw new AssertionError(e); } String entryDatabaseName = entryPath.substring(1); - String databaseName = globalState.getDatabaseName(); + databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - String createDatabaseCommand = getCreateDatabaseCommand(databaseName, con, globalState); + createDatabaseCommand = getCreateDatabaseCommand(databaseName, con, globalState); globalState.getState().logStatement(createDatabaseCommand); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); @@ -253,7 +264,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce int databaseIndex = entryURL.indexOf(entryPath) + 1; String preDatabaseName = entryURL.substring(0, databaseIndex); String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); - String testURL = preDatabaseName + databaseName + postDatabaseName; + testURL = preDatabaseName + databaseName + postDatabaseName; globalState.getState().logStatement(String.format("\\c %s;", databaseName)); con = DriverManager.getConnection("jdbc:" + testURL, username, password); return con; From c3a1303448dc53f7dc8c37dcaa7b8cc4de64af63 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 28 Jul 2020 15:42:27 +0300 Subject: [PATCH 0079/1284] Alternative constructors for PostgresProvider --- src/sqlancer/Main.java | 5 ++--- src/sqlancer/citus/CitusGlobalState.java | 2 -- src/sqlancer/citus/CitusOptions.java | 2 +- src/sqlancer/citus/CitusProvider.java | 2 +- src/sqlancer/citus/CitusSchema.java | 2 +- src/sqlancer/citus/gen/CitusCommon.java | 2 +- src/sqlancer/postgres/PostgresProvider.java | 6 +++++- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 9547d4e40..686408d22 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -537,7 +537,6 @@ private boolean run(MainOptions options, ExecutorService execService, return threadsShutdown == 0 ? 0 : options.getErrorExitCode(); } - @SuppressWarnings("unchecked") static List> getDBMSProviders() { List> providers = new ArrayList<>(); providers.add(new SQLite3Provider()); @@ -545,10 +544,10 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new MySQLProvider()); providers.add(new MariaDBProvider()); providers.add(new TiDBProvider()); - providers.add(new PostgresProvider(PostgresGlobalState.class, PostgresOptions.class)); + providers.add(new PostgresProvider()); + providers.add(new CitusProvider()); providers.add(new ClickHouseProvider()); providers.add(new DuckDBProvider()); - providers.add(new CitusProvider()); return providers; } diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java index a1d1f2a57..669911aba 100644 --- a/src/sqlancer/citus/CitusGlobalState.java +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -22,5 +22,3 @@ protected void updateSchema() throws SQLException { } } - - \ No newline at end of file diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index bc9e2b8f7..3abb0cece 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -9,4 +9,4 @@ public class CitusOptions extends PostgresOptions { @Parameter(names= "--repartition") public boolean repartition = true; -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index e16fbd3b0..b48a7de0a 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -257,4 +257,4 @@ public String getDBMSName() { return "citus"; } -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index ebbf223a9..0a1279320 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -94,4 +94,4 @@ public static CitusSchema fromConnection(Connection con, String databaseName) th throw new AssertionError(ex); } -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 4de76c230..06856df9a 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -49,4 +49,4 @@ public static void addCitusErrors(Set errors) { errors.add("unrecognized configuration parameter \"enable_groupingsets_hash_disk\""); } -} \ No newline at end of file +} diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 5ee60f5d7..35b527b13 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -68,7 +68,11 @@ public class PostgresProvider extends ProviderAdapter globalClass, Class optionClass) { + public PostgresProvider() { + super(PostgresGlobalState.class, PostgresOptions.class); + } + + protected PostgresProvider(Class globalClass, Class optionClass) { super(globalClass, optionClass); } From 559fdf18ab64bf1935c567ea125ce10e5bc59d67 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Wed, 29 Jul 2020 15:50:44 +0300 Subject: [PATCH 0080/1284] Create Citus gen classes --- src/sqlancer/GlobalState.java | 1 - src/sqlancer/Main.java | 2 - src/sqlancer/Query.java | 9 - src/sqlancer/QueryAdapter.java | 38 ---- src/sqlancer/citus/CitusProvider.java | 202 ++++++++++++++++-- src/sqlancer/citus/CitusSchema.java | 1 + .../citus/gen/CitusAlterTableGenerator.java | 40 ++++ src/sqlancer/citus/gen/CitusCommon.java | 9 + .../citus/gen/CitusDeleteGenerator.java | 19 ++ .../citus/gen/CitusInsertGenerator.java | 18 ++ src/sqlancer/citus/gen/CitusSetGenerator.java | 18 ++ .../citus/gen/CitusTableGenerator.java | 15 ++ .../citus/gen/CitusUpdateGenerator.java | 18 ++ .../citus/gen/CitusViewGenerator.java | 18 ++ src/sqlancer/postgres/PostgresProvider.java | 4 +- .../gen/PostgresAlterTableGenerator.java | 31 +-- src/sqlancer/postgres/gen/PostgresCommon.java | 2 +- .../postgres/gen/PostgresDeleteGenerator.java | 4 +- .../postgres/gen/PostgresInsertGenerator.java | 4 +- .../postgres/gen/PostgresSetGenerator.java | 4 +- .../postgres/gen/PostgresTableGenerator.java | 2 +- .../postgres/gen/PostgresUpdateGenerator.java | 4 +- .../postgres/gen/PostgresViewGenerator.java | 4 +- 23 files changed, 367 insertions(+), 100 deletions(-) create mode 100644 src/sqlancer/citus/gen/CitusAlterTableGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusDeleteGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusInsertGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusSetGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusTableGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusUpdateGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusViewGenerator.java diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 90f2dc730..f2e4e99ad 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -2,7 +2,6 @@ import java.sql.Connection; import java.sql.SQLException; -import java.util.List; import sqlancer.Main.QueryManager; import sqlancer.Main.StateLogger; diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 686408d22..c2c328a60 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -32,8 +32,6 @@ import sqlancer.duckdb.DuckDBProvider; import sqlancer.mariadb.MariaDBProvider; import sqlancer.mysql.MySQLProvider; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresOptions; import sqlancer.postgres.PostgresProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; diff --git a/src/sqlancer/Query.java b/src/sqlancer/Query.java index 2169f294e..fbac75548 100644 --- a/src/sqlancer/Query.java +++ b/src/sqlancer/Query.java @@ -2,7 +2,6 @@ import java.sql.SQLException; import java.util.Collection; -import java.util.List; public abstract class Query { @@ -25,9 +24,6 @@ public abstract class Query { */ public abstract boolean execute(GlobalState globalState, String... fills) throws SQLException; - public abstract boolean fillAndExecute(GlobalState globalState, String template, List fills) - throws SQLException; - public abstract Collection getExpectedErrors(); @Override @@ -39,11 +35,6 @@ public SQLancerResultSet executeAndGet(GlobalState globalState, String... throw new AssertionError(); } - public ResultSet fillAndExecuteAndGet(GlobalState globalState, String template, List fills) - throws SQLException { - throw new AssertionError(); - } - public boolean executeLogged(GlobalState globalState) throws SQLException { logQueryString(globalState); return execute(globalState); diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index 7a92cdc15..e0735463b 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -6,7 +6,6 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Collection; -import java.util.List; public class QueryAdapter extends Query { @@ -138,41 +137,4 @@ public Collection getExpectedErrors() { return expectedErrors; } - @Override - public boolean fillAndExecute(GlobalState globalState, String template, List fills) - throws SQLException { - try (PreparedStatement s = globalState.getConnection().prepareStatement(template)) { - for (int i = 1; i < fills.size() + 1; i++) { - s.setString(i, fills.get(i - 1)); - } - s.execute(); - Main.nrSuccessfulActions.addAndGet(1); - return true; - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - checkException(e); - return false; - } - } - - @Override - public ResultSet fillAndExecuteAndGet(GlobalState globalState, String template, List fills) - throws SQLException { - ResultSet result = null; - PreparedStatement s = globalState.getConnection().prepareStatement(template); - try { - for (int i = 1; i < fills.size() + 1; i++) { - s.setString(i, fills.get(i - 1)); - } - result = s.executeQuery(); - Main.nrSuccessfulActions.addAndGet(1); - return result; - } catch (Exception e) { - s.close(); - Main.nrUnsuccessfulActions.addAndGet(1); - checkException(e); - } - return null; - } - } diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index b48a7de0a..68483fcfe 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -12,15 +12,36 @@ import java.util.List; import java.util.Set; +import sqlancer.AbstractAction; +import sqlancer.IgnoreMeException; +import sqlancer.Query; import sqlancer.QueryAdapter; +import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.citus.gen.CitusCommon; +import sqlancer.StatementExecutor; +import sqlancer.citus.gen.*; +import sqlancer.postgres.gen.PostgresAnalyzeGenerator; +import sqlancer.postgres.gen.PostgresClusterGenerator; +import sqlancer.postgres.gen.PostgresCommentGenerator; +import sqlancer.postgres.gen.PostgresDiscardGenerator; +import sqlancer.postgres.gen.PostgresDropIndexGenerator; +import sqlancer.postgres.gen.PostgresIndexGenerator; +import sqlancer.postgres.gen.PostgresNotifyGenerator; +import sqlancer.postgres.gen.PostgresQueryCatalogGenerator; +import sqlancer.postgres.gen.PostgresReindexGenerator; +import sqlancer.postgres.gen.PostgresSequenceGenerator; +import sqlancer.postgres.gen.PostgresStatisticsGenerator; +import sqlancer.postgres.gen.PostgresTransactionGenerator; +import sqlancer.postgres.gen.PostgresTruncateGenerator; +import sqlancer.postgres.gen.PostgresVacuumGenerator; +import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresOptions; import sqlancer.postgres.PostgresProvider; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; public class CitusProvider extends PostgresProvider { @@ -33,6 +54,126 @@ public CitusProvider() { CitusCommon.addCitusErrors(errors); } + public enum Action implements AbstractAction { + ANALYZE(PostgresAnalyzeGenerator::create), // + ALTER_TABLE(g -> CitusAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g, + generateOnlyKnown)), // + CLUSTER(PostgresClusterGenerator::create), // + COMMIT(g -> { + Query query; + if (Randomly.getBoolean()) { + query = new QueryAdapter("COMMIT", true); + } else if (Randomly.getBoolean()) { + query = PostgresTransactionGenerator.executeBegin(); + } else { + query = new QueryAdapter("ROLLBACK", true); + } + return query; + }), // + CREATE_STATISTICS(PostgresStatisticsGenerator::insert), // + DROP_STATISTICS(PostgresStatisticsGenerator::remove), // + DELETE(CitusDeleteGenerator::create), // + DISCARD(PostgresDiscardGenerator::create), // + DROP_INDEX(PostgresDropIndexGenerator::create), // + INSERT(CitusInsertGenerator::insert), // + UPDATE(CitusUpdateGenerator::create), // + TRUNCATE(PostgresTruncateGenerator::create), // + VACUUM(PostgresVacuumGenerator::create), // + REINDEX(PostgresReindexGenerator::create), // + SET(CitusSetGenerator::create), // + CREATE_INDEX(PostgresIndexGenerator::generate), // + SET_CONSTRAINTS((g) -> { + StringBuilder sb = new StringBuilder(); + sb.append("SET CONSTRAINTS ALL "); + sb.append(Randomly.fromOptions("DEFERRED", "IMMEDIATE")); + return new QueryAdapter(sb.toString()); + }), // + RESET_ROLE((g) -> new QueryAdapter("RESET ROLE")), // + COMMENT_ON(PostgresCommentGenerator::generate), // + RESET((g) -> new QueryAdapter("RESET ALL") /* + * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also + * configuration parameter + */), // + NOTIFY(PostgresNotifyGenerator::createNotify), // + LISTEN((g) -> PostgresNotifyGenerator.createListen()), // + UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // + CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // + CREATE_VIEW(CitusViewGenerator::create), // + QUERY_CATALOG((g) -> PostgresQueryCatalogGenerator.query()); + + private final QueryProvider queryProvider; + + Action(QueryProvider queryProvider) { + this.queryProvider = queryProvider; + } + + @Override + public Query getQuery(PostgresGlobalState state) throws SQLException { + return queryProvider.getQuery(state); + } + } + + private static int mapActions(PostgresGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + switch (a) { + case CREATE_INDEX: + case CLUSTER: + nrPerformed = r.getInteger(0, 3); + break; + case CREATE_STATISTICS: + nrPerformed = r.getInteger(0, 5); + break; + case DISCARD: + case DROP_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case COMMIT: + nrPerformed = r.getInteger(0, 0); + break; + case ALTER_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case REINDEX: + case RESET: + nrPerformed = r.getInteger(0, 3); + break; + case DELETE: + case RESET_ROLE: + case SET: + case QUERY_CATALOG: + nrPerformed = r.getInteger(0, 5); + break; + case ANALYZE: + nrPerformed = r.getInteger(0, 3); + break; + case VACUUM: + case SET_CONSTRAINTS: + case COMMENT_ON: + case NOTIFY: + case LISTEN: + case UNLISTEN: + case CREATE_SEQUENCE: + case DROP_STATISTICS: + case TRUNCATE: + nrPerformed = r.getInteger(0, 2); + break; + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + + } + private class WorkerNode{ private final String host; @@ -56,10 +197,9 @@ public int get_port() { private static void distributeTable(List columns, String tableName, CitusGlobalState globalState, Connection con) throws SQLException { if (columns.size() != 0) { PostgresColumn columnToDistribute = Randomly.fromList(columns); - QueryAdapter query = new QueryAdapter("SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + "');", errors); String template = "SELECT create_distributed_table(?, ?);"; - List fills = Arrays.asList(tableName, columnToDistribute.getName()); - globalState.fillAndExecuteStatement(query, template, fills); + QueryAdapter query = new QueryAdapter(template, errors); + globalState.executeStatement(query, template, tableName, columnToDistribute.getName()); // distribution column cannot take NULL value // TODO: find a way to protect from SQL injection without '' around string input query = new QueryAdapter("ALTER TABLE " + tableName + " ALTER COLUMN " + columnToDistribute.getName() + " SET NOT NULL;", errors); @@ -69,11 +209,9 @@ private static void distributeTable(List columns, String tableNa private static List getTableConstraints(String tableName, CitusGlobalState globalState, Connection con) throws SQLException { List constraints = new ArrayList<>(); - QueryAdapter query = new QueryAdapter("SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + tableName + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"); String template = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; - List fills = new ArrayList<>(); - fills.add(tableName); - ResultSet rs = query.fillAndExecuteAndGet(globalState, template, fills); + QueryAdapter query = new QueryAdapter(template); + ResultSet rs = query.executeAndGet(globalState, template, tableName); while (rs.next()) { constraints.add(rs.getString("constraint_type")); } @@ -84,10 +222,9 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl List columns = new ArrayList<>(); List tableConstraints = getTableConstraints(tableName, globalState, con); if (tableConstraints.size() == 0) { - QueryAdapter query = new QueryAdapter("SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + tableName + "';"); String template = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;"; - List fills = Arrays.asList(tableName); - ResultSet rs = query.fillAndExecuteAndGet(globalState, template, fills); + QueryAdapter query = new QueryAdapter(template); + ResultSet rs = query.executeAndGet(globalState, template, tableName); while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); @@ -98,14 +235,10 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl } } } else { - // TODO: multiple constraints? HashMap> columnConstraints = new HashMap<>(); - QueryAdapter query = new QueryAdapter("SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + tableName + "';"); - // TODO: decide whether to log - // globalState.getState().statements.add(query); String template = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;"; - List fills = Arrays.asList(tableName); - ResultSet rs = query.fillAndExecuteAndGet(globalState, template, fills); + QueryAdapter query = new QueryAdapter(template); + ResultSet rs = query.executeAndGet(globalState, template, tableName); while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); @@ -136,15 +269,13 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio // TODO: function reading? add to Postgres implementation? createTables(globalState); for (PostgresTable table : globalState.getSchema().getDatabaseTables()) { - // TODO: random 0-1 range double - if (Randomly.getBooleanWithRatherLowProbability()) { + if (table.getTableType() == TableType.TEMPORARY || Randomly.getBooleanWithRatherLowProbability()) { // create local table } else if (Randomly.getBooleanWithRatherLowProbability()) { // create reference table - QueryAdapter query = new QueryAdapter("SELECT create_reference_table('" + table.getName() + "');", errors); String template = "SELECT create_reference_table(?);"; - List fills = Arrays.asList(table.getName()); - globalState.fillAndExecuteStatement(query, template, fills); + QueryAdapter query = new QueryAdapter(template, errors); + globalState.executeStatement(query, template, table.getName()); } else { // create distributed table createDistributedTable(table.getName(), (CitusGlobalState) globalState, globalState.getConnection()); @@ -158,7 +289,6 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio } } - //FIXME: pass in Postgres or CitusGlobalState? @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { synchronized(CitusProvider.class) { @@ -252,6 +382,32 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce } } + @Override + protected void createTables(PostgresGlobalState globalState) throws SQLException { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { + try { + String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); + Query createTable = CitusTableGenerator.generate(tableName, globalState.getSchema(), + generateOnlyKnown, globalState); + globalState.executeStatement(createTable); + } catch (IgnoreMeException e) { + + } + } + } + + @Override + protected void prepareTables(PostgresGlobalState globalState) throws SQLException { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), CitusProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new QueryAdapter("COMMIT", true)); + globalState.executeStatement(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); + } + @Override public String getDBMSName() { return "citus"; diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index 0a1279320..c20eaf87a 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -64,6 +64,7 @@ public static CitusSchema fromConnection(Connection con, String databaseName) th } // TODO: also check insertable // TODO: insert into view? + // FIXME: This part looks like there will be improvements, should we be concerned that I am overwriting the method? boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || // tableTypeStr.contains("LOCAL TEMPORARY") && // !isInsertable; diff --git a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java new file mode 100644 index 000000000..a84d359c5 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java @@ -0,0 +1,40 @@ +package sqlancer.citus.gen; + +import java.util.List; +import java.util.Set; + +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.gen.PostgresAlterTableGenerator; + +public class CitusAlterTableGenerator extends PostgresAlterTableGenerator { + + public CitusAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, + boolean generateOnlyKnown) { + super(randomTable, globalState, generateOnlyKnown); + } + + @Override + public List getActions(Set errors) { + List action = super.getActions(errors); + CitusCommon.addCitusErrors(errors); + action.remove(Action.ALTER_COLUMN_SET_STATISTICS); + action.remove(Action.ALTER_COLUMN_SET_ATTRIBUTE_OPTION); + action.remove(Action.ALTER_COLUMN_RESET_ATTRIBUTE_OPTION); + action.remove(Action.ALTER_COLUMN_SET_STORAGE); + action.remove(Action.DISABLE_ROW_LEVEL_SECURITY); + action.remove(Action.ENABLE_ROW_LEVEL_SECURITY); + action.remove(Action.FORCE_ROW_LEVEL_SECURITY); + action.remove(Action.NO_FORCE_ROW_LEVEL_SECURITY); + action.remove(Action.CLUSTER_ON); + action.remove(Action.SET_WITHOUT_CLUSTER); + action.remove(Action.SET_WITH_OIDS); + action.remove(Action.SET_WITHOUT_OIDS); + action.remove(Action.SET_LOGGED_UNLOGGED); + action.remove(Action.NOT_OF); + action.remove(Action.OWNER_TO); + action.remove(Action.REPLICA_IDENTITY); + return action; + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 06856df9a..094ed8fe4 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -2,6 +2,9 @@ import java.util.Set; +import sqlancer.Randomly; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.gen.PostgresCommon; public class CitusCommon extends PostgresCommon { @@ -48,5 +51,11 @@ public static void addCitusErrors(Set errors) { errors.add("unrecognized configuration parameter \"enable_hashagg_disk\""); errors.add("unrecognized configuration parameter \"enable_groupingsets_hash_disk\""); } + + public static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, + Set errors) { + addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); + CitusCommon.addCitusErrors(errors); + } } diff --git a/src/sqlancer/citus/gen/CitusDeleteGenerator.java b/src/sqlancer/citus/gen/CitusDeleteGenerator.java new file mode 100644 index 000000000..eafc79587 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusDeleteGenerator.java @@ -0,0 +1,19 @@ +package sqlancer.citus.gen; + +import java.util.Set; + +import sqlancer.Query; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresDeleteGenerator; + +public class CitusDeleteGenerator extends PostgresDeleteGenerator { + + public static Query create(PostgresGlobalState globalState) { + Query deleteQuery = PostgresDeleteGenerator.create(globalState); + Set errors = (Set) deleteQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return deleteQuery; + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/gen/CitusInsertGenerator.java b/src/sqlancer/citus/gen/CitusInsertGenerator.java new file mode 100644 index 000000000..c649abcfc --- /dev/null +++ b/src/sqlancer/citus/gen/CitusInsertGenerator.java @@ -0,0 +1,18 @@ +package sqlancer.citus.gen; + +import java.util.HashSet; + +import sqlancer.Query; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresInsertGenerator; + +public class CitusInsertGenerator extends PostgresInsertGenerator { + + public static Query insert(PostgresGlobalState globalState) { + Query insertQuery = PostgresInsertGenerator.insert(globalState); + HashSet errors = (HashSet) insertQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return insertQuery; + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/gen/CitusSetGenerator.java b/src/sqlancer/citus/gen/CitusSetGenerator.java new file mode 100644 index 000000000..c82077525 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusSetGenerator.java @@ -0,0 +1,18 @@ +package sqlancer.citus.gen; + +import java.util.Set; + +import sqlancer.Query; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresSetGenerator; + +public class CitusSetGenerator extends PostgresSetGenerator { + + public static Query create(PostgresGlobalState globalState) { + Query setQuery = PostgresSetGenerator.create(globalState); + Set errors = (Set) setQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return setQuery; + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java new file mode 100644 index 000000000..ddd657eba --- /dev/null +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -0,0 +1,15 @@ +package sqlancer.citus.gen; + +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.gen.PostgresTableGenerator; + +public class CitusTableGenerator extends PostgresTableGenerator { + + public CitusTableGenerator(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, + PostgresGlobalState globalState) { + super(tableName, newSchema, generateOnlyKnown, globalState); + CitusCommon.addCitusErrors(errors); + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/gen/CitusUpdateGenerator.java b/src/sqlancer/citus/gen/CitusUpdateGenerator.java new file mode 100644 index 000000000..71513f9f9 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusUpdateGenerator.java @@ -0,0 +1,18 @@ +package sqlancer.citus.gen; + +import java.util.Set; + +import sqlancer.Query; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresUpdateGenerator; + +public class CitusUpdateGenerator extends PostgresUpdateGenerator { + + public static Query create(PostgresGlobalState globalState) { + Query updateQuery = PostgresUpdateGenerator.create(globalState); + Set errors = (Set) updateQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return updateQuery; + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/gen/CitusViewGenerator.java b/src/sqlancer/citus/gen/CitusViewGenerator.java new file mode 100644 index 000000000..9fcbbcc95 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusViewGenerator.java @@ -0,0 +1,18 @@ +package sqlancer.citus.gen; + +import java.util.Set; + +import sqlancer.Query; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresViewGenerator; + +public class CitusViewGenerator extends PostgresViewGenerator { + + public static Query create(PostgresGlobalState globalState) { + Query viewQuery = PostgresViewGenerator.create(globalState); + Set errors = (Set) viewQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return viewQuery; + } + +} \ No newline at end of file diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 35b527b13..07bf53bd1 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -135,7 +135,7 @@ public Query getQuery(PostgresGlobalState state) throws SQLException { } } - private static int mapActions(PostgresGlobalState globalState, Action a) { + protected static int mapActions(PostgresGlobalState globalState, Action a) { Randomly r = globalState.getRandomly(); int nrPerformed; switch (a) { @@ -275,7 +275,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce } protected void createTables(PostgresGlobalState globalState) throws SQLException { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(4, 5, 6)) { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { try { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); Query createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 2fcdb7414..cac22d5c2 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -23,7 +23,7 @@ public class PostgresAlterTableGenerator { private List opClasses; private PostgresGlobalState globalState; - private enum Action { + protected enum Action { // ALTER_TABLE_ADD_COLUMN, // [ COLUMN ] column data_type [ COLLATE collation ] [ // column_constraint [ ... ] ] ALTER_TABLE_DROP_COLUMN, // DROP [ COLUMN ] [ IF EXISTS ] column [ RESTRICT | CASCADE ] @@ -76,8 +76,7 @@ private enum Attribute { } }; - public Query generate() { - Set errors = new HashSet<>(); + public List getActions(Set errors) { PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonInsertUpdateErrors(errors); PostgresCommon.addCommonTableErrors(errors); @@ -93,16 +92,6 @@ public Query generate() { errors.add("could not find cast from"); errors.add("does not exist"); // TODO: investigate errors.add("constraints on permanent tables may reference only permanent tables"); - StringBuilder sb = new StringBuilder(); - sb.append("ALTER TABLE "); - if (Randomly.getBoolean()) { - sb.append(" ONLY"); - errors.add("cannot use ONLY for foreign key on partitioned table"); - } - sb.append(" "); - sb.append(randomTable.getName()); - sb.append(" "); - int i = 0; List action; if (Randomly.getBoolean()) { action = Randomly.nonEmptySubset(Action.values()); @@ -121,6 +110,22 @@ public Query generate() { if (action.isEmpty()) { throw new IgnoreMeException(); } + return action; + } + + public Query generate() { + Set errors = new HashSet<>(); + int i = 0; + List action = getActions(errors); + StringBuilder sb = new StringBuilder(); + sb.append("ALTER TABLE "); + if (Randomly.getBoolean()) { + sb.append(" ONLY"); + errors.add("cannot use ONLY for foreign key on partitioned table"); + } + sb.append(" "); + sb.append(randomTable.getName()); + sb.append(" "); for (Action a : action) { if (i++ != 0) { sb.append(", "); diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 97a9af3b8..3df889c35 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -262,7 +262,7 @@ public static void addTableConstraint(StringBuilder sb, PostgresTable table, Pos addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); } - private static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, + protected static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, TableConstraints t, Set errors) { List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); List otherColumns; diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 1fc592235..025bdc80d 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -11,9 +11,9 @@ import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; -public final class PostgresDeleteGenerator { +public class PostgresDeleteGenerator { - private PostgresDeleteGenerator() { + protected PostgresDeleteGenerator() { } public static Query create(PostgresGlobalState globalState) { diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 759249326..6d8f8d01e 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -14,9 +14,9 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public final class PostgresInsertGenerator { +public class PostgresInsertGenerator { - private PostgresInsertGenerator() { + protected PostgresInsertGenerator() { } public static Query insert(PostgresGlobalState globalState) { diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index d403f0e0a..dad7f4d37 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -9,9 +9,9 @@ import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; -public final class PostgresSetGenerator { +public class PostgresSetGenerator { - private PostgresSetGenerator() { + protected PostgresSetGenerator() { } private enum ConfigurationOption { diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 68226d1dd..78ae99907 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -27,7 +27,7 @@ public class PostgresTableGenerator { private boolean isTemporaryTable; private final PostgresSchema newSchema; private final List columnsToBeAdded = new ArrayList<>(); - private final Set errors = new HashSet<>(); + protected final Set errors = new HashSet<>(); private final PostgresTable table; private final boolean generateOnlyKnown; private final PostgresGlobalState globalState; diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 16d0cb9e3..a92280451 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -15,9 +15,9 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public final class PostgresUpdateGenerator { +public class PostgresUpdateGenerator { - private PostgresUpdateGenerator() { + protected PostgresUpdateGenerator() { } public static Query create(PostgresGlobalState globalState) { diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index afc91d341..3289b25d4 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -11,9 +11,9 @@ import sqlancer.postgres.ast.PostgresSelect; import sqlancer.sqlite3.gen.SQLite3Common; -public final class PostgresViewGenerator { +public class PostgresViewGenerator { - private PostgresViewGenerator() { + protected PostgresViewGenerator() { } public static Query create(PostgresGlobalState globalState) { From f3640429dc0d7a8d6d91e63ed6ebde7c064ea6c6 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 30 Jul 2020 10:55:04 +0300 Subject: [PATCH 0081/1284] Split connection reading into two iterations, read distribution-related info in 2nd iteration --- src/sqlancer/citus/CitusSchema.java | 51 +++++++++---------- .../citus/gen/CitusDeleteGenerator.java | 1 - src/sqlancer/postgres/PostgresSchema.java | 4 ++ 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index c20eaf87a..f8634e5a5 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -20,14 +20,23 @@ public CitusSchema(List databaseTables, String databaseName) { public static class CitusTable extends PostgresTable { - private PostgresColumn distributionColumn = null; - private Integer colocationId = null; + private PostgresColumn distributionColumn; + private Integer colocationId; public CitusTable(String tableName, List columns, List indexes, - TableType tableType, List statistics, boolean isView, boolean isInsertable) { + TableType tableType, List statistics, boolean isView, boolean isInsertable, PostgresColumn distributionColumn, Integer colocationId) { super(tableName, columns, indexes, tableType, statistics, isView, isInsertable); + this.distributionColumn = distributionColumn; + this.colocationId = colocationId; } + public CitusTable(PostgresTable table, PostgresColumn distributionColumn, Integer colocationId) { + super(table.getName(), table.getColumns(), table.getIndexes(), table.getTableType(), + table.getStatistics(), table.isView(), table.isInsertable()); + this.distributionColumn = distributionColumn; + this.colocationId = colocationId; + } + public void setDistributionColumn(PostgresColumn distributionColumn) { this.distributionColumn = distributionColumn; } @@ -47,44 +56,32 @@ public Integer getColocationId() { } public static CitusSchema fromConnection(Connection con, String databaseName) throws SQLException { + PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); Exception ex = null; try { List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery( - "SELECT table_name, table_schema, table_type, is_insertable_into, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { + "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { while (rs.next()) { String tableName = rs.getString("table_name"); - String tableTypeSchema = rs.getString("table_schema"); - boolean isInsertable = rs.getBoolean("is_insertable_into"); String distributionColumnName = rs.getString("dist_col_name"); Integer colocationId = rs.getInt("colocationid"); if (rs.wasNull()) { colocationId = null; } - // TODO: also check insertable - // TODO: insert into view? - // FIXME: This part looks like there will be improvements, should we be concerned that I am overwriting the method? - boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || - // tableTypeStr.contains("LOCAL TEMPORARY") && - // !isInsertable; - PostgresTable.TableType tableType = getTableType(tableTypeSchema); - List databaseColumns = getTableColumns(con, tableName); - List indexes = getIndexes(con, tableName); - List statistics = getStatistics(con); - CitusTable t = new CitusTable(tableName, databaseColumns, indexes, tableType, statistics, - isView, isInsertable); - if (distributionColumnName != null && !distributionColumnName.equals("")) { - PostgresColumn distributionColumn = databaseColumns.stream().filter(c -> c.getName().equals(distributionColumnName)).collect(Collectors.toList()).get(0); - t.setDistributionColumn(distributionColumn); - } - if (colocationId != null) { - t.setColocationId(colocationId); + // FIXME: Are the CitusTable-specific features I'm adding going to persist after the function call? + PostgresTable t = schema.getDatabaseTable(tableName); + PostgresColumn distributionColumn = null; + if (t == null) { + continue; } - for (PostgresColumn c : databaseColumns) { - c.setTable(t); + if (distributionColumnName != null && !distributionColumnName.equals("")) { + distributionColumn = t.getColumns().stream().filter(c -> c.getName(). + equals(distributionColumnName)).collect(Collectors.toList()).get(0); } - databaseTables.add(t); + CitusTable tCitus = new CitusTable(t, distributionColumn, colocationId); + databaseTables.add(tCitus); } } } diff --git a/src/sqlancer/citus/gen/CitusDeleteGenerator.java b/src/sqlancer/citus/gen/CitusDeleteGenerator.java index eafc79587..134b7d51d 100644 --- a/src/sqlancer/citus/gen/CitusDeleteGenerator.java +++ b/src/sqlancer/citus/gen/CitusDeleteGenerator.java @@ -3,7 +3,6 @@ import java.util.Set; import sqlancer.Query; -import sqlancer.citus.gen.CitusCommon; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresDeleteGenerator; diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 99efe2f98..a48ebf1f6 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -380,6 +380,10 @@ public List getDatabaseTables() { return databaseTables; } + public PostgresTable getDatabaseTable(String name) { + return databaseTables.stream().filter(t -> t.getName().equals(name)).findAny().orElse(null); + } + public List getDatabaseTablesRandomSubsetNotEmpty() { return Randomly.nonEmptySubset(databaseTables); } From bcec992ad92e8a60c4d502dab75ff405b42559ae Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 30 Jul 2020 11:31:39 +0300 Subject: [PATCH 0082/1284] Create Citus oracle classes --- src/sqlancer/citus/CitusGlobalState.java | 2 +- src/sqlancer/citus/CitusOptions.java | 6 +- src/sqlancer/citus/CitusProvider.java | 128 +++++++------ src/sqlancer/citus/CitusSchema.java | 19 +- .../citus/gen/CitusAlterTableGenerator.java | 4 +- src/sqlancer/citus/gen/CitusCommon.java | 18 +- .../citus/gen/CitusDeleteGenerator.java | 4 +- .../citus/gen/CitusInsertGenerator.java | 2 +- src/sqlancer/citus/gen/CitusSetGenerator.java | 4 +- .../citus/gen/CitusTableGenerator.java | 4 +- .../citus/gen/CitusUpdateGenerator.java | 2 +- .../citus/gen/CitusViewGenerator.java | 4 +- .../citus/oracle/CitusNoRECOracle.java | 14 ++ .../oracle/tlp/CitusTLPAggregateOracle.java | 27 +++ .../citus/oracle/tlp/CitusTLPBase.java | 181 ++++++++++++++++++ .../oracle/tlp/CitusTLPHavingOracle.java | 27 +++ .../citus/oracle/tlp/CitusTLPWhereOracle.java | 26 +++ src/sqlancer/postgres/PostgresProvider.java | 2 +- .../postgres/oracle/PostgresNoRECOracle.java | 4 +- .../tlp/PostgresTLPAggregateOracle.java | 5 +- .../postgres/oracle/tlp/PostgresTLPBase.java | 26 ++- .../oracle/tlp/PostgresTLPHavingOracle.java | 4 + .../oracle/tlp/PostgresTLPWhereOracle.java | 4 + 23 files changed, 418 insertions(+), 99 deletions(-) create mode 100644 src/sqlancer/citus/oracle/CitusNoRECOracle.java create mode 100644 src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java create mode 100644 src/sqlancer/citus/oracle/tlp/CitusTLPBase.java create mode 100644 src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java create mode 100644 src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java index 669911aba..1eb9427d2 100644 --- a/src/sqlancer/citus/CitusGlobalState.java +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -7,7 +7,7 @@ public class CitusGlobalState extends PostgresGlobalState { private boolean repartition; - + public void setRepartition(boolean repartition) { this.repartition = repartition; } diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index 3abb0cece..ba5daf3a7 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -5,8 +5,8 @@ import sqlancer.postgres.PostgresOptions; public class CitusOptions extends PostgresOptions { - - @Parameter(names= "--repartition") + + @Parameter(names = "--repartition") public boolean repartition = true; - + } diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 68483fcfe..089bf2f9e 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -19,7 +19,21 @@ import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StatementExecutor; -import sqlancer.citus.gen.*; +import sqlancer.citus.gen.CitusAlterTableGenerator; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.citus.gen.CitusDeleteGenerator; +import sqlancer.citus.gen.CitusInsertGenerator; +import sqlancer.citus.gen.CitusSetGenerator; +import sqlancer.citus.gen.CitusTableGenerator; +import sqlancer.citus.gen.CitusUpdateGenerator; +import sqlancer.citus.gen.CitusViewGenerator; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.PostgresProvider; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; import sqlancer.postgres.gen.PostgresAnalyzeGenerator; import sqlancer.postgres.gen.PostgresClusterGenerator; import sqlancer.postgres.gen.PostgresCommentGenerator; @@ -35,22 +49,15 @@ import sqlancer.postgres.gen.PostgresTruncateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; import sqlancer.sqlite3.gen.SQLite3Common; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresOptions; -import sqlancer.postgres.PostgresProvider; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; public class CitusProvider extends PostgresProvider { - private static final Set errors = new HashSet<>(); - - + private static Set errors = new HashSet<>(); + @SuppressWarnings("unchecked") public CitusProvider() { - super((Class)(Object) CitusGlobalState.class, (Class)(Object) CitusOptions.class); + super((Class) (Object) CitusGlobalState.class, + (Class) (Object) CitusOptions.class); CitusCommon.addCitusErrors(errors); } @@ -174,27 +181,28 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { } - private class WorkerNode{ + private class WorkerNode { private final String host; private final int port; - public WorkerNode(String node_host, int node_port) { - this.host = node_host; - this.port = node_port; + WorkerNode(String nodeHost, int nodePort) { + this.host = nodeHost; + this.port = nodePort; } - public String get_host() { + public String getHost() { return this.host; } - public int get_port() { + public int getPort() { return this.port; } } - private static void distributeTable(List columns, String tableName, CitusGlobalState globalState, Connection con) throws SQLException { + private static void distributeTable(List columns, String tableName, CitusGlobalState globalState, + Connection con) throws SQLException { if (columns.size() != 0) { PostgresColumn columnToDistribute = Randomly.fromList(columns); String template = "SELECT create_distributed_table(?, ?);"; @@ -202,12 +210,15 @@ private static void distributeTable(List columns, String tableNa globalState.executeStatement(query, template, tableName, columnToDistribute.getName()); // distribution column cannot take NULL value // TODO: find a way to protect from SQL injection without '' around string input - query = new QueryAdapter("ALTER TABLE " + tableName + " ALTER COLUMN " + columnToDistribute.getName() + " SET NOT NULL;", errors); + query = new QueryAdapter( + "ALTER TABLE " + tableName + " ALTER COLUMN " + columnToDistribute.getName() + " SET NOT NULL;", + errors); globalState.executeStatement(query); } } - private static List getTableConstraints(String tableName, CitusGlobalState globalState, Connection con) throws SQLException { + private static List getTableConstraints(String tableName, CitusGlobalState globalState, Connection con) + throws SQLException { List constraints = new ArrayList<>(); String template = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; QueryAdapter query = new QueryAdapter(template); @@ -218,7 +229,8 @@ private static List getTableConstraints(String tableName, CitusGlobalSta return constraints; } - private static void createDistributedTable(String tableName, CitusGlobalState globalState, Connection con) throws SQLException { + private static void createDistributedTable(String tableName, CitusGlobalState globalState, Connection con) + throws SQLException { List columns = new ArrayList<>(); List tableConstraints = getTableConstraints(tableName, globalState, con); if (tableConstraints.size() == 0) { @@ -229,7 +241,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); // data types money & bit varying have no default operator class for specified partition method - if (! (dataType.equals("money") || dataType.equals("bit varying"))) { + if (!(dataType.equals("money") || dataType.equals("bit varying"))) { PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); columns.add(c); } @@ -244,14 +256,14 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl String dataType = rs.getString("data_type"); String constraintType = rs.getString("constraint_type"); // data types money & bit varying have no default operator class for specified partition method - if (! (dataType.equals("money") || dataType.equals("bit varying"))) { + if (!(dataType.equals("money") || dataType.equals("bit varying"))) { PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); if (columnConstraints.containsKey(c)) { columnConstraints.get(c).add(constraintType); } else { columnConstraints.put(c, new ArrayList<>(Arrays.asList(constraintType))); } - } + } } for (PostgresColumn c : columnConstraints.keySet()) { // TODO: check if table and column constraint sets are equal? but then it's O(N) instead of O(1) @@ -269,17 +281,19 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio // TODO: function reading? add to Postgres implementation? createTables(globalState); for (PostgresTable table : globalState.getSchema().getDatabaseTables()) { - if (table.getTableType() == TableType.TEMPORARY || Randomly.getBooleanWithRatherLowProbability()) { - // create local table - } else if (Randomly.getBooleanWithRatherLowProbability()) { - // create reference table - String template = "SELECT create_reference_table(?);"; - QueryAdapter query = new QueryAdapter(template, errors); - globalState.executeStatement(query, template, table.getName()); - } else { - // create distributed table - createDistributedTable(table.getName(), (CitusGlobalState) globalState, globalState.getConnection()); + if (!(table.getTableType() == TableType.TEMPORARY || Randomly.getBooleanWithRatherLowProbability())) { + if (Randomly.getBooleanWithRatherLowProbability()) { + // create reference table + String template = "SELECT create_reference_table(?);"; + QueryAdapter query = new QueryAdapter(template, errors); + globalState.executeStatement(query, template, table.getName()); + } else { + // create distributed table + createDistributedTable(table.getName(), (CitusGlobalState) globalState, + globalState.getConnection()); + } } + // else: keep local table } ((CitusGlobalState) globalState).updateSchema(); prepareTables(globalState); @@ -291,7 +305,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { - synchronized(CitusProvider.class) { + synchronized (CitusProvider.class) { // returns connection to coordinator node, test database Connection con = super.createDatabase(globalState); String entryDatabaseName = entryPath.substring(1); @@ -302,20 +316,20 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce s.execute("CREATE EXTENSION citus;"); } con.close(); - + // reconnect to coordinator node, entry database globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); con = DriverManager.getConnection("jdbc:" + entryURL, username, password); - + // read info about worker nodes globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); List workerNodes = new ArrayList<>(); try (Statement s = con.createStatement()) { ResultSet rs = s.executeQuery("SELECT * FROM master_get_active_worker_nodes();"); while (rs.next()) { - String node_host = rs.getString("node_name"); - int node_port = rs.getInt("node_port"); - WorkerNode w = new WorkerNode(node_host, node_port); + String nodeHost = rs.getString("node_name"); + int nodePort = rs.getInt("node_port"); + WorkerNode w = new WorkerNode(nodeHost, nodePort); workerNodes.add(w); } } @@ -326,7 +340,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce int hostIndex = entryURL.indexOf(host); String preHost = entryURL.substring(0, hostIndex); String postHost = entryURL.substring(databaseIndex - 1); - String entryWorkerURL = preHost + w.get_host() + ":" + w.get_port() + postHost; + String entryWorkerURL = preHost + w.getHost() + ":" + w.getPort() + postHost; // TODO: better way of logging this globalState.getState().logStatement("\\q"); globalState.getState().logStatement(entryWorkerURL); @@ -347,29 +361,30 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce // connect to worker node, test database int databaseIndexWorker = entryWorkerURL.indexOf(entryPath) + 1; String preDatabaseNameWorker = entryWorkerURL.substring(0, databaseIndexWorker); - String postDatabaseNameWorker = entryWorkerURL.substring(databaseIndexWorker + entryDatabaseName.length()); + String postDatabaseNameWorker = entryWorkerURL + .substring(databaseIndexWorker + entryDatabaseName.length()); String testWorkerURL = preDatabaseNameWorker + databaseName + postDatabaseNameWorker; globalState.getState().logStatement(String.format("\\c %s;", databaseName)); con = DriverManager.getConnection("jdbc:" + testWorkerURL, username, password); - + // add citus extension to worker node, test database globalState.getState().logStatement("CREATE EXTENSION citus;"); try (Statement s = con.createStatement()) { s.execute("CREATE EXTENSION citus;"); } con.close(); - } - + } + // reconnect to coordinator node, test database // TODO: better way of logging this globalState.getState().logStatement("\\q"); globalState.getState().logStatement(testURL); con = DriverManager.getConnection("jdbc:" + testURL, username, password); - + // add worker nodes to coordinator node for test database for (WorkerNode w : workerNodes) { // TODO: protect from sql injection - is it necessary though since these are read from the system? - String addWorkers = "SELECT * from master_add_node('" + w.get_host() + "', " + w.get_port() + ");"; + String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; globalState.getState().logStatement(addWorkers); try (Statement s = con.createStatement()) { s.execute(addWorkers); @@ -387,22 +402,23 @@ protected void createTables(PostgresGlobalState globalState) throws SQLException while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { try { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = CitusTableGenerator.generate(tableName, globalState.getSchema(), - generateOnlyKnown, globalState); + Query createTable = CitusTableGenerator.generate(tableName, globalState.getSchema(), generateOnlyKnown, + globalState); globalState.executeStatement(createTable); } catch (IgnoreMeException e) { } } } - + @Override protected void prepareTables(PostgresGlobalState globalState) throws SQLException { - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), CitusProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + CitusProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); se.executeStatements(); globalState.executeStatement(new QueryAdapter("COMMIT", true)); globalState.executeStatement(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index f8634e5a5..8f2837994 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -24,19 +24,20 @@ public static class CitusTable extends PostgresTable { private Integer colocationId; public CitusTable(String tableName, List columns, List indexes, - TableType tableType, List statistics, boolean isView, boolean isInsertable, PostgresColumn distributionColumn, Integer colocationId) { + TableType tableType, List statistics, boolean isView, boolean isInsertable, + PostgresColumn distributionColumn, Integer colocationId) { super(tableName, columns, indexes, tableType, statistics, isView, isInsertable); this.distributionColumn = distributionColumn; this.colocationId = colocationId; } public CitusTable(PostgresTable table, PostgresColumn distributionColumn, Integer colocationId) { - super(table.getName(), table.getColumns(), table.getIndexes(), table.getTableType(), - table.getStatistics(), table.isView(), table.isInsertable()); + super(table.getName(), table.getColumns(), table.getIndexes(), table.getTableType(), table.getStatistics(), + table.isView(), table.isInsertable()); this.distributionColumn = distributionColumn; this.colocationId = colocationId; } - + public void setDistributionColumn(PostgresColumn distributionColumn) { this.distributionColumn = distributionColumn; } @@ -70,15 +71,17 @@ public static CitusSchema fromConnection(Connection con, String databaseName) th if (rs.wasNull()) { colocationId = null; } - // FIXME: Are the CitusTable-specific features I'm adding going to persist after the function call? + // FIXME: Are the CitusTable-specific features I'm adding going to persist after the function + // call? PostgresTable t = schema.getDatabaseTable(tableName); PostgresColumn distributionColumn = null; if (t == null) { continue; } if (distributionColumnName != null && !distributionColumnName.equals("")) { - distributionColumn = t.getColumns().stream().filter(c -> c.getName(). - equals(distributionColumnName)).collect(Collectors.toList()).get(0); + distributionColumn = t.getColumns().stream() + .filter(c -> c.getName().equals(distributionColumnName)) + .collect(Collectors.toList()).get(0); } CitusTable tCitus = new CitusTable(t, distributionColumn, colocationId); databaseTables.add(tCitus); @@ -91,5 +94,5 @@ public static CitusSchema fromConnection(Connection con, String databaseName) th } throw new AssertionError(ex); } - + } diff --git a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java index a84d359c5..19c8b73a9 100644 --- a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java @@ -8,7 +8,7 @@ import sqlancer.postgres.gen.PostgresAlterTableGenerator; public class CitusAlterTableGenerator extends PostgresAlterTableGenerator { - + public CitusAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, boolean generateOnlyKnown) { super(randomTable, globalState, generateOnlyKnown); @@ -37,4 +37,4 @@ public List getActions(Set errors) { return action; } -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 094ed8fe4..c34b78134 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -16,24 +16,28 @@ public static void addCitusErrors(Set errors) { errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); errors.add("functions used in UPDATE queries on distributed tables must not be VOLATILE"); errors.add("STABLE functions used in UPDATE queries cannot be called with column references"); - errors.add("functions used in the WHERE clause of modification queries on distributed tables must not be VOLATILE"); + errors.add( + "functions used in the WHERE clause of modification queries on distributed tables must not be VOLATILE"); errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); errors.add("cannot execute ALTER TABLE command involving partition column"); errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); errors.add("is not a regular, foreign or partitioned table"); errors.add("must be a distributed table or a reference table"); // ERROR: cannot create foreign key constraint - // Detail: SET NULL or SET DEFAULT is not supported in ON DELETE operation when distribution key is included in the foreign key constraint + // Detail: SET NULL or SET DEFAULT is not supported in ON DELETE operation when distribution key is included in + // the foreign key constraint errors.add("cannot create foreign key constraint"); - + // Citus restrictions on SELECT queries - errors.add("complex joins are only supported when all distributed tables are co-located and joined on their distribution columns"); - errors.add("complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator"); + errors.add( + "complex joins are only supported when all distributed tables are co-located and joined on their distribution columns"); + errors.add( + "complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator"); errors.add("cannot perform distributed planning on this query"); errors.add("cannot pushdown the subquery"); // Check for whether repartition joins are enabled is made during query generation // errors.add("the query contains a join that requires repartitioning"); - + // SQLancer errors errors.add("non-integer constant in GROUP BY"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); @@ -57,5 +61,5 @@ public static void addTableConstraint(StringBuilder sb, PostgresTable table, Pos addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); CitusCommon.addCitusErrors(errors); } - + } diff --git a/src/sqlancer/citus/gen/CitusDeleteGenerator.java b/src/sqlancer/citus/gen/CitusDeleteGenerator.java index 134b7d51d..571afcdc7 100644 --- a/src/sqlancer/citus/gen/CitusDeleteGenerator.java +++ b/src/sqlancer/citus/gen/CitusDeleteGenerator.java @@ -7,7 +7,7 @@ import sqlancer.postgres.gen.PostgresDeleteGenerator; public class CitusDeleteGenerator extends PostgresDeleteGenerator { - + public static Query create(PostgresGlobalState globalState) { Query deleteQuery = PostgresDeleteGenerator.create(globalState); Set errors = (Set) deleteQuery.getExpectedErrors(); @@ -15,4 +15,4 @@ public static Query create(PostgresGlobalState globalState) { return deleteQuery; } -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/gen/CitusInsertGenerator.java b/src/sqlancer/citus/gen/CitusInsertGenerator.java index c649abcfc..e9ecbcb9c 100644 --- a/src/sqlancer/citus/gen/CitusInsertGenerator.java +++ b/src/sqlancer/citus/gen/CitusInsertGenerator.java @@ -15,4 +15,4 @@ public static Query insert(PostgresGlobalState globalState) { return insertQuery; } -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/gen/CitusSetGenerator.java b/src/sqlancer/citus/gen/CitusSetGenerator.java index c82077525..03d8c5663 100644 --- a/src/sqlancer/citus/gen/CitusSetGenerator.java +++ b/src/sqlancer/citus/gen/CitusSetGenerator.java @@ -7,7 +7,7 @@ import sqlancer.postgres.gen.PostgresSetGenerator; public class CitusSetGenerator extends PostgresSetGenerator { - + public static Query create(PostgresGlobalState globalState) { Query setQuery = PostgresSetGenerator.create(globalState); Set errors = (Set) setQuery.getExpectedErrors(); @@ -15,4 +15,4 @@ public static Query create(PostgresGlobalState globalState) { return setQuery; } -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java index ddd657eba..45d80e6aa 100644 --- a/src/sqlancer/citus/gen/CitusTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -11,5 +11,5 @@ public CitusTableGenerator(String tableName, PostgresSchema newSchema, boolean g super(tableName, newSchema, generateOnlyKnown, globalState); CitusCommon.addCitusErrors(errors); } - -} \ No newline at end of file + +} diff --git a/src/sqlancer/citus/gen/CitusUpdateGenerator.java b/src/sqlancer/citus/gen/CitusUpdateGenerator.java index 71513f9f9..bae463267 100644 --- a/src/sqlancer/citus/gen/CitusUpdateGenerator.java +++ b/src/sqlancer/citus/gen/CitusUpdateGenerator.java @@ -15,4 +15,4 @@ public static Query create(PostgresGlobalState globalState) { return updateQuery; } -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/gen/CitusViewGenerator.java b/src/sqlancer/citus/gen/CitusViewGenerator.java index 9fcbbcc95..72af318b2 100644 --- a/src/sqlancer/citus/gen/CitusViewGenerator.java +++ b/src/sqlancer/citus/gen/CitusViewGenerator.java @@ -14,5 +14,5 @@ public static Query create(PostgresGlobalState globalState) { CitusCommon.addCitusErrors(errors); return viewQuery; } - -} \ No newline at end of file + +} diff --git a/src/sqlancer/citus/oracle/CitusNoRECOracle.java b/src/sqlancer/citus/oracle/CitusNoRECOracle.java new file mode 100644 index 000000000..88ec3391a --- /dev/null +++ b/src/sqlancer/citus/oracle/CitusNoRECOracle.java @@ -0,0 +1,14 @@ +package sqlancer.citus.oracle; + +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.oracle.PostgresNoRECOracle; + +public class CitusNoRECOracle extends PostgresNoRECOracle { + + public CitusNoRECOracle(PostgresGlobalState globalState) { + super(globalState); + CitusCommon.addCitusErrors(errors); + } + +} diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java new file mode 100644 index 000000000..d884bbf04 --- /dev/null +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java @@ -0,0 +1,27 @@ +package sqlancer.citus.oracle.tlp; + +import java.sql.SQLException; + +import sqlancer.citus.CitusGlobalState; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPBase; + +public class CitusTLPAggregateOracle extends PostgresTLPAggregateOracle { + + private CitusTLPBase citusTLPBase; + + public CitusTLPAggregateOracle(CitusGlobalState state) { + super(state); + CitusCommon.addCitusErrors(errors); + citusTLPBase = (CitusTLPBase)(PostgresTLPBase) this; + } + + @Override + public void check() throws SQLException { + // FIXME: does this affect "this" too? + citusTLPBase.check(); + aggregateCheck(); + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java new file mode 100644 index 000000000..ddb744a7a --- /dev/null +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -0,0 +1,181 @@ +package sqlancer.citus.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.citus.CitusGlobalState; +import sqlancer.citus.CitusSchema.CitusTable; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresSchema.PostgresTables; +import sqlancer.postgres.ast.PostgresBinaryComparisonOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; +import sqlancer.postgres.ast.PostgresColumnValue; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; +import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.oracle.tlp.PostgresTLPBase; + +public class CitusTLPBase extends PostgresTLPBase { + + HashMap distributedTables; + List referenceTables; + List localTables; + + public CitusTLPBase(CitusGlobalState state) { + super(state); + CitusCommon.addCitusErrors(errors); + } + + @Override + public void check() throws SQLException { + // clear left-over query string from previous test + state.getState().queryString = null; + s = state.getSchema(); + storeCitusTableTypes(); + List tables = null; + List joins = generateJoins(tables); + generateSelectBase(tables, joins); + } + + private List generateJoins(List tables) { + List joins = null; + if (distributedTables.isEmpty() + || (!referenceTables.isEmpty() && !Randomly.getBooleanWithRatherLowProbability())) { + if (!localTables.isEmpty()) { + // joins including only local tables + // supports complex joins + targetTables = new PostgresTables(Randomly.nonEmptySubset(localTables)); + } + if (!referenceTables.isEmpty()) { + // joins including reference tables + // supports complex joins + List targetTableList = new ArrayList<>(referenceTables); + if (!distributedTables.isEmpty()) { + // joins including distributed and reference tables + // supports complex joins + targetTableList.add(Randomly.fromList(new ArrayList<>(distributedTables.keySet()))); + } + targetTables = new PostgresTables(Randomly.nonEmptySubset(targetTableList)); + } + tables = new ArrayList<>(targetTables.getTables()); + joins = getJoinStatements(state, targetTables.getColumns(), tables); + } else { + // joins between distributed tables + // join including distribution columns + // supports complex joins if colocated + tables = Randomly.nonEmptySubset(new ArrayList<>(distributedTables.keySet())); + targetTables = new PostgresTables(tables); + CitusTable fromTable = (CitusTable) Randomly.fromList(tables); + joins = getCitusJoinStatements(state, tables, fromTable); + if (Randomly.getBooleanWithRatherLowProbability() && !localTables.isEmpty()) { + addSubqueryJoinStatements(state, joins, fromTable); + } + } + return joins; + } + + private void storeCitusTableTypes() { + distributedTables = new HashMap<>(); + referenceTables = new ArrayList<>(); + localTables = new ArrayList<>(); + for (PostgresTable table : s.getDatabaseTables()) { + CitusTable citusTable = (CitusTable) table; + Integer colocationId = citusTable.getColocationId(); + PostgresColumn distributionColumn = citusTable.getDistributionColumn(); + if (colocationId != null && distributionColumn != null) { + distributedTables.put(citusTable, colocationId); + } else if (colocationId != null) { + referenceTables.add(citusTable); + } else { + localTables.add(citusTable); + } + } + } + + List getCitusJoinStatements(PostgresGlobalState globalState, List joinTables, + CitusTable fromTable) { + List columns = new ArrayList<>(); + for (PostgresTable t : joinTables) { + columns.add(((CitusTable) t).getDistributionColumn()); + } + List joinStatements = new ArrayList<>(); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); + joinTables.remove(fromTable); + boolean allColocated = true; + for (PostgresTable t : joinTables) { + boolean colocated = (distributedTables.get(fromTable) == distributedTables.get(t)); + allColocated = allColocated && colocated; + } + while (!joinTables.isEmpty()) { + CitusTable table = (CitusTable) Randomly.fromList(joinTables); + // FIXME: can I remove even if reference types don't match due to casting? + joinTables.remove(table); + PostgresExpression joinClause = null; + PostgresExpression equiJoinClause = null; + if (allColocated) { + PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); + PostgresExpression rightExpr = new PostgresColumnValue(table.getDistributionColumn(), null); + equiJoinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, + PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); + } else { + // check if repartition joins are allowed + if (!((CitusGlobalState) globalState).getRepartition()) { + continue; + } + PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); + List candidateRightColumns = table.getColumns().stream() + .filter(c -> c.getType().equals(fromTable.getDistributionColumn().getType())) + .collect(Collectors.toList()); + if (candidateRightColumns.isEmpty()) { + continue; + } + PostgresExpression rightExpr = new PostgresColumnValue(Randomly.fromList(candidateRightColumns), null); + equiJoinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, + PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); + } + if (allColocated && Randomly.getBooleanWithSmallProbability()) { + joinClause = new PostgresBinaryLogicalOperation(equiJoinClause, + gen.generateExpression(PostgresDataType.BOOLEAN), + PostgresBinaryLogicalOperation.BinaryLogicalOperator.AND); + } else { + joinClause = equiJoinClause; + } + PostgresJoinType options = Randomly.fromOptions(PostgresJoinType.INNER, PostgresJoinType.LEFT, + PostgresJoinType.RIGHT, PostgresJoinType.FULL); + if (!allColocated) { + options = PostgresJoinType.INNER; + } + PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + joinTables.add(fromTable); + return joinStatements; + } + + private void addSubqueryJoinStatements(PostgresGlobalState globalState, List joinStatements, + PostgresTable fromTable) { + for (int i = 0; i < Randomly.smallNumber(); i++) { + PostgresTables subqueryTables = new PostgresTables(Randomly.nonEmptySubset(localTables)); + List columns = subqueryTables.getColumns(); + columns.addAll(fromTable.getColumns()); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); + PostgresExpression subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); + PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(subquery, joinClause, options); + joinStatements.add(j); + } + + } + +} diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java new file mode 100644 index 000000000..2d993a1ad --- /dev/null +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java @@ -0,0 +1,27 @@ +package sqlancer.citus.oracle.tlp; + +import java.sql.SQLException; + +import sqlancer.citus.CitusGlobalState; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPBase; + +public class CitusTLPHavingOracle extends PostgresTLPHavingOracle { + + private CitusTLPBase citusTLPBase; + + public CitusTLPHavingOracle(CitusGlobalState state) { + super(state); + CitusCommon.addCitusErrors(errors); + citusTLPBase = (CitusTLPBase)(PostgresTLPBase) this; + } + + @Override + public void check() throws SQLException { + // FIXME: does this affect "this" too? + citusTLPBase.check(); + havingCheck(); + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java new file mode 100644 index 000000000..2cea2b7c3 --- /dev/null +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java @@ -0,0 +1,26 @@ +package sqlancer.citus.oracle.tlp; + +import java.sql.SQLException; + +import sqlancer.citus.CitusGlobalState; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPBase; + +public class CitusTLPWhereOracle extends PostgresTLPWhereOracle { + + private CitusTLPBase citusTLPBase; + + public CitusTLPWhereOracle(CitusGlobalState state) { + super(state); + CitusCommon.addCitusErrors(errors); + citusTLPBase = (CitusTLPBase)(PostgresTLPBase) this; + } + + @Override + public void check() throws SQLException { + // FIXME: does this affect "this" too? + citusTLPBase.check(); + whereCheck(); + } +} \ No newline at end of file diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 07bf53bd1..f05485d25 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -71,7 +71,7 @@ public class PostgresProvider extends ProviderAdapter globalClass, Class optionClass) { super(globalClass, optionClass); } diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 0fc170f26..a12f10811 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -89,7 +89,9 @@ public static List getJoinStatements(PostgresGlobalState globalSta } // JOIN subqueries for (int i = 0; i < Randomly.smallNumber(); i++) { - PostgresSubquery subquery = PostgresTLPBase.createSubquery(globalState, String.format("sub%d", i)); + PostgresTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PostgresSubquery subquery = PostgresTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); PostgresJoinType options = PostgresJoinType.getRandom(); PostgresJoin j = new PostgresJoin(subquery, joinClause, options); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index f7a350284..a5edbd894 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -44,6 +44,10 @@ public PostgresTLPAggregateOracle(PostgresGlobalState state) { @Override public void check() throws SQLException { super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() throws SQLException { PostgresAggregateFunction aggregateFunction = Randomly.fromOptions(PostgresAggregateFunction.MAX, PostgresAggregateFunction.MIN, PostgresAggregateFunction.SUM, PostgresAggregateFunction.BIT_AND, PostgresAggregateFunction.BIT_OR, PostgresAggregateFunction.BOOL_AND, PostgresAggregateFunction.BOOL_OR, @@ -78,7 +82,6 @@ public void check() throws SQLException { secondQueryString); throw new AssertionError(assertionMessage); } - } private String createMetamorphicUnionQuery(PostgresSelect select, PostgresAggregate aggregate, diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 36a353a2e..228251345 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -31,8 +31,8 @@ public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { - PostgresSchema s; - PostgresTables targetTables; + protected PostgresSchema s; + protected PostgresTables targetTables; PostgresExpressionGenerator gen; PostgresSelect select; @@ -46,15 +46,24 @@ public PostgresTLPBase(PostgresGlobalState state) { public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); + List tables = targetTables.getTables(); + List joins = getJoinStatements(state, targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + protected List getJoinStatements(PostgresGlobalState globalState, List columns, + List tables) { + return PostgresNoRECOracle.getJoinStatements(state, targetTables.getColumns(), tables); + // TODO joins + } + + protected void generateSelectBase(List tables, List joins) { + List tableList = tables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); gen = new PostgresExpressionGenerator(state).setColumns(targetTables.getColumns()); initializeTernaryPredicateVariants(); select = new PostgresSelect(); select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List joins = PostgresNoRECOracle.getJoinStatements(state, targetTables.getColumns(), tables); - List tableList = tables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) - .collect(Collectors.toList()); - // TODO joins select.setFromList(tableList); select.setWhereClause(null); select.setJoinClauses(joins); @@ -80,9 +89,8 @@ protected ExpressionGenerator getGen() { return gen; } - public static PostgresSubquery createSubquery(PostgresGlobalState globalState, String name) { + public static PostgresSubquery createSubquery(PostgresGlobalState globalState, String name, PostgresTables tables) { List columns = new ArrayList<>(); - PostgresTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(tables.getColumns()); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { columns.add(gen.generateExpression(0)); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java index 01a0d57f5..3a1275299 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java @@ -22,6 +22,10 @@ public PostgresTLPHavingOracle(PostgresGlobalState state) { @Override public void check() throws SQLException { super.check(); + havingCheck(); + } + + protected void havingCheck() throws SQLException { if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(PostgresDataType.BOOLEAN)); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java index e8b8d3146..b043f6981 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java @@ -19,6 +19,10 @@ public PostgresTLPWhereOracle(PostgresGlobalState state) { @Override public void check() throws SQLException { super.check(); + whereCheck(); + } + + protected void whereCheck() throws SQLException { if (Randomly.getBooleanWithRatherLowProbability()) { select.setOrderByExpressions(gen.generateOrderBy()); } From faaab37dadb007d18a78df3fc1c4f7dbab0854cb Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 4 Aug 2020 12:28:25 +0300 Subject: [PATCH 0083/1284] add to Citus errors --- src/sqlancer/citus/CitusOptions.java | 57 +++++++++++++++++++ src/sqlancer/citus/CitusProvider.java | 12 ++-- src/sqlancer/citus/gen/CitusCommon.java | 6 +- src/sqlancer/citus/gen/CitusSetGenerator.java | 6 +- .../citus/oracle/tlp/CitusTLPBase.java | 2 - 5 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index ba5daf3a7..fc3993f76 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -1,12 +1,69 @@ package sqlancer.citus; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import com.beust.jcommander.Parameter; +import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; +import sqlancer.CompositeTestOracle; +import sqlancer.TestOracle; +import sqlancer.citus.oracle.CitusNoRECOracle; +import sqlancer.citus.oracle.tlp.CitusTLPAggregateOracle; +import sqlancer.citus.oracle.tlp.CitusTLPHavingOracle; +import sqlancer.citus.oracle.tlp.CitusTLPWhereOracle; public class CitusOptions extends PostgresOptions { @Parameter(names = "--repartition") public boolean repartition = true; + + /* @Override + @Parameter(names = "--citusOracle") + public List oracle = Arrays.asList(CitusOracle.QUERY_PARTITIONING); + + @Override */ + public enum PostgresOracle { + NOREC { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; + return new CitusNoRECOracle(citusGlobalState); + } + }, + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + }, + HAVING { + + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; + return new CitusTLPHavingOracle(citusGlobalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; + List oracles = new ArrayList<>(); + oracles.add(new CitusTLPWhereOracle(citusGlobalState)); + oracles.add(new CitusTLPHavingOracle(citusGlobalState)); + oracles.add(new CitusTLPAggregateOracle(citusGlobalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + + public abstract TestOracle create(PostgresGlobalState globalState) throws SQLException; + + } } diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 089bf2f9e..eba053c61 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -207,7 +207,7 @@ private static void distributeTable(List columns, String tableNa PostgresColumn columnToDistribute = Randomly.fromList(columns); String template = "SELECT create_distributed_table(?, ?);"; QueryAdapter query = new QueryAdapter(template, errors); - globalState.executeStatement(query, template, tableName, columnToDistribute.getName()); + globalState.executeStatement(query, tableName, columnToDistribute.getName()); // distribution column cannot take NULL value // TODO: find a way to protect from SQL injection without '' around string input query = new QueryAdapter( @@ -222,7 +222,7 @@ private static List getTableConstraints(String tableName, CitusGlobalSta List constraints = new ArrayList<>(); String template = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; QueryAdapter query = new QueryAdapter(template); - ResultSet rs = query.executeAndGet(globalState, template, tableName); + ResultSet rs = query.executeAndGet(globalState, tableName); while (rs.next()) { constraints.add(rs.getString("constraint_type")); } @@ -236,7 +236,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl if (tableConstraints.size() == 0) { String template = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;"; QueryAdapter query = new QueryAdapter(template); - ResultSet rs = query.executeAndGet(globalState, template, tableName); + ResultSet rs = query.executeAndGet(globalState, tableName); while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); @@ -250,7 +250,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl HashMap> columnConstraints = new HashMap<>(); String template = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;"; QueryAdapter query = new QueryAdapter(template); - ResultSet rs = query.executeAndGet(globalState, template, tableName); + ResultSet rs = query.executeAndGet(globalState, tableName); while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); @@ -286,7 +286,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio // create reference table String template = "SELECT create_reference_table(?);"; QueryAdapter query = new QueryAdapter(template, errors); - globalState.executeStatement(query, template, table.getName()); + globalState.executeStatement(query, table.getName()); } else { // create distributed table createDistributedTable(table.getName(), (CitusGlobalState) globalState, @@ -399,7 +399,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce @Override protected void createTables(PostgresGlobalState globalState) throws SQLException { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(4, 5, 6)) { try { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); Query createTable = CitusTableGenerator.generate(tableName, globalState.getSchema(), generateOnlyKnown, diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index c34b78134..4d5eae147 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -1,5 +1,6 @@ package sqlancer.citus.gen; +import java.util.Collection; import java.util.Set; import sqlancer.Randomly; @@ -9,7 +10,7 @@ public class CitusCommon extends PostgresCommon { - public static void addCitusErrors(Set errors) { + public static void addCitusErrors(Collection errors) { errors.add("recursive CTEs are not supported in distributed queries"); errors.add("could not run distributed query with GROUPING SETS, CUBE, or ROLLUP"); errors.add("Subqueries in HAVING cannot refer to outer query"); @@ -23,6 +24,9 @@ public static void addCitusErrors(Set errors) { errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); errors.add("is not a regular, foreign or partitioned table"); errors.add("must be a distributed table or a reference table"); + errors.add("creating unique indexes on non-partition columns is currently unsupported"); + errors.add("modifying the partition value of rows is not allowed"); + errors.add("creating unique indexes on non-partition columns is currently unsupported"); // ERROR: cannot create foreign key constraint // Detail: SET NULL or SET DEFAULT is not supported in ON DELETE operation when distribution key is included in // the foreign key constraint diff --git a/src/sqlancer/citus/gen/CitusSetGenerator.java b/src/sqlancer/citus/gen/CitusSetGenerator.java index 03d8c5663..d324713ee 100644 --- a/src/sqlancer/citus/gen/CitusSetGenerator.java +++ b/src/sqlancer/citus/gen/CitusSetGenerator.java @@ -1,6 +1,6 @@ package sqlancer.citus.gen; -import java.util.Set; +import java.util.Collection; import sqlancer.Query; import sqlancer.postgres.PostgresGlobalState; @@ -10,9 +10,9 @@ public class CitusSetGenerator extends PostgresSetGenerator { public static Query create(PostgresGlobalState globalState) { Query setQuery = PostgresSetGenerator.create(globalState); - Set errors = (Set) setQuery.getExpectedErrors(); + Collection errors = setQuery.getExpectedErrors(); CitusCommon.addCitusErrors(errors); return setQuery; } -} +} \ No newline at end of file diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java index ddb744a7a..3ef24f115 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -38,8 +38,6 @@ public CitusTLPBase(CitusGlobalState state) { @Override public void check() throws SQLException { - // clear left-over query string from previous test - state.getState().queryString = null; s = state.getSchema(); storeCitusTableTypes(); List tables = null; From 14500499dd17873b1682f76fb817cbf656f26d57 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Thu, 6 Aug 2020 12:19:51 +0300 Subject: [PATCH 0084/1284] direct oracles to Citus oracles --- src/sqlancer/citus/CitusOptions.java | 4 +-- src/sqlancer/citus/CitusProvider.java | 20 +++++++++++++-- .../citus/gen/CitusAlterTableGenerator.java | 9 +++++++ .../citus/gen/CitusIndexGenerator.java | 18 +++++++++++++ .../oracle/tlp/CitusTLPAggregateOracle.java | 7 ++++-- .../citus/oracle/tlp/CitusTLPBase.java | 25 ++++++++++++++++--- .../oracle/tlp/CitusTLPHavingOracle.java | 7 ++++-- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 7 ++++-- .../gen/PostgresAlterTableGenerator.java | 2 +- .../postgres/oracle/tlp/PostgresTLPBase.java | 4 +-- 10 files changed, 86 insertions(+), 17 deletions(-) create mode 100644 src/sqlancer/citus/gen/CitusIndexGenerator.java diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index fc3993f76..458001107 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -22,11 +22,9 @@ public class CitusOptions extends PostgresOptions { @Parameter(names = "--repartition") public boolean repartition = true; - /* @Override @Parameter(names = "--citusOracle") - public List oracle = Arrays.asList(CitusOracle.QUERY_PARTITIONING); + public List citusOracle = Arrays.asList(PostgresOracle.QUERY_PARTITIONING); - @Override */ public enum PostgresOracle { NOREC { @Override diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index eba053c61..d672a35fd 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -11,17 +11,21 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import sqlancer.AbstractAction; +import sqlancer.CompositeTestOracle; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StatementExecutor; +import sqlancer.TestOracle; import sqlancer.citus.gen.CitusAlterTableGenerator; import sqlancer.citus.gen.CitusCommon; import sqlancer.citus.gen.CitusDeleteGenerator; +import sqlancer.citus.gen.CitusIndexGenerator; import sqlancer.citus.gen.CitusInsertGenerator; import sqlancer.citus.gen.CitusSetGenerator; import sqlancer.citus.gen.CitusTableGenerator; @@ -39,7 +43,6 @@ import sqlancer.postgres.gen.PostgresCommentGenerator; import sqlancer.postgres.gen.PostgresDiscardGenerator; import sqlancer.postgres.gen.PostgresDropIndexGenerator; -import sqlancer.postgres.gen.PostgresIndexGenerator; import sqlancer.postgres.gen.PostgresNotifyGenerator; import sqlancer.postgres.gen.PostgresQueryCatalogGenerator; import sqlancer.postgres.gen.PostgresReindexGenerator; @@ -88,7 +91,7 @@ public enum Action implements AbstractAction { VACUUM(PostgresVacuumGenerator::create), // REINDEX(PostgresReindexGenerator::create), // SET(CitusSetGenerator::create), // - CREATE_INDEX(PostgresIndexGenerator::generate), // + CREATE_INDEX(CitusIndexGenerator::generate), // SET_CONSTRAINTS((g) -> { StringBuilder sb = new StringBuilder(); sb.append("SET CONSTRAINTS ALL "); @@ -303,6 +306,18 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio } } + @Override + protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { + List oracles = ((CitusOptions) globalState.getDmbsSpecificOptions()).citusOracle.stream().map(o -> { + try { + return o.create(globalState); + } catch (SQLException e1) { + throw new AssertionError(e1); + } + }).collect(Collectors.toList()); + return new CompositeTestOracle(oracles, globalState); + } + @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { synchronized (CitusProvider.class) { @@ -393,6 +408,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce con.close(); // reconnect to coordinator node, test database con = DriverManager.getConnection("jdbc:" + testURL, username, password); + ((CitusGlobalState) globalState).setRepartition(((CitusOptions) globalState.getDmbsSpecificOptions()).repartition); return con; } } diff --git a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java index 19c8b73a9..67aba77db 100644 --- a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java @@ -3,6 +3,8 @@ import java.util.List; import java.util.Set; +import sqlancer.Query; +import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.gen.PostgresAlterTableGenerator; @@ -14,6 +16,10 @@ public CitusAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState g super(randomTable, globalState, generateOnlyKnown); } + public static Query create(PostgresTable randomTable, PostgresGlobalState globalState, boolean generateOnlyKnown) { + return new CitusAlterTableGenerator(randomTable, globalState, generateOnlyKnown).generate(); + } + @Override public List getActions(Set errors) { List action = super.getActions(errors); @@ -34,6 +40,9 @@ public List getActions(Set errors) { action.remove(Action.NOT_OF); action.remove(Action.OWNER_TO); action.remove(Action.REPLICA_IDENTITY); + if (action.isEmpty()) { + action.addAll(Randomly.nonEmptySubset(Action.ALTER_TABLE_DROP_COLUMN, Action.ALTER_COLUMN_TYPE, Action.ALTER_COLUMN_SET_DROP_DEFAULT, Action.ALTER_COLUMN_SET_DROP_NULL, Action.ADD_TABLE_CONSTRAINT, Action.ADD_TABLE_CONSTRAINT_USING_INDEX, Action.VALIDATE_CONSTRAINT)); + } return action; } diff --git a/src/sqlancer/citus/gen/CitusIndexGenerator.java b/src/sqlancer/citus/gen/CitusIndexGenerator.java new file mode 100644 index 000000000..110dae037 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusIndexGenerator.java @@ -0,0 +1,18 @@ +package sqlancer.citus.gen; + +import java.util.HashSet; + +import sqlancer.Query; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresInsertGenerator; + +public class CitusIndexGenerator extends PostgresInsertGenerator { + + public static Query generate(PostgresGlobalState globalState) { + Query createIndexQuery = PostgresInsertGenerator.insert(globalState); + HashSet errors = (HashSet) createIndexQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return createIndexQuery; + } + +} \ No newline at end of file diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java index d884bbf04..6a152cbfe 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java @@ -14,13 +14,16 @@ public class CitusTLPAggregateOracle extends PostgresTLPAggregateOracle { public CitusTLPAggregateOracle(CitusGlobalState state) { super(state); CitusCommon.addCitusErrors(errors); - citusTLPBase = (CitusTLPBase)(PostgresTLPBase) this; + citusTLPBase = new CitusTLPBase(state); } @Override public void check() throws SQLException { - // FIXME: does this affect "this" too? citusTLPBase.check(); + s = citusTLPBase.getSchema(); + targetTables = citusTLPBase.getTargetTables(); + gen = citusTLPBase.getGenerator(); + select = citusTLPBase.getSelect(); aggregateCheck(); } diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java index 3ef24f115..867c0efb1 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -11,6 +12,7 @@ import sqlancer.citus.CitusSchema.CitusTable; import sqlancer.citus.gen.CitusCommon; import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -20,6 +22,7 @@ import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.gen.PostgresExpressionGenerator; @@ -36,11 +39,27 @@ public CitusTLPBase(CitusGlobalState state) { CitusCommon.addCitusErrors(errors); } + public PostgresSchema getSchema() { + return s; + } + + public PostgresTables getTargetTables() { + return targetTables; + } + + public PostgresExpressionGenerator getGenerator() { + return gen; + } + + public PostgresSelect getSelect() { + return select; + } + @Override public void check() throws SQLException { s = state.getSchema(); storeCitusTableTypes(); - List tables = null; + List tables = new ArrayList<>(); List joins = generateJoins(tables); generateSelectBase(tables, joins); } @@ -65,13 +84,13 @@ private List generateJoins(List tables) { } targetTables = new PostgresTables(Randomly.nonEmptySubset(targetTableList)); } - tables = new ArrayList<>(targetTables.getTables()); + tables.addAll(targetTables.getTables()); joins = getJoinStatements(state, targetTables.getColumns(), tables); } else { // joins between distributed tables // join including distribution columns // supports complex joins if colocated - tables = Randomly.nonEmptySubset(new ArrayList<>(distributedTables.keySet())); + tables.addAll(Randomly.nonEmptySubset(new ArrayList<>(distributedTables.keySet()))); targetTables = new PostgresTables(tables); CitusTable fromTable = (CitusTable) Randomly.fromList(tables); joins = getCitusJoinStatements(state, tables, fromTable); diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java index 2d993a1ad..f6be6e001 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java @@ -14,13 +14,16 @@ public class CitusTLPHavingOracle extends PostgresTLPHavingOracle { public CitusTLPHavingOracle(CitusGlobalState state) { super(state); CitusCommon.addCitusErrors(errors); - citusTLPBase = (CitusTLPBase)(PostgresTLPBase) this; + citusTLPBase = new CitusTLPBase(state); } @Override public void check() throws SQLException { - // FIXME: does this affect "this" too? citusTLPBase.check(); + s = citusTLPBase.getSchema(); + targetTables = citusTLPBase.getTargetTables(); + gen = citusTLPBase.getGenerator(); + select = citusTLPBase.getSelect(); havingCheck(); } diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java index 2cea2b7c3..ee3f46cb1 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java @@ -14,13 +14,16 @@ public class CitusTLPWhereOracle extends PostgresTLPWhereOracle { public CitusTLPWhereOracle(CitusGlobalState state) { super(state); CitusCommon.addCitusErrors(errors); - citusTLPBase = (CitusTLPBase)(PostgresTLPBase) this; + citusTLPBase = new CitusTLPBase(state); } @Override public void check() throws SQLException { - // FIXME: does this affect "this" too? citusTLPBase.check(); + s = citusTLPBase.getSchema(); + targetTables = citusTLPBase.getTargetTables(); + gen = citusTLPBase.getGenerator(); + select = citusTLPBase.getSelect(); whereCheck(); } } \ No newline at end of file diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index cac22d5c2..4bf842524 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -278,7 +278,7 @@ public Query generate() { break; case ADD_TABLE_CONSTRAINT_USING_INDEX: sb.append("ADD "); - // sb.append("CONSTRAINT 'asdf' "); + sb.append("CONSTRAINT '" + r.getChar() +"' "); sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); errors.add("not valid"); sb.append(" USING INDEX "); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 228251345..5cf317930 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -33,8 +33,8 @@ public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase Date: Fri, 7 Aug 2020 17:22:06 +0300 Subject: [PATCH 0085/1284] read function volatilities & specify allowed functions --- src/sqlancer/citus/CitusGlobalState.java | 2 +- .../oracle/tlp/CitusTLPAggregateOracle.java | 7 +++++- .../citus/oracle/tlp/CitusTLPBase.java | 19 +++++++++++---- .../oracle/tlp/CitusTLPHavingOracle.java | 7 +++++- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 7 +++++- .../postgres/PostgresGlobalState.java | 24 +++++++++++++++++++ src/sqlancer/postgres/PostgresProvider.java | 11 +++++++++ .../gen/PostgresExpressionGenerator.java | 11 +++++++++ 8 files changed, 80 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java index 1eb9427d2..669911aba 100644 --- a/src/sqlancer/citus/CitusGlobalState.java +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -7,7 +7,7 @@ public class CitusGlobalState extends PostgresGlobalState { private boolean repartition; - + public void setRepartition(boolean repartition) { this.repartition = repartition; } diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java index 6a152cbfe..6c0c4c6d5 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java @@ -1,11 +1,11 @@ package sqlancer.citus.oracle.tlp; import java.sql.SQLException; +import java.util.Arrays; import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPBase; public class CitusTLPAggregateOracle extends PostgresTLPAggregateOracle { @@ -19,12 +19,17 @@ public CitusTLPAggregateOracle(CitusGlobalState state) { @Override public void check() throws SQLException { + state.setAllowedFunctionTypes(Arrays.asList('i')); citusTLPBase.check(); s = citusTLPBase.getSchema(); targetTables = citusTLPBase.getTargetTables(); gen = citusTLPBase.getGenerator(); select = citusTLPBase.getSelect(); + predicate = citusTLPBase.getPredicate(); + negatedPredicate = citusTLPBase.getNegatedPredicate(); + isNullPredicate = citusTLPBase.getIsNullPredicate(); aggregateCheck(); + state.setDefaultAllowedFunctionTypes(); } } \ No newline at end of file diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java index 867c0efb1..a16ef96e2 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -55,6 +54,18 @@ public PostgresSelect getSelect() { return select; } + public PostgresExpression getPredicate() { + return predicate; + } + + public PostgresExpression getNegatedPredicate() { + return negatedPredicate; + } + + public PostgresExpression getIsNullPredicate() { + return isNullPredicate; + } + @Override public void check() throws SQLException { s = state.getSchema(); @@ -93,7 +104,7 @@ private List generateJoins(List tables) { tables.addAll(Randomly.nonEmptySubset(new ArrayList<>(distributedTables.keySet()))); targetTables = new PostgresTables(tables); CitusTable fromTable = (CitusTable) Randomly.fromList(tables); - joins = getCitusJoinStatements(state, tables, fromTable); + joins = getCitusJoinStatements((CitusGlobalState) state, tables, fromTable); if (Randomly.getBooleanWithRatherLowProbability() && !localTables.isEmpty()) { addSubqueryJoinStatements(state, joins, fromTable); } @@ -119,7 +130,7 @@ private void storeCitusTableTypes() { } } - List getCitusJoinStatements(PostgresGlobalState globalState, List joinTables, + List getCitusJoinStatements(CitusGlobalState globalState, List joinTables, CitusTable fromTable) { List columns = new ArrayList<>(); for (PostgresTable t : joinTables) { @@ -146,7 +157,7 @@ List getCitusJoinStatements(PostgresGlobalState globalState, List< PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); } else { // check if repartition joins are allowed - if (!((CitusGlobalState) globalState).getRepartition()) { + if (! globalState.getRepartition()) { continue; } PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java index f6be6e001..8e925cd62 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java @@ -1,11 +1,11 @@ package sqlancer.citus.oracle.tlp; import java.sql.SQLException; +import java.util.Arrays; import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPBase; public class CitusTLPHavingOracle extends PostgresTLPHavingOracle { @@ -19,12 +19,17 @@ public CitusTLPHavingOracle(CitusGlobalState state) { @Override public void check() throws SQLException { + state.setAllowedFunctionTypes(Arrays.asList('i')); citusTLPBase.check(); s = citusTLPBase.getSchema(); targetTables = citusTLPBase.getTargetTables(); gen = citusTLPBase.getGenerator(); select = citusTLPBase.getSelect(); + predicate = citusTLPBase.getPredicate(); + negatedPredicate = citusTLPBase.getNegatedPredicate(); + isNullPredicate = citusTLPBase.getIsNullPredicate(); havingCheck(); + state.setDefaultAllowedFunctionTypes(); } } \ No newline at end of file diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java index ee3f46cb1..195229be2 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java @@ -1,11 +1,11 @@ package sqlancer.citus.oracle.tlp; import java.sql.SQLException; +import java.util.Arrays; import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPBase; public class CitusTLPWhereOracle extends PostgresTLPWhereOracle { @@ -19,11 +19,16 @@ public CitusTLPWhereOracle(CitusGlobalState state) { @Override public void check() throws SQLException { + state.setAllowedFunctionTypes(Arrays.asList('i')); citusTLPBase.check(); s = citusTLPBase.getSchema(); targetTables = citusTLPBase.getTargetTables(); gen = citusTLPBase.getGenerator(); select = citusTLPBase.getSelect(); + predicate = citusTLPBase.getPredicate(); + negatedPredicate = citusTLPBase.getNegatedPredicate(); + isNullPredicate = citusTLPBase.getIsNullPredicate(); whereCheck(); + state.setDefaultAllowedFunctionTypes(); } } \ No newline at end of file diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 58e5be0a4..0943f245e 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -5,6 +5,8 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import sqlancer.GlobalState; @@ -15,6 +17,8 @@ public class PostgresGlobalState extends GlobalState operators; private List collates; private List opClasses; + private HashMap functionsAndTypes = new HashMap<>(); + private List allowedFunctionTypes = Arrays.asList('s', 'v', 'i'); @Override public void setConnection(Connection con) { @@ -94,4 +98,24 @@ protected void updateSchema() throws SQLException { setSchema(PostgresSchema.fromConnection(getConnection(), getDatabaseName())); } + public void addFunctionAndType(String functionName, Character functionType) { + this.functionsAndTypes.put(functionName, functionType); + } + + public HashMap getFunctionsAndTypes() { + return this.functionsAndTypes; + } + + public void setAllowedFunctionTypes(List types) { + this.allowedFunctionTypes = types; + } + + public void setDefaultAllowedFunctionTypes() { + this.allowedFunctionTypes = Arrays.asList('s', 'v', 'i'); + } + + public List getAllowedFunctionTypes() { + return this.allowedFunctionTypes; + } + } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index f05485d25..51ca63c03 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -6,6 +6,7 @@ import java.net.URISyntaxException; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; @@ -274,6 +275,16 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce return con; } + protected void readFunctions(PostgresGlobalState globalState) throws SQLException { + QueryAdapter query = new QueryAdapter("SELECT proname, provolatile FROM pg_proc;"); + ResultSet rs = query.executeAndGet(globalState); + while (rs.next()) { + String functionName = rs.getString("proname"); + Character functionType = rs.getString("provolatile").charAt(0); + globalState.addFunctionAndType(functionName, functionType); + } + } + protected void createTables(PostgresGlobalState globalState) throws SQLException { while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { try { diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index d4584f10b..79f7e99ab 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -66,10 +67,16 @@ public class PostgresExpressionGenerator implements ExpressionGenerator functionsAndTypes; + + private List allowedFunctionTypes; + public PostgresExpressionGenerator(PostgresGlobalState globalState) { this.r = globalState.getRandomly(); this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); this.globalState = globalState; + this.functionsAndTypes = globalState.getFunctionsAndTypes(); + this.allowedFunctionTypes = globalState.getAllowedFunctionTypes(); } public PostgresExpressionGenerator setColumns(List columns) { @@ -112,6 +119,8 @@ private enum BooleanExpression { private PostgresExpression generateFunctionWithUnknownResult(int depth, PostgresDataType type) { List supportedFunctions = PostgresFunctionWithUnknownResult .getSupportedFunctions(type); + // filters functions by allowed type (stable 's', immutable 'i', volatile 'v') + supportedFunctions = supportedFunctions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))).collect(Collectors.toList()); if (supportedFunctions.isEmpty()) { throw new IgnoreMeException(); } @@ -122,6 +131,8 @@ private PostgresExpression generateFunctionWithUnknownResult(int depth, Postgres private PostgresExpression generateFunctionWithKnownResult(int depth, PostgresDataType type) { List functions = Stream.of(PostgresFunction.PostgresFunctionWithResult.values()) .filter(f -> f.supportsReturnType(type)).collect(Collectors.toList()); + // filters functions by allowed type (stable 's', immutable 'i', volatile 'v') + functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))).collect(Collectors.toList()); if (functions.isEmpty()) { throw new IgnoreMeException(); } From aa125c10f3378f7e5921d0f96d8a47da31129b58 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Sat, 8 Aug 2020 13:02:11 +0300 Subject: [PATCH 0086/1284] add to Citus errors --- src/sqlancer/citus/CitusProvider.java | 41 ++++++++----------- src/sqlancer/citus/gen/CitusCommon.java | 2 + src/sqlancer/postgres/PostgresProvider.java | 6 +-- .../gen/PostgresAlterTableGenerator.java | 7 +++- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index d672a35fd..4ed48a4fb 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -209,8 +209,9 @@ private static void distributeTable(List columns, String tableNa if (columns.size() != 0) { PostgresColumn columnToDistribute = Randomly.fromList(columns); String template = "SELECT create_distributed_table(?, ?);"; - QueryAdapter query = new QueryAdapter(template, errors); - globalState.executeStatement(query, tableName, columnToDistribute.getName()); + String filled = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + "');"; + QueryAdapter query = new QueryAdapter(filled, errors); + globalState.executeStatement(query, template, tableName, columnToDistribute.getName()); // distribution column cannot take NULL value // TODO: find a way to protect from SQL injection without '' around string input query = new QueryAdapter( @@ -224,8 +225,9 @@ private static List getTableConstraints(String tableName, CitusGlobalSta throws SQLException { List constraints = new ArrayList<>(); String template = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; - QueryAdapter query = new QueryAdapter(template); - ResultSet rs = query.executeAndGet(globalState, tableName); + String filled = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + tableName + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; + QueryAdapter query = new QueryAdapter(filled); + ResultSet rs = query.executeAndGet(globalState, template, tableName); while (rs.next()) { constraints.add(rs.getString("constraint_type")); } @@ -238,8 +240,9 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl List tableConstraints = getTableConstraints(tableName, globalState, con); if (tableConstraints.size() == 0) { String template = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;"; - QueryAdapter query = new QueryAdapter(template); - ResultSet rs = query.executeAndGet(globalState, tableName); + String filled = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + tableName + "';"; + QueryAdapter query = new QueryAdapter(filled); + ResultSet rs = query.executeAndGet(globalState, template, tableName); while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); @@ -252,8 +255,9 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl } else { HashMap> columnConstraints = new HashMap<>(); String template = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;"; - QueryAdapter query = new QueryAdapter(template); - ResultSet rs = query.executeAndGet(globalState, tableName); + String filled = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + tableName + "';"; + QueryAdapter query = new QueryAdapter(filled); + ResultSet rs = query.executeAndGet(globalState, template, tableName); while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); @@ -282,14 +286,15 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl @Override public void generateDatabase(PostgresGlobalState globalState) throws SQLException { // TODO: function reading? add to Postgres implementation? - createTables(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); for (PostgresTable table : globalState.getSchema().getDatabaseTables()) { if (!(table.getTableType() == TableType.TEMPORARY || Randomly.getBooleanWithRatherLowProbability())) { if (Randomly.getBooleanWithRatherLowProbability()) { // create reference table String template = "SELECT create_reference_table(?);"; - QueryAdapter query = new QueryAdapter(template, errors); - globalState.executeStatement(query, table.getName()); + String filled = "SELECT create_reference_table('" + table.getName() + "');"; + QueryAdapter query = new QueryAdapter(filled, errors); + globalState.executeStatement(query, template, table.getName()); } else { // create distributed table createDistributedTable(table.getName(), (CitusGlobalState) globalState, @@ -413,20 +418,6 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce } } - @Override - protected void createTables(PostgresGlobalState globalState) throws SQLException { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(4, 5, 6)) { - try { - String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = CitusTableGenerator.generate(tableName, globalState.getSchema(), generateOnlyKnown, - globalState); - globalState.executeStatement(createTable); - } catch (IgnoreMeException e) { - - } - } - } - @Override protected void prepareTables(PostgresGlobalState globalState) throws SQLException { StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 4d5eae147..2b79c01b9 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -54,6 +54,8 @@ public static void addCitusErrors(Collection errors) { errors.add("failed to evaluate partition key in insert"); errors.add("cannot perform an INSERT without a partition column value"); errors.add("cannot perform an INSERT with NULL in the partition column"); + errors.add("ERROR: LIMIT must not be negative"); + errors.add("value too long for type"); // current errors to be removed once upgraded to PostgreSQL 13? errors.add("unrecognized configuration parameter \"enable_hashagg_disk\""); diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 51ca63c03..9b0ee7e55 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -199,7 +199,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { @Override public void generateDatabase(PostgresGlobalState globalState) throws SQLException { - createTables(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); prepareTables(globalState); } @@ -285,8 +285,8 @@ protected void readFunctions(PostgresGlobalState globalState) throws SQLExceptio } } - protected void createTables(PostgresGlobalState globalState) throws SQLException { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.fromOptions(1, 2)) { + protected void createTables(PostgresGlobalState globalState, int numTables) throws SQLException { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { try { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); Query createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 4bf842524..9946de609 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -249,6 +249,7 @@ public Query generate() { break; case ADD_TABLE_CONSTRAINT: sb.append("ADD "); + sb.append("CONSTRAINT '" + r.getChar() +"' "); PostgresCommon.addTableConstraint(sb, randomTable, globalState, errors); errors.add("multiple primary keys for table"); errors.add("could not create unique index"); @@ -281,8 +282,10 @@ public Query generate() { sb.append("CONSTRAINT '" + r.getChar() +"' "); sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); errors.add("not valid"); - sb.append(" USING INDEX "); - sb.append(randomTable.getRandomIndex().getIndexName()); + if (randomTable.hasIndexes()) { + sb.append(" USING INDEX "); + sb.append(randomTable.getRandomIndex().getIndexName()); + } errors.add("is not a unique index"); errors.add("is already associated with a constraint"); errors.add("Cannot create a primary key or unique constraint using such an index"); From 5ff26145794aca5758307ca0d83fffecc165b634 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 10 Aug 2020 13:19:01 +0300 Subject: [PATCH 0087/1284] Alter table with constraint name --- src/sqlancer/Randomly.java | 9 +++++ src/sqlancer/citus/CitusGlobalState.java | 2 +- src/sqlancer/citus/CitusOptions.java | 8 ++-- src/sqlancer/citus/CitusProvider.java | 37 +++++++++++-------- .../citus/gen/CitusAlterTableGenerator.java | 4 +- src/sqlancer/citus/gen/CitusCommon.java | 2 + .../citus/gen/CitusIndexGenerator.java | 4 +- src/sqlancer/citus/gen/CitusSetGenerator.java | 2 +- .../oracle/tlp/CitusTLPAggregateOracle.java | 2 +- .../citus/oracle/tlp/CitusTLPBase.java | 4 +- .../oracle/tlp/CitusTLPHavingOracle.java | 4 +- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 2 +- src/sqlancer/postgres/PostgresProvider.java | 8 ++-- .../gen/PostgresAlterTableGenerator.java | 13 ++++--- .../gen/PostgresExpressionGenerator.java | 7 +++- .../postgres/oracle/tlp/PostgresTLPBase.java | 2 +- 16 files changed, 65 insertions(+), 45 deletions(-) diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 1488ed43f..75ae7b600 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -403,6 +403,15 @@ public String getChar() { } } + public String getAlphabeticChar() { + while (true) { + String s = getChar(); + if (Character.isAlphabetic(s.charAt(0))) { + return s; + } + } + } + // see https://stackoverflow.com/a/2546158 // uniformity does not seem to be important for us // SQLancer previously used ThreadLocalRandom.current().nextLong(lower, upper) diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java index 669911aba..1eb9427d2 100644 --- a/src/sqlancer/citus/CitusGlobalState.java +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -7,7 +7,7 @@ public class CitusGlobalState extends PostgresGlobalState { private boolean repartition; - + public void setRepartition(boolean repartition) { this.repartition = repartition; } diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index 458001107..6d3076caf 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -7,21 +7,21 @@ import com.beust.jcommander.Parameter; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresOptions; -import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; import sqlancer.CompositeTestOracle; import sqlancer.TestOracle; import sqlancer.citus.oracle.CitusNoRECOracle; import sqlancer.citus.oracle.tlp.CitusTLPAggregateOracle; import sqlancer.citus.oracle.tlp.CitusTLPHavingOracle; import sqlancer.citus.oracle.tlp.CitusTLPWhereOracle; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; public class CitusOptions extends PostgresOptions { @Parameter(names = "--repartition") public boolean repartition = true; - + @Parameter(names = "--citusOracle") public List citusOracle = Arrays.asList(PostgresOracle.QUERY_PARTITIONING); diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 4ed48a4fb..19dda5fcd 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -20,6 +20,7 @@ import sqlancer.QueryAdapter; import sqlancer.QueryProvider; import sqlancer.Randomly; +import sqlancer.SQLancerResultSet; import sqlancer.StatementExecutor; import sqlancer.TestOracle; import sqlancer.citus.gen.CitusAlterTableGenerator; @@ -28,7 +29,6 @@ import sqlancer.citus.gen.CitusIndexGenerator; import sqlancer.citus.gen.CitusInsertGenerator; import sqlancer.citus.gen.CitusSetGenerator; -import sqlancer.citus.gen.CitusTableGenerator; import sqlancer.citus.gen.CitusUpdateGenerator; import sqlancer.citus.gen.CitusViewGenerator; import sqlancer.postgres.PostgresGlobalState; @@ -51,7 +51,6 @@ import sqlancer.postgres.gen.PostgresTransactionGenerator; import sqlancer.postgres.gen.PostgresTruncateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; -import sqlancer.sqlite3.gen.SQLite3Common; public class CitusProvider extends PostgresProvider { @@ -209,7 +208,8 @@ private static void distributeTable(List columns, String tableNa if (columns.size() != 0) { PostgresColumn columnToDistribute = Randomly.fromList(columns); String template = "SELECT create_distributed_table(?, ?);"; - String filled = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + "');"; + String filled = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + + "');"; QueryAdapter query = new QueryAdapter(filled, errors); globalState.executeStatement(query, template, tableName, columnToDistribute.getName()); // distribution column cannot take NULL value @@ -225,11 +225,13 @@ private static List getTableConstraints(String tableName, CitusGlobalSta throws SQLException { List constraints = new ArrayList<>(); String template = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; - String filled = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + tableName + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; + String filled = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + + tableName + + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; QueryAdapter query = new QueryAdapter(filled); - ResultSet rs = query.executeAndGet(globalState, template, tableName); + SQLancerResultSet rs = query.executeAndGet(globalState, template, tableName); while (rs.next()) { - constraints.add(rs.getString("constraint_type")); + constraints.add(rs.getString(1)); } return constraints; } @@ -240,12 +242,13 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl List tableConstraints = getTableConstraints(tableName, globalState, con); if (tableConstraints.size() == 0) { String template = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;"; - String filled = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + tableName + "';"; + String filled = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + + tableName + "';"; QueryAdapter query = new QueryAdapter(filled); - ResultSet rs = query.executeAndGet(globalState, template, tableName); + SQLancerResultSet rs = query.executeAndGet(globalState, template, tableName); while (rs.next()) { - String columnName = rs.getString("column_name"); - String dataType = rs.getString("data_type"); + String columnName = rs.getString(1); + String dataType = rs.getString(2); // data types money & bit varying have no default operator class for specified partition method if (!(dataType.equals("money") || dataType.equals("bit varying"))) { PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); @@ -255,13 +258,14 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl } else { HashMap> columnConstraints = new HashMap<>(); String template = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;"; - String filled = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + tableName + "';"; + String filled = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + + tableName + "';"; QueryAdapter query = new QueryAdapter(filled); - ResultSet rs = query.executeAndGet(globalState, template, tableName); + SQLancerResultSet rs = query.executeAndGet(globalState, template, tableName); while (rs.next()) { - String columnName = rs.getString("column_name"); - String dataType = rs.getString("data_type"); - String constraintType = rs.getString("constraint_type"); + String columnName = rs.getString(1); + String dataType = rs.getString(2); + String constraintType = rs.getString(3); // data types money & bit varying have no default operator class for specified partition method if (!(dataType.equals("money") || dataType.equals("bit varying"))) { PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); @@ -413,7 +417,8 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce con.close(); // reconnect to coordinator node, test database con = DriverManager.getConnection("jdbc:" + testURL, username, password); - ((CitusGlobalState) globalState).setRepartition(((CitusOptions) globalState.getDmbsSpecificOptions()).repartition); + ((CitusGlobalState) globalState) + .setRepartition(((CitusOptions) globalState.getDmbsSpecificOptions()).repartition); return con; } } diff --git a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java index 67aba77db..6d264763a 100644 --- a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java @@ -3,8 +3,8 @@ import java.util.List; import java.util.Set; +import sqlancer.IgnoreMeException; import sqlancer.Query; -import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.gen.PostgresAlterTableGenerator; @@ -41,7 +41,7 @@ public List getActions(Set errors) { action.remove(Action.OWNER_TO); action.remove(Action.REPLICA_IDENTITY); if (action.isEmpty()) { - action.addAll(Randomly.nonEmptySubset(Action.ALTER_TABLE_DROP_COLUMN, Action.ALTER_COLUMN_TYPE, Action.ALTER_COLUMN_SET_DROP_DEFAULT, Action.ALTER_COLUMN_SET_DROP_NULL, Action.ADD_TABLE_CONSTRAINT, Action.ADD_TABLE_CONSTRAINT_USING_INDEX, Action.VALIDATE_CONSTRAINT)); + throw new IgnoreMeException(); } return action; } diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 2b79c01b9..e863c6f79 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -27,6 +27,8 @@ public static void addCitusErrors(Collection errors) { errors.add("creating unique indexes on non-partition columns is currently unsupported"); errors.add("modifying the partition value of rows is not allowed"); errors.add("creating unique indexes on non-partition columns is currently unsupported"); + errors.add("Distributed relations must not use GENERATED ... AS IDENTITY"); + errors.add("cannot drop multiple distributed objects in a single command"); // ERROR: cannot create foreign key constraint // Detail: SET NULL or SET DEFAULT is not supported in ON DELETE operation when distribution key is included in // the foreign key constraint diff --git a/src/sqlancer/citus/gen/CitusIndexGenerator.java b/src/sqlancer/citus/gen/CitusIndexGenerator.java index 110dae037..68d5d0071 100644 --- a/src/sqlancer/citus/gen/CitusIndexGenerator.java +++ b/src/sqlancer/citus/gen/CitusIndexGenerator.java @@ -14,5 +14,5 @@ public static Query generate(PostgresGlobalState globalState) { CitusCommon.addCitusErrors(errors); return createIndexQuery; } - -} \ No newline at end of file + +} diff --git a/src/sqlancer/citus/gen/CitusSetGenerator.java b/src/sqlancer/citus/gen/CitusSetGenerator.java index d324713ee..58cfbeafd 100644 --- a/src/sqlancer/citus/gen/CitusSetGenerator.java +++ b/src/sqlancer/citus/gen/CitusSetGenerator.java @@ -15,4 +15,4 @@ public static Query create(PostgresGlobalState globalState) { return setQuery; } -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java index 6c0c4c6d5..5b8e8d927 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java @@ -32,4 +32,4 @@ public void check() throws SQLException { state.setDefaultAllowedFunctionTypes(); } -} \ No newline at end of file +} diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java index a16ef96e2..530ca2997 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -21,8 +21,8 @@ import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; +import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.gen.PostgresExpressionGenerator; import sqlancer.postgres.oracle.tlp.PostgresTLPBase; @@ -157,7 +157,7 @@ List getCitusJoinStatements(CitusGlobalState globalState, List getActions(Set errors) { action.remove(Action.CLUSTER_ON); } action.remove(Action.SET_WITH_OIDS); + if (!randomTable.hasIndexes()) { + action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); + } if (action.isEmpty()) { throw new IgnoreMeException(); } @@ -249,7 +252,7 @@ public Query generate() { break; case ADD_TABLE_CONSTRAINT: sb.append("ADD "); - sb.append("CONSTRAINT '" + r.getChar() +"' "); + sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); PostgresCommon.addTableConstraint(sb, randomTable, globalState, errors); errors.add("multiple primary keys for table"); errors.add("could not create unique index"); @@ -279,13 +282,11 @@ public Query generate() { break; case ADD_TABLE_CONSTRAINT_USING_INDEX: sb.append("ADD "); - sb.append("CONSTRAINT '" + r.getChar() +"' "); + sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); errors.add("not valid"); - if (randomTable.hasIndexes()) { - sb.append(" USING INDEX "); - sb.append(randomTable.getRandomIndex().getIndexName()); - } + sb.append(" USING INDEX "); + sb.append(randomTable.getRandomIndex().getIndexName()); errors.add("is not a unique index"); errors.add("is already associated with a constraint"); errors.add("Cannot create a primary key or unique constraint using such an index"); diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 79f7e99ab..b6d436a68 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -120,7 +120,9 @@ private PostgresExpression generateFunctionWithUnknownResult(int depth, Postgres List supportedFunctions = PostgresFunctionWithUnknownResult .getSupportedFunctions(type); // filters functions by allowed type (stable 's', immutable 'i', volatile 'v') - supportedFunctions = supportedFunctions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))).collect(Collectors.toList()); + supportedFunctions = supportedFunctions.stream() + .filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); if (supportedFunctions.isEmpty()) { throw new IgnoreMeException(); } @@ -132,7 +134,8 @@ private PostgresExpression generateFunctionWithKnownResult(int depth, PostgresDa List functions = Stream.of(PostgresFunction.PostgresFunctionWithResult.values()) .filter(f -> f.supportsReturnType(type)).collect(Collectors.toList()); // filters functions by allowed type (stable 's', immutable 'i', volatile 'v') - functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))).collect(Collectors.toList()); + functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); if (functions.isEmpty()) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 5cf317930..39c920704 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -53,7 +53,7 @@ public void check() throws SQLException { protected List getJoinStatements(PostgresGlobalState globalState, List columns, List tables) { - return PostgresNoRECOracle.getJoinStatements(state, targetTables.getColumns(), tables); + return PostgresNoRECOracle.getJoinStatements(state, columns, tables); // TODO joins } From 840246661d4b75f89eb8757d9bca441565d17f7a Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 11 Aug 2020 10:57:18 +0300 Subject: [PATCH 0088/1284] add to Citus errors --- src/sqlancer/citus/gen/CitusCommon.java | 1 + src/sqlancer/citus/oracle/tlp/CitusTLPBase.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index e863c6f79..92ed0d621 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -29,6 +29,7 @@ public static void addCitusErrors(Collection errors) { errors.add("creating unique indexes on non-partition columns is currently unsupported"); errors.add("Distributed relations must not use GENERATED ... AS IDENTITY"); errors.add("cannot drop multiple distributed objects in a single command"); + errors.add("is not distributed"); // ERROR: cannot create foreign key constraint // Detail: SET NULL or SET DEFAULT is not supported in ON DELETE operation when distribution key is included in // the foreign key constraint diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java index 530ca2997..c7b10ee7e 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -77,6 +77,7 @@ public void check() throws SQLException { private List generateJoins(List tables) { List joins = null; + tables.clear(); if (distributedTables.isEmpty() || (!referenceTables.isEmpty() && !Randomly.getBooleanWithRatherLowProbability())) { if (!localTables.isEmpty()) { From 6ed32b269068f1b18137303b5fcc89483fd92873 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 11 Aug 2020 11:27:10 +0300 Subject: [PATCH 0089/1284] Change some castings, namings, and errors in CitusProvider --- src/sqlancer/citus/CitusOptions.java | 4 +-- src/sqlancer/citus/CitusProvider.java | 45 +++++++++++++++------------ 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index 6d3076caf..625b6e7ff 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -23,9 +23,9 @@ public class CitusOptions extends PostgresOptions { public boolean repartition = true; @Parameter(names = "--citusOracle") - public List citusOracle = Arrays.asList(PostgresOracle.QUERY_PARTITIONING); + public List citusOracle = Arrays.asList(CitusOracle.QUERY_PARTITIONING); - public enum PostgresOracle { + public enum CitusOracle { NOREC { @Override public TestOracle create(PostgresGlobalState globalState) throws SQLException { diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 19dda5fcd..11a0ce1d0 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -54,13 +54,10 @@ public class CitusProvider extends PostgresProvider { - private static Set errors = new HashSet<>(); - @SuppressWarnings("unchecked") public CitusProvider() { super((Class) (Object) CitusGlobalState.class, (Class) (Object) CitusOptions.class); - CitusCommon.addCitusErrors(errors); } public enum Action implements AbstractAction { @@ -183,12 +180,12 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { } - private class WorkerNode { + private class CitusWorkerNode { private final String host; private final int port; - WorkerNode(String nodeHost, int nodePort) { + CitusWorkerNode(String nodeHost, int nodePort) { this.host = nodeHost; this.port = nodePort; } @@ -205,18 +202,18 @@ public int getPort() { private static void distributeTable(List columns, String tableName, CitusGlobalState globalState, Connection con) throws SQLException { - if (columns.size() != 0) { + if (columns.isEmpty()) { PostgresColumn columnToDistribute = Randomly.fromList(columns); String template = "SELECT create_distributed_table(?, ?);"; String filled = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + "');"; - QueryAdapter query = new QueryAdapter(filled, errors); + QueryAdapter query = new QueryAdapter(filled, getCitusErrors()); globalState.executeStatement(query, template, tableName, columnToDistribute.getName()); // distribution column cannot take NULL value // TODO: find a way to protect from SQL injection without '' around string input query = new QueryAdapter( "ALTER TABLE " + tableName + " ALTER COLUMN " + columnToDistribute.getName() + " SET NOT NULL;", - errors); + getCitusErrors()); globalState.executeStatement(query); } } @@ -240,7 +237,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl throws SQLException { List columns = new ArrayList<>(); List tableConstraints = getTableConstraints(tableName, globalState, con); - if (tableConstraints.size() == 0) { + if (tableConstraints.isEmpty()) { String template = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;"; String filled = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + tableName + "';"; @@ -249,8 +246,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl while (rs.next()) { String columnName = rs.getString(1); String dataType = rs.getString(2); - // data types money & bit varying have no default operator class for specified partition method - if (!(dataType.equals("money") || dataType.equals("bit varying"))) { + if (dataTypeHasDefaultOperatorForPartition(dataType)) { PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); columns.add(c); } @@ -266,8 +262,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl String columnName = rs.getString(1); String dataType = rs.getString(2); String constraintType = rs.getString(3); - // data types money & bit varying have no default operator class for specified partition method - if (!(dataType.equals("money") || dataType.equals("bit varying"))) { + if (dataTypeHasDefaultOperatorForPartition(dataType)) { PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); if (columnConstraints.containsKey(c)) { columnConstraints.get(c).add(constraintType); @@ -297,7 +292,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio // create reference table String template = "SELECT create_reference_table(?);"; String filled = "SELECT create_reference_table('" + table.getName() + "');"; - QueryAdapter query = new QueryAdapter(filled, errors); + QueryAdapter query = new QueryAdapter(filled, getCitusErrors()); globalState.executeStatement(query, template, table.getName()); } else { // create distributed table @@ -311,7 +306,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio prepareTables(globalState); if (((CitusGlobalState) globalState).getRepartition()) { // allow repartition joins - globalState.executeStatement(new QueryAdapter("SET citus.enable_repartition_joins to ON;\n", errors)); + globalState.executeStatement(new QueryAdapter("SET citus.enable_repartition_joins to ON;\n", getCitusErrors())); } } @@ -347,19 +342,19 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce // read info about worker nodes globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); - List workerNodes = new ArrayList<>(); + List citusWorkerNodes = new ArrayList<>(); try (Statement s = con.createStatement()) { ResultSet rs = s.executeQuery("SELECT * FROM master_get_active_worker_nodes();"); while (rs.next()) { String nodeHost = rs.getString("node_name"); int nodePort = rs.getInt("node_port"); - WorkerNode w = new WorkerNode(nodeHost, nodePort); - workerNodes.add(w); + CitusWorkerNode w = new CitusWorkerNode(nodeHost, nodePort); + citusWorkerNodes.add(w); } } con.close(); - for (WorkerNode w : workerNodes) { + for (CitusWorkerNode w : citusWorkerNodes) { // connect to worker node, entry database int hostIndex = entryURL.indexOf(host); String preHost = entryURL.substring(0, hostIndex); @@ -406,7 +401,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce con = DriverManager.getConnection("jdbc:" + testURL, username, password); // add worker nodes to coordinator node for test database - for (WorkerNode w : workerNodes) { + for (CitusWorkerNode w : citusWorkerNodes) { // TODO: protect from sql injection - is it necessary though since these are read from the system? String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; globalState.getState().logStatement(addWorkers); @@ -441,4 +436,14 @@ public String getDBMSName() { return "citus"; } + private static Set getCitusErrors() { + Set errors = new HashSet<>(); + CitusCommon.addCitusErrors(errors); + return errors; + } + + private static boolean dataTypeHasDefaultOperatorForPartition(String dataType) { + return !(dataType.equals("money") || dataType.equals("bit varying")); + } + } From daf567ae102f395fbf6d62fdc07ac9db3f49d4b0 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 11 Aug 2020 11:47:58 +0300 Subject: [PATCH 0090/1284] Remove redundant Postgres constructors --- src/sqlancer/postgres/gen/PostgresCommon.java | 3 --- src/sqlancer/postgres/gen/PostgresDeleteGenerator.java | 3 --- src/sqlancer/postgres/gen/PostgresIndexGenerator.java | 3 --- src/sqlancer/postgres/gen/PostgresInsertGenerator.java | 3 --- src/sqlancer/postgres/gen/PostgresSetGenerator.java | 3 --- src/sqlancer/postgres/gen/PostgresUpdateGenerator.java | 3 --- src/sqlancer/postgres/gen/PostgresViewGenerator.java | 3 --- 7 files changed, 21 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 3df889c35..40ead0106 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -19,9 +19,6 @@ public class PostgresCommon { - protected PostgresCommon() { - } - public static void addCommonFetchErrors(Set errors) { errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); errors.add("but it cannot be referenced from this part of the query"); diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 025bdc80d..e690535c8 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -13,9 +13,6 @@ public class PostgresDeleteGenerator { - protected PostgresDeleteGenerator() { - } - public static Query create(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); Set errors = new HashSet<>(); diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 410a970f7..49fc389e8 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -19,9 +19,6 @@ public final class PostgresIndexGenerator { - private PostgresIndexGenerator() { - } - public enum IndexType { BTREE, HASH, GIST, GIN } diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 6d8f8d01e..6333555c3 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -16,9 +16,6 @@ public class PostgresInsertGenerator { - protected PostgresInsertGenerator() { - } - public static Query insert(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); Set errors = new HashSet<>(); diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index dad7f4d37..35763cb47 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -11,9 +11,6 @@ public class PostgresSetGenerator { - protected PostgresSetGenerator() { - } - private enum ConfigurationOption { // https://www.postgresql.org/docs/11/runtime-config-wal.html // This parameter can only be set at server start. diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index a92280451..b874ddc15 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -17,9 +17,6 @@ public class PostgresUpdateGenerator { - protected PostgresUpdateGenerator() { - } - public static Query create(PostgresGlobalState globalState) { PostgresTable randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); StringBuilder sb = new StringBuilder(); diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index 3289b25d4..fadffac68 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -13,9 +13,6 @@ public class PostgresViewGenerator { - protected PostgresViewGenerator() { - } - public static Query create(PostgresGlobalState globalState) { Set errors = new HashSet<>(); StringBuilder sb = new StringBuilder("CREATE"); From 9e042596cf91518eb2582023299bebe97bad5038 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 11 Aug 2020 12:19:45 +0300 Subject: [PATCH 0091/1284] Clarify function volatility information --- .../citus/oracle/tlp/CitusTLPAggregateOracle.java | 3 ++- src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java | 2 +- src/sqlancer/postgres/PostgresGlobalState.java | 9 +++++++-- .../postgres/gen/PostgresExpressionGenerator.java | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java index 5b8e8d927..de0cde20b 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java @@ -5,6 +5,7 @@ import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; public class CitusTLPAggregateOracle extends PostgresTLPAggregateOracle { @@ -19,7 +20,7 @@ public CitusTLPAggregateOracle(CitusGlobalState state) { @Override public void check() throws SQLException { - state.setAllowedFunctionTypes(Arrays.asList('i')); + state.setAllowedFunctionTypes(Arrays.asList(PostgresGlobalState.IMMUTABLE)); citusTLPBase.check(); s = citusTLPBase.getSchema(); targetTables = citusTLPBase.getTargetTables(); diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java index b2f50d608..49e5f38e7 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java @@ -19,7 +19,7 @@ public CitusTLPHavingOracle(CitusGlobalState state) { @Override public void check() throws SQLException { - state.setAllowedFunctionTypes(Arrays.asList('i')); + state.setAllowedFunctionTypes(Arrays.asList(PostgresGlobalState.IMMUTABLE)); citusTLPBase.check(); s = citusTLPBase.getSchema(); targetTables = citusTLPBase.getTargetTables(); diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 0943f245e..22e58f21e 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -14,11 +14,16 @@ public class PostgresGlobalState extends GlobalState { + public static final char IMMUTABLE = 'i'; + public static final char STABLE = 's'; + public static final char VOLATILE = 'v'; + private List operators; private List collates; private List opClasses; + // store and allow filtering by function volatility classifications private HashMap functionsAndTypes = new HashMap<>(); - private List allowedFunctionTypes = Arrays.asList('s', 'v', 'i'); + private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); @Override public void setConnection(Connection con) { @@ -111,7 +116,7 @@ public void setAllowedFunctionTypes(List types) { } public void setDefaultAllowedFunctionTypes() { - this.allowedFunctionTypes = Arrays.asList('s', 'v', 'i'); + this.allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); } public List getAllowedFunctionTypes() { diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index b6d436a68..f26a455ab 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -119,7 +119,7 @@ private enum BooleanExpression { private PostgresExpression generateFunctionWithUnknownResult(int depth, PostgresDataType type) { List supportedFunctions = PostgresFunctionWithUnknownResult .getSupportedFunctions(type); - // filters functions by allowed type (stable 's', immutable 'i', volatile 'v') + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') supportedFunctions = supportedFunctions.stream() .filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) .collect(Collectors.toList()); @@ -133,7 +133,7 @@ private PostgresExpression generateFunctionWithUnknownResult(int depth, Postgres private PostgresExpression generateFunctionWithKnownResult(int depth, PostgresDataType type) { List functions = Stream.of(PostgresFunction.PostgresFunctionWithResult.values()) .filter(f -> f.supportsReturnType(type)).collect(Collectors.toList()); - // filters functions by allowed type (stable 's', immutable 'i', volatile 'v') + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) .collect(Collectors.toList()); if (functions.isEmpty()) { From a66692f49fba1ce6dc73119aedc4c6ebe19e3fea Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 11 Aug 2020 12:27:52 +0300 Subject: [PATCH 0092/1284] Hard-code template strings --- src/sqlancer/citus/CitusGlobalState.java | 2 +- src/sqlancer/citus/CitusProvider.java | 37 ++++++++----------- src/sqlancer/citus/CitusSchema.java | 5 +-- .../oracle/tlp/CitusTLPHavingOracle.java | 1 + .../citus/oracle/tlp/CitusTLPWhereOracle.java | 3 +- .../postgres/PostgresGlobalState.java | 6 +-- src/sqlancer/postgres/PostgresSchema.java | 4 +- 7 files changed, 26 insertions(+), 32 deletions(-) diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java index 1eb9427d2..225f5134d 100644 --- a/src/sqlancer/citus/CitusGlobalState.java +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -17,7 +17,7 @@ public boolean getRepartition() { } @Override - protected void updateSchema() throws SQLException { + public void updateSchema() throws SQLException { setSchema(CitusSchema.fromConnection(getConnection(), getDatabaseName())); } diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 11a0ce1d0..bba49559b 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -204,11 +204,10 @@ private static void distributeTable(List columns, String tableNa Connection con) throws SQLException { if (columns.isEmpty()) { PostgresColumn columnToDistribute = Randomly.fromList(columns); - String template = "SELECT create_distributed_table(?, ?);"; - String filled = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + String queryString = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + "');"; - QueryAdapter query = new QueryAdapter(filled, getCitusErrors()); - globalState.executeStatement(query, template, tableName, columnToDistribute.getName()); + QueryAdapter query = new QueryAdapter(queryString, getCitusErrors()); + globalState.executeStatement(query, "SELECT create_distributed_table(?, ?);", tableName, columnToDistribute.getName()); // distribution column cannot take NULL value // TODO: find a way to protect from SQL injection without '' around string input query = new QueryAdapter( @@ -221,12 +220,11 @@ private static void distributeTable(List columns, String tableNa private static List getTableConstraints(String tableName, CitusGlobalState globalState, Connection con) throws SQLException { List constraints = new ArrayList<>(); - String template = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; - String filled = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + String queryString = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + tableName + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; - QueryAdapter query = new QueryAdapter(filled); - SQLancerResultSet rs = query.executeAndGet(globalState, template, tableName); + QueryAdapter query = new QueryAdapter(queryString); + SQLancerResultSet rs = query.executeAndGet(globalState, "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');", tableName); while (rs.next()) { constraints.add(rs.getString(1)); } @@ -238,11 +236,10 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl List columns = new ArrayList<>(); List tableConstraints = getTableConstraints(tableName, globalState, con); if (tableConstraints.isEmpty()) { - String template = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;"; - String filled = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + String queryString = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + tableName + "';"; - QueryAdapter query = new QueryAdapter(filled); - SQLancerResultSet rs = query.executeAndGet(globalState, template, tableName); + QueryAdapter query = new QueryAdapter(queryString); + SQLancerResultSet rs = query.executeAndGet(globalState, "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;", tableName); while (rs.next()) { String columnName = rs.getString(1); String dataType = rs.getString(2); @@ -253,11 +250,10 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl } } else { HashMap> columnConstraints = new HashMap<>(); - String template = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;"; - String filled = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + String queryString = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + tableName + "';"; - QueryAdapter query = new QueryAdapter(filled); - SQLancerResultSet rs = query.executeAndGet(globalState, template, tableName); + QueryAdapter query = new QueryAdapter(queryString); + SQLancerResultSet rs = query.executeAndGet(globalState, "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;", tableName); while (rs.next()) { String columnName = rs.getString(1); String dataType = rs.getString(2); @@ -290,10 +286,9 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio if (!(table.getTableType() == TableType.TEMPORARY || Randomly.getBooleanWithRatherLowProbability())) { if (Randomly.getBooleanWithRatherLowProbability()) { // create reference table - String template = "SELECT create_reference_table(?);"; - String filled = "SELECT create_reference_table('" + table.getName() + "');"; - QueryAdapter query = new QueryAdapter(filled, getCitusErrors()); - globalState.executeStatement(query, template, table.getName()); + String queryString = "SELECT create_reference_table('" + table.getName() + "');"; + QueryAdapter query = new QueryAdapter(queryString, getCitusErrors()); + globalState.executeStatement(query, "SELECT create_reference_table(?);", table.getName()); } else { // create distributed table createDistributedTable(table.getName(), (CitusGlobalState) globalState, @@ -302,7 +297,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio } // else: keep local table } - ((CitusGlobalState) globalState).updateSchema(); + globalState.updateSchema(); prepareTables(globalState); if (((CitusGlobalState) globalState).getRepartition()) { // allow repartition joins diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index 8f2837994..a902cb0b6 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -21,6 +21,7 @@ public CitusSchema(List databaseTables, String databaseName) { public static class CitusTable extends PostgresTable { private PostgresColumn distributionColumn; + // colocationId is null for local tables private Integer colocationId; public CitusTable(String tableName, List columns, List indexes, @@ -58,7 +59,6 @@ public Integer getColocationId() { public static CitusSchema fromConnection(Connection con, String databaseName) throws SQLException { PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); - Exception ex = null; try { List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -90,9 +90,8 @@ public static CitusSchema fromConnection(Connection con, String databaseName) th } return new CitusSchema(databaseTables, databaseName); } catch (SQLIntegrityConstraintViolationException e) { - ex = e; + throw new AssertionError(e); } - throw new AssertionError(ex); } } diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java index 49e5f38e7..851ebb0b4 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java @@ -5,6 +5,7 @@ import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; public class CitusTLPHavingOracle extends PostgresTLPHavingOracle { diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java index b9bdedb14..fc1afb562 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java @@ -5,6 +5,7 @@ import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; public class CitusTLPWhereOracle extends PostgresTLPWhereOracle { @@ -19,7 +20,7 @@ public CitusTLPWhereOracle(CitusGlobalState state) { @Override public void check() throws SQLException { - state.setAllowedFunctionTypes(Arrays.asList('i')); + state.setAllowedFunctionTypes(Arrays.asList(PostgresGlobalState.IMMUTABLE)); citusTLPBase.check(); s = citusTLPBase.getSchema(); targetTables = citusTLPBase.getTargetTables(); diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 22e58f21e..1973cec13 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -17,11 +17,11 @@ public class PostgresGlobalState extends GlobalState operators; private List collates; private List opClasses; - // store and allow filtering by function volatility classifications + // store and allow filtering by function volatility classifications private HashMap functionsAndTypes = new HashMap<>(); private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); @@ -99,7 +99,7 @@ public String getRandomOpclass() { } @Override - protected void updateSchema() throws SQLException { + public void updateSchema() throws SQLException { setSchema(PostgresSchema.fromConnection(getConnection(), getDatabaseName())); } diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index a48ebf1f6..641acfcf0 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -259,7 +259,6 @@ public String getIndexName() { } public static PostgresSchema fromConnection(Connection con, String databaseName) throws SQLException { - Exception ex = null; try { List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -289,9 +288,8 @@ public static PostgresSchema fromConnection(Connection con, String databaseName) } return new PostgresSchema(databaseTables, databaseName); } catch (SQLIntegrityConstraintViolationException e) { - ex = e; + throw new AssertionError(e); } - throw new AssertionError(ex); } protected static List getStatistics(Connection con) throws SQLException { From 425ea6ad5baa335732b5d4d6d45ef7f63b0e847a Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 11 Aug 2020 13:03:22 +0300 Subject: [PATCH 0093/1284] Refactoring Citus connections & preparations --- src/sqlancer/citus/CitusProvider.java | 142 ++++++++++++++------------ 1 file changed, 78 insertions(+), 64 deletions(-) diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index bba49559b..d814dbb4d 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -317,6 +317,78 @@ protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLEx return new CompositeTestOracle(oracles, globalState); } + private List readCitusWorkerNodes(PostgresGlobalState globalState, Connection con) throws SQLException { + globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); + List citusWorkerNodes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet rs = s.executeQuery("SELECT * FROM master_get_active_worker_nodes();"); + while (rs.next()) { + String nodeHost = rs.getString("node_name"); + int nodePort = rs.getInt("node_port"); + CitusWorkerNode w = new CitusWorkerNode(nodeHost, nodePort); + citusWorkerNodes.add(w); + } + } + return citusWorkerNodes; + } + + private void addCitusExtension(PostgresGlobalState globalState, Connection con) throws SQLException { + globalState.getState().logStatement("CREATE EXTENSION citus;"); + try (Statement s = con.createStatement()) { + s.execute("CREATE EXTENSION citus;"); + } + } + + private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes, int databaseIndex, String entryDatabaseName) throws SQLException { + for (CitusWorkerNode w : citusWorkerNodes) { + // connect to worker node, entry database + int hostIndex = entryURL.indexOf(host); + String preHost = entryURL.substring(0, hostIndex); + String postHost = entryURL.substring(databaseIndex - 1); + String entryWorkerURL = preHost + w.getHost() + ":" + w.getPort() + postHost; + // TODO: better way of logging this + globalState.getState().logStatement("\\q"); + globalState.getState().logStatement(entryWorkerURL); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + Connection con = DriverManager.getConnection("jdbc:" + entryWorkerURL, username, password); + + // create test database at worker node + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + + // connect to worker node, test database + int databaseIndexWorker = entryWorkerURL.indexOf(entryPath) + 1; + String preDatabaseNameWorker = entryWorkerURL.substring(0, databaseIndexWorker); + String postDatabaseNameWorker = entryWorkerURL + .substring(databaseIndexWorker + entryDatabaseName.length()); + String testWorkerURL = preDatabaseNameWorker + databaseName + postDatabaseNameWorker; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + con = DriverManager.getConnection("jdbc:" + testWorkerURL, username, password); + + // add citus extension to worker node, test database + addCitusExtension(globalState, con); + con.close(); + } + } + + private void addCitusWorkerNodes(PostgresGlobalState globalState, Connection con, List citusWorkerNodes) throws SQLException{ + for (CitusWorkerNode w : citusWorkerNodes) { + // TODO: protect from sql injection - is it necessary though since these are read from the system? + String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; + globalState.getState().logStatement(addWorkers); + try (Statement s = con.createStatement()) { + s.execute(addWorkers); + } + } + } + @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { synchronized (CitusProvider.class) { @@ -325,86 +397,28 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce String entryDatabaseName = entryPath.substring(1); int databaseIndex = entryURL.indexOf(entryPath) + 1; // add citus extension to coordinator node, test database - globalState.getState().logStatement(new QueryAdapter("CREATE EXTENSION citus;")); - try (Statement s = con.createStatement()) { - s.execute("CREATE EXTENSION citus;"); - } + addCitusExtension(globalState, con); con.close(); // reconnect to coordinator node, entry database globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); con = DriverManager.getConnection("jdbc:" + entryURL, username, password); - // read info about worker nodes - globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); - List citusWorkerNodes = new ArrayList<>(); - try (Statement s = con.createStatement()) { - ResultSet rs = s.executeQuery("SELECT * FROM master_get_active_worker_nodes();"); - while (rs.next()) { - String nodeHost = rs.getString("node_name"); - int nodePort = rs.getInt("node_port"); - CitusWorkerNode w = new CitusWorkerNode(nodeHost, nodePort); - citusWorkerNodes.add(w); - } - } + List citusWorkerNodes = readCitusWorkerNodes(globalState, con); con.close(); - for (CitusWorkerNode w : citusWorkerNodes) { - // connect to worker node, entry database - int hostIndex = entryURL.indexOf(host); - String preHost = entryURL.substring(0, hostIndex); - String postHost = entryURL.substring(databaseIndex - 1); - String entryWorkerURL = preHost + w.getHost() + ":" + w.getPort() + postHost; - // TODO: better way of logging this - globalState.getState().logStatement("\\q"); - globalState.getState().logStatement(entryWorkerURL); - globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - con = DriverManager.getConnection("jdbc:" + entryWorkerURL, username, password); - - // create test database at worker node - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - globalState.getState().logStatement(createDatabaseCommand); - try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); - } - try (Statement s = con.createStatement()) { - s.execute(createDatabaseCommand); - } - con.close(); - - // connect to worker node, test database - int databaseIndexWorker = entryWorkerURL.indexOf(entryPath) + 1; - String preDatabaseNameWorker = entryWorkerURL.substring(0, databaseIndexWorker); - String postDatabaseNameWorker = entryWorkerURL - .substring(databaseIndexWorker + entryDatabaseName.length()); - String testWorkerURL = preDatabaseNameWorker + databaseName + postDatabaseNameWorker; - globalState.getState().logStatement(String.format("\\c %s;", databaseName)); - con = DriverManager.getConnection("jdbc:" + testWorkerURL, username, password); - - // add citus extension to worker node, test database - globalState.getState().logStatement("CREATE EXTENSION citus;"); - try (Statement s = con.createStatement()) { - s.execute("CREATE EXTENSION citus;"); - } - con.close(); - } + // prepare worker nodes for test database + prepareCitusWorkerNodes(globalState, citusWorkerNodes, databaseIndex, entryDatabaseName); // reconnect to coordinator node, test database // TODO: better way of logging this globalState.getState().logStatement("\\q"); globalState.getState().logStatement(testURL); con = DriverManager.getConnection("jdbc:" + testURL, username, password); - // add worker nodes to coordinator node for test database - for (CitusWorkerNode w : citusWorkerNodes) { - // TODO: protect from sql injection - is it necessary though since these are read from the system? - String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; - globalState.getState().logStatement(addWorkers); - try (Statement s = con.createStatement()) { - s.execute(addWorkers); - } - } + addCitusWorkerNodes(globalState, con, citusWorkerNodes); con.close(); + // reconnect to coordinator node, test database con = DriverManager.getConnection("jdbc:" + testURL, username, password); ((CitusGlobalState) globalState) From 9bf8da0c2ed236a406328f2dd99fa9faff81326f Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 11 Aug 2020 13:45:22 +0300 Subject: [PATCH 0094/1284] Refactor Citus joins --- src/sqlancer/citus/CitusOptions.java | 2 +- src/sqlancer/citus/CitusProvider.java | 47 +++++----- src/sqlancer/citus/gen/CitusCommon.java | 8 +- .../citus/gen/CitusDeleteGenerator.java | 5 +- .../citus/gen/CitusIndexGenerator.java | 5 +- .../citus/gen/CitusInsertGenerator.java | 5 +- src/sqlancer/citus/gen/CitusSetGenerator.java | 5 +- .../citus/gen/CitusUpdateGenerator.java | 5 +- .../citus/gen/CitusViewGenerator.java | 5 +- .../oracle/tlp/CitusTLPAggregateOracle.java | 2 +- .../citus/oracle/tlp/CitusTLPBase.java | 90 ++++++++++--------- .../oracle/tlp/CitusTLPHavingOracle.java | 2 +- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 2 +- .../postgres/PostgresGlobalState.java | 5 +- src/sqlancer/postgres/gen/PostgresCommon.java | 7 +- .../postgres/gen/PostgresDeleteGenerator.java | 5 +- .../gen/PostgresExpressionGenerator.java | 6 +- .../postgres/gen/PostgresIndexGenerator.java | 3 + .../postgres/gen/PostgresInsertGenerator.java | 5 +- .../postgres/gen/PostgresSetGenerator.java | 5 +- .../postgres/gen/PostgresUpdateGenerator.java | 5 +- .../postgres/gen/PostgresViewGenerator.java | 5 +- 22 files changed, 143 insertions(+), 86 deletions(-) diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index 625b6e7ff..ebea40e4a 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -22,7 +22,7 @@ public class CitusOptions extends PostgresOptions { @Parameter(names = "--repartition") public boolean repartition = true; - @Parameter(names = "--citusOracle") + @Parameter(names = "--citusoracle") public List citusOracle = Arrays.asList(CitusOracle.QUERY_PARTITIONING); public enum CitusOracle { diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index d814dbb4d..f2e1c2b3a 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -200,14 +200,15 @@ public int getPort() { } - private static void distributeTable(List columns, String tableName, CitusGlobalState globalState, - Connection con) throws SQLException { - if (columns.isEmpty()) { + private static void distributeTable(List columns, String tableName, CitusGlobalState globalState) + throws SQLException { + if (!columns.isEmpty()) { PostgresColumn columnToDistribute = Randomly.fromList(columns); String queryString = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + "');"; QueryAdapter query = new QueryAdapter(queryString, getCitusErrors()); - globalState.executeStatement(query, "SELECT create_distributed_table(?, ?);", tableName, columnToDistribute.getName()); + globalState.executeStatement(query, "SELECT create_distributed_table(?, ?);", tableName, + columnToDistribute.getName()); // distribution column cannot take NULL value // TODO: find a way to protect from SQL injection without '' around string input query = new QueryAdapter( @@ -217,29 +218,31 @@ private static void distributeTable(List columns, String tableNa } } - private static List getTableConstraints(String tableName, CitusGlobalState globalState, Connection con) + private static List getTableConstraints(String tableName, CitusGlobalState globalState) throws SQLException { List constraints = new ArrayList<>(); String queryString = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + tableName + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; QueryAdapter query = new QueryAdapter(queryString); - SQLancerResultSet rs = query.executeAndGet(globalState, "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');", tableName); + SQLancerResultSet rs = query.executeAndGet(globalState, + "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');", + tableName); while (rs.next()) { constraints.add(rs.getString(1)); } return constraints; } - private static void createDistributedTable(String tableName, CitusGlobalState globalState, Connection con) - throws SQLException { + private static void createDistributedTable(String tableName, CitusGlobalState globalState) throws SQLException { List columns = new ArrayList<>(); - List tableConstraints = getTableConstraints(tableName, globalState, con); + List tableConstraints = getTableConstraints(tableName, globalState); if (tableConstraints.isEmpty()) { String queryString = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + tableName + "';"; QueryAdapter query = new QueryAdapter(queryString); - SQLancerResultSet rs = query.executeAndGet(globalState, "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;", tableName); + SQLancerResultSet rs = query.executeAndGet(globalState, + "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;", tableName); while (rs.next()) { String columnName = rs.getString(1); String dataType = rs.getString(2); @@ -253,7 +256,9 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl String queryString = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + tableName + "';"; QueryAdapter query = new QueryAdapter(queryString); - SQLancerResultSet rs = query.executeAndGet(globalState, "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;", tableName); + SQLancerResultSet rs = query.executeAndGet(globalState, + "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;", + tableName); while (rs.next()) { String columnName = rs.getString(1); String dataType = rs.getString(2); @@ -275,7 +280,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl } // TODO: figure out how to use EXCLUDE } - distributeTable(columns, tableName, globalState, con); + distributeTable(columns, tableName, globalState); } @Override @@ -291,8 +296,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio globalState.executeStatement(query, "SELECT create_reference_table(?);", table.getName()); } else { // create distributed table - createDistributedTable(table.getName(), (CitusGlobalState) globalState, - globalState.getConnection()); + createDistributedTable(table.getName(), (CitusGlobalState) globalState); } } // else: keep local table @@ -301,7 +305,8 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio prepareTables(globalState); if (((CitusGlobalState) globalState).getRepartition()) { // allow repartition joins - globalState.executeStatement(new QueryAdapter("SET citus.enable_repartition_joins to ON;\n", getCitusErrors())); + globalState.executeStatement( + new QueryAdapter("SET citus.enable_repartition_joins to ON;\n", getCitusErrors())); } } @@ -317,7 +322,8 @@ protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLEx return new CompositeTestOracle(oracles, globalState); } - private List readCitusWorkerNodes(PostgresGlobalState globalState, Connection con) throws SQLException { + private List readCitusWorkerNodes(PostgresGlobalState globalState, Connection con) + throws SQLException { globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); List citusWorkerNodes = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -339,7 +345,8 @@ private void addCitusExtension(PostgresGlobalState globalState, Connection con) } } - private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes, int databaseIndex, String entryDatabaseName) throws SQLException { + private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes, + int databaseIndex, String entryDatabaseName) throws SQLException { for (CitusWorkerNode w : citusWorkerNodes) { // connect to worker node, entry database int hostIndex = entryURL.indexOf(host); @@ -366,8 +373,7 @@ private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes) throws SQLException{ + private void addCitusWorkerNodes(PostgresGlobalState globalState, Connection con, + List citusWorkerNodes) throws SQLException { for (CitusWorkerNode w : citusWorkerNodes) { // TODO: protect from sql injection - is it necessary though since these are read from the system? String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 92ed0d621..def1f8690 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -3,12 +3,14 @@ import java.util.Collection; import java.util.Set; -import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.gen.PostgresCommon; -public class CitusCommon extends PostgresCommon { +public final class CitusCommon { + + private CitusCommon() { + } public static void addCitusErrors(Collection errors) { errors.add("recursive CTEs are not supported in distributed queries"); @@ -67,7 +69,7 @@ public static void addCitusErrors(Collection errors) { public static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, Set errors) { - addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); + PostgresCommon.addTableConstraint(sb, table, globalState, errors); CitusCommon.addCitusErrors(errors); } diff --git a/src/sqlancer/citus/gen/CitusDeleteGenerator.java b/src/sqlancer/citus/gen/CitusDeleteGenerator.java index 571afcdc7..771480eaf 100644 --- a/src/sqlancer/citus/gen/CitusDeleteGenerator.java +++ b/src/sqlancer/citus/gen/CitusDeleteGenerator.java @@ -6,7 +6,10 @@ import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresDeleteGenerator; -public class CitusDeleteGenerator extends PostgresDeleteGenerator { +public final class CitusDeleteGenerator { + + private CitusDeleteGenerator() { + } public static Query create(PostgresGlobalState globalState) { Query deleteQuery = PostgresDeleteGenerator.create(globalState); diff --git a/src/sqlancer/citus/gen/CitusIndexGenerator.java b/src/sqlancer/citus/gen/CitusIndexGenerator.java index 68d5d0071..ca51ec2ec 100644 --- a/src/sqlancer/citus/gen/CitusIndexGenerator.java +++ b/src/sqlancer/citus/gen/CitusIndexGenerator.java @@ -6,7 +6,10 @@ import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresInsertGenerator; -public class CitusIndexGenerator extends PostgresInsertGenerator { +public final class CitusIndexGenerator { + + private CitusIndexGenerator() { + } public static Query generate(PostgresGlobalState globalState) { Query createIndexQuery = PostgresInsertGenerator.insert(globalState); diff --git a/src/sqlancer/citus/gen/CitusInsertGenerator.java b/src/sqlancer/citus/gen/CitusInsertGenerator.java index e9ecbcb9c..28334177f 100644 --- a/src/sqlancer/citus/gen/CitusInsertGenerator.java +++ b/src/sqlancer/citus/gen/CitusInsertGenerator.java @@ -6,7 +6,10 @@ import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresInsertGenerator; -public class CitusInsertGenerator extends PostgresInsertGenerator { +public final class CitusInsertGenerator { + + private CitusInsertGenerator() { + } public static Query insert(PostgresGlobalState globalState) { Query insertQuery = PostgresInsertGenerator.insert(globalState); diff --git a/src/sqlancer/citus/gen/CitusSetGenerator.java b/src/sqlancer/citus/gen/CitusSetGenerator.java index 58cfbeafd..26baf43a9 100644 --- a/src/sqlancer/citus/gen/CitusSetGenerator.java +++ b/src/sqlancer/citus/gen/CitusSetGenerator.java @@ -6,7 +6,10 @@ import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresSetGenerator; -public class CitusSetGenerator extends PostgresSetGenerator { +public final class CitusSetGenerator { + + private CitusSetGenerator() { + } public static Query create(PostgresGlobalState globalState) { Query setQuery = PostgresSetGenerator.create(globalState); diff --git a/src/sqlancer/citus/gen/CitusUpdateGenerator.java b/src/sqlancer/citus/gen/CitusUpdateGenerator.java index bae463267..9604a12f0 100644 --- a/src/sqlancer/citus/gen/CitusUpdateGenerator.java +++ b/src/sqlancer/citus/gen/CitusUpdateGenerator.java @@ -6,7 +6,10 @@ import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresUpdateGenerator; -public class CitusUpdateGenerator extends PostgresUpdateGenerator { +public final class CitusUpdateGenerator { + + private CitusUpdateGenerator() { + } public static Query create(PostgresGlobalState globalState) { Query updateQuery = PostgresUpdateGenerator.create(globalState); diff --git a/src/sqlancer/citus/gen/CitusViewGenerator.java b/src/sqlancer/citus/gen/CitusViewGenerator.java index 72af318b2..65dc2fbea 100644 --- a/src/sqlancer/citus/gen/CitusViewGenerator.java +++ b/src/sqlancer/citus/gen/CitusViewGenerator.java @@ -6,7 +6,10 @@ import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresViewGenerator; -public class CitusViewGenerator extends PostgresViewGenerator { +public final class CitusViewGenerator { + + private CitusViewGenerator() { + } public static Query create(PostgresGlobalState globalState) { Query viewQuery = PostgresViewGenerator.create(globalState); diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java index de0cde20b..1ab27ab85 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java @@ -10,7 +10,7 @@ public class CitusTLPAggregateOracle extends PostgresTLPAggregateOracle { - private CitusTLPBase citusTLPBase; + private final CitusTLPBase citusTLPBase; public CitusTLPAggregateOracle(CitusGlobalState state) { super(state); diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java index c7b10ee7e..01e2ad6fb 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -29,7 +30,7 @@ public class CitusTLPBase extends PostgresTLPBase { - HashMap distributedTables; + Map distributedTables; List referenceTables; List localTables; @@ -79,7 +80,7 @@ private List generateJoins(List tables) { List joins = null; tables.clear(); if (distributedTables.isEmpty() - || (!referenceTables.isEmpty() && !Randomly.getBooleanWithRatherLowProbability())) { + || !referenceTables.isEmpty() && !Randomly.getBooleanWithRatherLowProbability()) { if (!localTables.isEmpty()) { // joins including only local tables // supports complex joins @@ -131,6 +132,40 @@ private void storeCitusTableTypes() { } } + private PostgresJoin allColocatedJoins(CitusTable joinTable, CitusTable fromTable, + PostgresExpressionGenerator citusJoinGen) { + PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); + PostgresExpression rightExpr = new PostgresColumnValue(joinTable.getDistributionColumn(), null); + PostgresExpression equiJoinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, + PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); + PostgresExpression joinClause = null; + if (Randomly.getBooleanWithSmallProbability()) { + joinClause = new PostgresBinaryLogicalOperation(equiJoinClause, + citusJoinGen.generateExpression(PostgresDataType.BOOLEAN), + PostgresBinaryLogicalOperation.BinaryLogicalOperator.AND); + } else { + joinClause = equiJoinClause; + } + PostgresJoinType options = Randomly.fromOptions(PostgresJoinType.INNER, PostgresJoinType.LEFT, + PostgresJoinType.RIGHT, PostgresJoinType.FULL); + return new PostgresJoin(new PostgresFromTable(joinTable, Randomly.getBoolean()), joinClause, options); + } + + private PostgresJoin repartitionJoins(CitusTable joinTable, CitusTable fromTable) { + PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); + List candidateRightColumns = joinTable.getColumns().stream() + .filter(c -> c.getType().equals(fromTable.getDistributionColumn().getType())) + .collect(Collectors.toList()); + if (candidateRightColumns.isEmpty()) { + return null; + } + PostgresExpression rightExpr = new PostgresColumnValue(Randomly.fromList(candidateRightColumns), null); + PostgresExpression joinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, + PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); + PostgresJoinType options = PostgresJoinType.INNER; + return new PostgresJoin(new PostgresFromTable(joinTable, Randomly.getBoolean()), joinClause, options); + } + List getCitusJoinStatements(CitusGlobalState globalState, List joinTables, CitusTable fromTable) { List columns = new ArrayList<>(); @@ -138,54 +173,26 @@ List getCitusJoinStatements(CitusGlobalState globalState, List joinStatements = new ArrayList<>(); - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); + PostgresExpressionGenerator citusJoinGen = new PostgresExpressionGenerator(globalState).setColumns(columns); joinTables.remove(fromTable); boolean allColocated = true; for (PostgresTable t : joinTables) { - boolean colocated = (distributedTables.get(fromTable) == distributedTables.get(t)); + boolean colocated = distributedTables.get(fromTable).equals(distributedTables.get(t)); allColocated = allColocated && colocated; } while (!joinTables.isEmpty()) { CitusTable table = (CitusTable) Randomly.fromList(joinTables); - // FIXME: can I remove even if reference types don't match due to casting? joinTables.remove(table); - PostgresExpression joinClause = null; - PostgresExpression equiJoinClause = null; + PostgresJoin j = null; if (allColocated) { - PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); - PostgresExpression rightExpr = new PostgresColumnValue(table.getDistributionColumn(), null); - equiJoinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, - PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); - } else { - // check if repartition joins are allowed - if (!globalState.getRepartition()) { - continue; - } - PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); - List candidateRightColumns = table.getColumns().stream() - .filter(c -> c.getType().equals(fromTable.getDistributionColumn().getType())) - .collect(Collectors.toList()); - if (candidateRightColumns.isEmpty()) { - continue; - } - PostgresExpression rightExpr = new PostgresColumnValue(Randomly.fromList(candidateRightColumns), null); - equiJoinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, - PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); + j = allColocatedJoins(table, fromTable, citusJoinGen); + // check if repartition joins are allowed if all tables are not colocated + } else if (globalState.getRepartition()) { + j = repartitionJoins(table, fromTable); } - if (allColocated && Randomly.getBooleanWithSmallProbability()) { - joinClause = new PostgresBinaryLogicalOperation(equiJoinClause, - gen.generateExpression(PostgresDataType.BOOLEAN), - PostgresBinaryLogicalOperation.BinaryLogicalOperator.AND); - } else { - joinClause = equiJoinClause; - } - PostgresJoinType options = Randomly.fromOptions(PostgresJoinType.INNER, PostgresJoinType.LEFT, - PostgresJoinType.RIGHT, PostgresJoinType.FULL); - if (!allColocated) { - options = PostgresJoinType.INNER; + if (j != null) { + joinStatements.add(j); } - PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); - joinStatements.add(j); } joinTables.add(fromTable); return joinStatements; @@ -197,9 +204,10 @@ private void addSubqueryJoinStatements(PostgresGlobalState globalState, List columns = subqueryTables.getColumns(); columns.addAll(fromTable.getColumns()); - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); + PostgresExpressionGenerator subqueryJoinGen = new PostgresExpressionGenerator(globalState) + .setColumns(columns); PostgresExpression subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); - PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); + PostgresExpression joinClause = subqueryJoinGen.generateExpression(PostgresDataType.BOOLEAN); PostgresJoinType options = PostgresJoinType.getRandom(); PostgresJoin j = new PostgresJoin(subquery, joinClause, options); joinStatements.add(j); diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java index 851ebb0b4..4162cb9a9 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java @@ -10,7 +10,7 @@ public class CitusTLPHavingOracle extends PostgresTLPHavingOracle { - private CitusTLPBase citusTLPBase; + private final CitusTLPBase citusTLPBase; public CitusTLPHavingOracle(CitusGlobalState state) { super(state); diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java index fc1afb562..1ccecd0c5 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java @@ -10,7 +10,7 @@ public class CitusTLPWhereOracle extends PostgresTLPWhereOracle { - private CitusTLPBase citusTLPBase; + private final CitusTLPBase citusTLPBase; public CitusTLPWhereOracle(CitusGlobalState state) { super(state); diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 1973cec13..b3ec5cfd6 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -8,6 +8,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; import sqlancer.GlobalState; import sqlancer.Randomly; @@ -22,7 +23,7 @@ public class PostgresGlobalState extends GlobalState collates; private List opClasses; // store and allow filtering by function volatility classifications - private HashMap functionsAndTypes = new HashMap<>(); + private final Map functionsAndTypes = new HashMap<>(); private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); @Override @@ -107,7 +108,7 @@ public void addFunctionAndType(String functionName, Character functionType) { this.functionsAndTypes.put(functionName, functionType); } - public HashMap getFunctionsAndTypes() { + public Map getFunctionsAndTypes() { return this.functionsAndTypes; } diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 40ead0106..f45180467 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -17,7 +17,10 @@ import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; -public class PostgresCommon { +public final class PostgresCommon { + + private PostgresCommon() { + } public static void addCommonFetchErrors(Set errors) { errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); @@ -259,7 +262,7 @@ public static void addTableConstraint(StringBuilder sb, PostgresTable table, Pos addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); } - protected static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, + private static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, TableConstraints t, Set errors) { List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); List otherColumns; diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index e690535c8..1fc592235 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -11,7 +11,10 @@ import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; -public class PostgresDeleteGenerator { +public final class PostgresDeleteGenerator { + + private PostgresDeleteGenerator() { + } public static Query create(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index f26a455ab..9c1091bf7 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -3,8 +3,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -67,9 +67,9 @@ public class PostgresExpressionGenerator implements ExpressionGenerator functionsAndTypes; + private final Map functionsAndTypes; - private List allowedFunctionTypes; + private final List allowedFunctionTypes; public PostgresExpressionGenerator(PostgresGlobalState globalState) { this.r = globalState.getRandomly(); diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 49fc389e8..410a970f7 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -19,6 +19,9 @@ public final class PostgresIndexGenerator { + private PostgresIndexGenerator() { + } + public enum IndexType { BTREE, HASH, GIST, GIN } diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 6333555c3..759249326 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -14,7 +14,10 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public class PostgresInsertGenerator { +public final class PostgresInsertGenerator { + + private PostgresInsertGenerator() { + } public static Query insert(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 35763cb47..d403f0e0a 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -9,7 +9,10 @@ import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; -public class PostgresSetGenerator { +public final class PostgresSetGenerator { + + private PostgresSetGenerator() { + } private enum ConfigurationOption { // https://www.postgresql.org/docs/11/runtime-config-wal.html diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index b874ddc15..16d0cb9e3 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -15,7 +15,10 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public class PostgresUpdateGenerator { +public final class PostgresUpdateGenerator { + + private PostgresUpdateGenerator() { + } public static Query create(PostgresGlobalState globalState) { PostgresTable randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index fadffac68..afc91d341 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -11,7 +11,10 @@ import sqlancer.postgres.ast.PostgresSelect; import sqlancer.sqlite3.gen.SQLite3Common; -public class PostgresViewGenerator { +public final class PostgresViewGenerator { + + private PostgresViewGenerator() { + } public static Query create(PostgresGlobalState globalState) { Set errors = new HashSet<>(); From 04926507d62a953800b86ff45944a0861d9baf4e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Aug 2020 16:54:31 +0200 Subject: [PATCH 0095/1284] Add an expected error class This class introduces additional flexibility when dealing with expected errors. For now, it unifies the data structure used for handling expected errors. Future improvements could, for example, include support for regular expressions. --- src/sqlancer/ComparatorHelper.java | 12 ++-- src/sqlancer/ExpectedErrors.java | 58 +++++++++++++++++++ src/sqlancer/NoRECBase.java | 4 +- src/sqlancer/Query.java | 3 +- src/sqlancer/QueryAdapter.java | 28 +++------ .../TernaryLogicPartitioningOracleBase.java | 5 +- src/sqlancer/citus/CitusProvider.java | 7 +-- .../citus/gen/CitusAlterTableGenerator.java | 4 +- src/sqlancer/citus/gen/CitusCommon.java | 8 +-- .../citus/gen/CitusDeleteGenerator.java | 5 +- .../citus/gen/CitusIndexGenerator.java | 5 +- .../citus/gen/CitusInsertGenerator.java | 5 +- src/sqlancer/citus/gen/CitusSetGenerator.java | 5 +- .../citus/gen/CitusUpdateGenerator.java | 5 +- .../citus/gen/CitusViewGenerator.java | 5 +- src/sqlancer/clickhouse/ClickHouseErrors.java | 12 ++-- .../gen/ClickHouseInsertGenerator.java | 5 +- .../gen/ClickHouseTableGenerator.java | 5 +- .../cockroachdb/CockroachDBErrors.java | 16 ++--- .../cockroachdb/CockroachDBProvider.java | 15 +++-- .../gen/CockroachDBCommentOnGenerator.java | 5 +- .../CockroachDBCreateStatisticsGenerator.java | 5 +- .../gen/CockroachDBDeleteGenerator.java | 6 +- .../gen/CockroachDBInsertGenerator.java | 5 +- ...CockroachDBSetClusterSettingGenerator.java | 5 +- .../gen/CockroachDBSetSessionGenerator.java | 5 +- .../gen/CockroachDBShowGenerator.java | 6 +- .../gen/CockroachDBTruncateGenerator.java | 6 +- .../gen/CockroachDBUpdateGenerator.java | 5 +- .../gen/CockroachDBViewGenerator.java | 6 +- .../oracle/CockroachDBNoRECOracle.java | 6 +- .../tlp/CockroachDBTLPAggregateOracle.java | 5 +- .../oracle/tlp/CockroachDBTLPJoinOracle.java | 5 +- src/sqlancer/duckdb/DuckDBErrors.java | 12 ++-- src/sqlancer/duckdb/DuckDBProvider.java | 5 +- .../duckdb/gen/DuckDBAlterTableGenerator.java | 6 +- .../duckdb/gen/DuckDBDeleteGenerator.java | 6 +- .../duckdb/gen/DuckDBIndexGenerator.java | 5 +- .../duckdb/gen/DuckDBInsertGenerator.java | 5 +- .../duckdb/gen/DuckDBTableGenerator.java | 5 +- .../duckdb/gen/DuckDBUpdateGenerator.java | 5 +- .../duckdb/gen/DuckDBViewGenerator.java | 6 +- src/sqlancer/gen/AbstractGenerator.java | 6 +- src/sqlancer/mariadb/MariaDBErrors.java | 4 +- .../mariadb/gen/MariaDBIndexGenerator.java | 4 +- .../mariadb/gen/MariaDBInsertGenerator.java | 6 +- .../mariadb/gen/MariaDBTableGenerator.java | 3 +- .../mariadb/gen/MariaDBUpdateGenerator.java | 6 +- src/sqlancer/mysql/MySQLErrors.java | 4 +- src/sqlancer/mysql/gen/MySQLAlterTable.java | 6 +- .../mysql/gen/MySQLDeleteGenerator.java | 5 +- src/sqlancer/mysql/gen/MySQLDropIndex.java | 8 +-- .../mysql/gen/MySQLInsertGenerator.java | 5 +- .../mysql/gen/MySQLTableGenerator.java | 5 +- .../gen/MySQLTruncateTableGenerator.java | 5 +- .../gen/datadef/MySQLIndexGenerator.java | 5 +- .../gen/PostgresAlterTableGenerator.java | 7 +-- .../gen/PostgresAnalyzeGenerator.java | 4 +- .../gen/PostgresClusterGenerator.java | 6 +- src/sqlancer/postgres/gen/PostgresCommon.java | 33 ++++++----- .../postgres/gen/PostgresDeleteGenerator.java | 6 +- .../gen/PostgresDiscardGenerator.java | 5 +- .../gen/PostgresDropIndexGenerator.java | 6 +- .../postgres/gen/PostgresIndexGenerator.java | 5 +- .../postgres/gen/PostgresInsertGenerator.java | 5 +- .../gen/PostgresReindexGenerator.java | 4 +- .../gen/PostgresSequenceGenerator.java | 6 +- .../gen/PostgresStatisticsGenerator.java | 5 +- .../postgres/gen/PostgresTableGenerator.java | 5 +- .../gen/PostgresTransactionGenerator.java | 6 +- .../gen/PostgresTruncateGenerator.java | 6 +- .../postgres/gen/PostgresUpdateGenerator.java | 8 +-- .../postgres/gen/PostgresVacuumGenerator.java | 4 +- .../postgres/gen/PostgresViewGenerator.java | 6 +- src/sqlancer/sqlite3/SQLite3Errors.java | 17 +++--- src/sqlancer/sqlite3/SQLite3Provider.java | 5 +- .../sqlite3/gen/SQLite3AnalyzeGenerator.java | 5 +- ...Lite3CreateVirtualRtreeTabelGenerator.java | 5 +- .../sqlite3/gen/SQLite3PragmaGenerator.java | 5 +- .../sqlite3/gen/SQLite3ReindexGenerator.java | 6 +- .../gen/SQLite3TransactionGenerator.java | 11 ++-- .../sqlite3/gen/SQLite3VacuumGenerator.java | 7 +-- ...QLite3VirtualFTSTableCommandGenerator.java | 6 +- .../sqlite3/gen/ddl/SQLite3AlterTable.java | 5 +- .../ddl/SQLite3CreateTriggerGenerator.java | 4 +- ...SQLite3CreateVirtualFTSTableGenerator.java | 5 +- .../gen/ddl/SQLite3DropIndexGenerator.java | 5 +- .../gen/ddl/SQLite3DropTableGenerator.java | 5 +- .../gen/ddl/SQLite3IndexGenerator.java | 5 +- .../gen/ddl/SQLite3TableGenerator.java | 3 +- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 5 +- .../gen/dml/SQLite3DeleteGenerator.java | 5 +- .../gen/dml/SQLite3InsertGenerator.java | 6 +- .../gen/dml/SQLite3UpdateGenerator.java | 4 +- .../SQLite3PivotedQuerySynthesisOracle.java | 14 ++--- .../oracle/tlp/SQLite3TLPAggregateOracle.java | 4 +- .../oracle/tlp/SQLite3TLPHavingOracle.java | 4 +- .../sqlite3/schema/SQLite3Schema.java | 3 +- src/sqlancer/tidb/TiDBErrors.java | 8 +-- src/sqlancer/tidb/TiDBProvider.java | 5 +- .../tidb/gen/TiDBAlterTableGenerator.java | 5 +- .../tidb/gen/TiDBAnalyzeTableGenerator.java | 4 +- .../tidb/gen/TiDBDeleteGenerator.java | 5 +- src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 5 +- .../tidb/gen/TiDBInsertGenerator.java | 5 +- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 5 +- .../tidb/gen/TiDBUpdateGenerator.java | 5 +- src/sqlancer/tidb/gen/TiDBViewGenerator.java | 6 +- test/sqlancer/TestExpectedErrors.java | 38 ++++++++++++ 109 files changed, 382 insertions(+), 389 deletions(-) create mode 100644 src/sqlancer/ExpectedErrors.java create mode 100644 test/sqlancer/TestExpectedErrors.java diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index d65713e23..97f47941c 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -31,7 +31,7 @@ static boolean equals(double a, double b) { return Math.abs(a - b) < 0.0001 * Math.max(Math.abs(a), Math.abs(b)); } - public static List getResultSetFirstColumnAsString(String queryString, Set errors, + public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, GlobalState state) throws SQLException { if (state.getOptions().logEachSelect()) { // TODO: refactor me @@ -65,10 +65,8 @@ public static List getResultSetFirstColumnAsString(String queryString, S if (e.getMessage() == null) { throw new AssertionError(queryString, e); } - for (String error : errors) { - if (e.getMessage().contains(error)) { - throw new IgnoreMeException(); - } + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); } throw new AssertionError(queryString, e); } finally { @@ -114,7 +112,7 @@ public static void assumeResultSetsAreEqual(List resultSet, List public static List getCombinedResultSet(String firstQueryString, String secondQueryString, String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, - Set errors) throws SQLException { + ExpectedErrors errors) throws SQLException { List secondResultSet; if (asUnion) { String unionString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL " @@ -135,7 +133,7 @@ public static List getCombinedResultSet(String firstQueryString, String public static List getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString, String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, - Set errors) throws SQLException { + ExpectedErrors errors) throws SQLException { String unionString; if (asUnion) { unionString = firstQueryString + " UNION " + secondQueryString + " UNION " + thirdQueryString; diff --git a/src/sqlancer/ExpectedErrors.java b/src/sqlancer/ExpectedErrors.java new file mode 100644 index 000000000..9551956e8 --- /dev/null +++ b/src/sqlancer/ExpectedErrors.java @@ -0,0 +1,58 @@ +package sqlancer; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +/** + * This class represents the errors that executing a statement might result in. For example, an INSERT statement might + * result in an error "UNIQUE constraint violated" when it attempts to insert a duplicate value in a column declared as + * UNIQUE. + */ +public class ExpectedErrors { + + private final Set errors = new HashSet<>(); + + public ExpectedErrors add(String error) { + if (error == null) { + throw new IllegalArgumentException(); + } + errors.add(error); + return this; + } + + /** + * Checks whether the error message (e.g., returned by the DBMS under test) contains any of the added error + * messages. + * + * @param error + * the error message + * + * @return whether the error message contains any of the substrings specified as expected errors + */ + public boolean errorIsExpected(String error) { + if (error == null) { + throw new IllegalArgumentException(); + } + for (String s : errors) { + if (error.contains(s)) { + return true; + } + } + return false; + } + + public ExpectedErrors addAll(Collection list) { + errors.addAll(list); + return this; + } + + public static ExpectedErrors from(String... errors) { + ExpectedErrors expectedErrors = new ExpectedErrors(); + for (String error : errors) { + expectedErrors.add(error); + } + return expectedErrors; + } + +} diff --git a/src/sqlancer/NoRECBase.java b/src/sqlancer/NoRECBase.java index 92ec27b0e..55be742fd 100644 --- a/src/sqlancer/NoRECBase.java +++ b/src/sqlancer/NoRECBase.java @@ -1,15 +1,13 @@ package sqlancer; import java.sql.Connection; -import java.util.HashSet; -import java.util.Set; import sqlancer.Main.StateLogger; public abstract class NoRECBase> implements TestOracle { protected final S state; - protected final Set errors = new HashSet<>(); + protected final ExpectedErrors errors = new ExpectedErrors(); protected final StateLogger logger; protected final MainOptions options; protected final Connection con; diff --git a/src/sqlancer/Query.java b/src/sqlancer/Query.java index fbac75548..ba535c223 100644 --- a/src/sqlancer/Query.java +++ b/src/sqlancer/Query.java @@ -1,7 +1,6 @@ package sqlancer; import java.sql.SQLException; -import java.util.Collection; public abstract class Query { @@ -24,7 +23,7 @@ public abstract class Query { */ public abstract boolean execute(GlobalState globalState, String... fills) throws SQLException; - public abstract Collection getExpectedErrors(); + public abstract ExpectedErrors getExpectedErrors(); @Override public String toString() { diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java index e0735463b..9fd1b35e2 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/QueryAdapter.java @@ -4,31 +4,26 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collection; public class QueryAdapter extends Query { private final String query; - private final Collection expectedErrors; + private final ExpectedErrors expectedErrors; private final boolean couldAffectSchema; public QueryAdapter(String query) { - this(query, new ArrayList<>()); + this(query, new ExpectedErrors()); } public QueryAdapter(String query, boolean couldAffectSchema) { - this(query, new ArrayList<>(), couldAffectSchema); + this(query, new ExpectedErrors(), couldAffectSchema); } - public QueryAdapter(String query, Collection expectedErrors) { - this.query = canonicalizeString(query); - this.expectedErrors = expectedErrors; - this.couldAffectSchema = false; - checkQueryString(); + public QueryAdapter(String query, ExpectedErrors expectedErrors) { + this(query, expectedErrors, false); } - public QueryAdapter(String query, Collection expectedErrors, boolean couldAffectSchema) { + public QueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema) { this.query = canonicalizeString(query); this.expectedErrors = expectedErrors; this.couldAffectSchema = couldAffectSchema; @@ -84,14 +79,7 @@ public boolean execute(GlobalState globalState, String... fills) throws SQ } public void checkException(Exception e) throws AssertionError { - boolean isExcluded = false; - for (String expectedError : expectedErrors) { - if (e.getMessage().contains(expectedError)) { - isExcluded = true; - break; - } - } - if (!isExcluded) { + if (!expectedErrors.errorIsExpected(e.getMessage())) { throw new AssertionError(query, e); } } @@ -133,7 +121,7 @@ public boolean couldAffectSchema() { } @Override - public Collection getExpectedErrors() { + public ExpectedErrors getExpectedErrors() { return expectedErrors; } diff --git a/src/sqlancer/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/TernaryLogicPartitioningOracleBase.java index dca63bb45..76bc9b31d 100644 --- a/src/sqlancer/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/TernaryLogicPartitioningOracleBase.java @@ -1,8 +1,5 @@ package sqlancer; -import java.util.HashSet; -import java.util.Set; - import sqlancer.gen.ExpressionGenerator; /** @@ -20,7 +17,7 @@ public abstract class TernaryLogicPartitioningOracleBase implements TestOr protected E isNullPredicate; protected final S state; - protected final Set errors = new HashSet<>(); + protected final ExpectedErrors errors = new ExpectedErrors(); protected TernaryLogicPartitioningOracleBase(S state) { this.state = state; diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index f2e1c2b3a..152c84bbd 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -8,13 +8,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.AbstractAction; import sqlancer.CompositeTestOracle; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -452,8 +451,8 @@ public String getDBMSName() { return "citus"; } - private static Set getCitusErrors() { - Set errors = new HashSet<>(); + private static ExpectedErrors getCitusErrors() { + ExpectedErrors errors = new ExpectedErrors(); CitusCommon.addCitusErrors(errors); return errors; } diff --git a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java index 6d264763a..6271ae5cd 100644 --- a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java @@ -1,8 +1,8 @@ package sqlancer.citus.gen; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.postgres.PostgresGlobalState; @@ -21,7 +21,7 @@ public static Query create(PostgresTable randomTable, PostgresGlobalState global } @Override - public List getActions(Set errors) { + public List getActions(ExpectedErrors errors) { List action = super.getActions(errors); CitusCommon.addCitusErrors(errors); action.remove(Action.ALTER_COLUMN_SET_STATISTICS); diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index def1f8690..7e1a3f14a 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -1,8 +1,6 @@ package sqlancer.citus.gen; -import java.util.Collection; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.gen.PostgresCommon; @@ -12,7 +10,7 @@ public final class CitusCommon { private CitusCommon() { } - public static void addCitusErrors(Collection errors) { + public static void addCitusErrors(ExpectedErrors errors) { errors.add("recursive CTEs are not supported in distributed queries"); errors.add("could not run distributed query with GROUPING SETS, CUBE, or ROLLUP"); errors.add("Subqueries in HAVING cannot refer to outer query"); @@ -68,7 +66,7 @@ public static void addCitusErrors(Collection errors) { } public static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, - Set errors) { + ExpectedErrors errors) { PostgresCommon.addTableConstraint(sb, table, globalState, errors); CitusCommon.addCitusErrors(errors); } diff --git a/src/sqlancer/citus/gen/CitusDeleteGenerator.java b/src/sqlancer/citus/gen/CitusDeleteGenerator.java index 771480eaf..558372d23 100644 --- a/src/sqlancer/citus/gen/CitusDeleteGenerator.java +++ b/src/sqlancer/citus/gen/CitusDeleteGenerator.java @@ -1,7 +1,6 @@ package sqlancer.citus.gen; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresDeleteGenerator; @@ -13,7 +12,7 @@ private CitusDeleteGenerator() { public static Query create(PostgresGlobalState globalState) { Query deleteQuery = PostgresDeleteGenerator.create(globalState); - Set errors = (Set) deleteQuery.getExpectedErrors(); + ExpectedErrors errors = deleteQuery.getExpectedErrors(); CitusCommon.addCitusErrors(errors); return deleteQuery; } diff --git a/src/sqlancer/citus/gen/CitusIndexGenerator.java b/src/sqlancer/citus/gen/CitusIndexGenerator.java index ca51ec2ec..583895136 100644 --- a/src/sqlancer/citus/gen/CitusIndexGenerator.java +++ b/src/sqlancer/citus/gen/CitusIndexGenerator.java @@ -1,7 +1,6 @@ package sqlancer.citus.gen; -import java.util.HashSet; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresInsertGenerator; @@ -13,7 +12,7 @@ private CitusIndexGenerator() { public static Query generate(PostgresGlobalState globalState) { Query createIndexQuery = PostgresInsertGenerator.insert(globalState); - HashSet errors = (HashSet) createIndexQuery.getExpectedErrors(); + ExpectedErrors errors = createIndexQuery.getExpectedErrors(); CitusCommon.addCitusErrors(errors); return createIndexQuery; } diff --git a/src/sqlancer/citus/gen/CitusInsertGenerator.java b/src/sqlancer/citus/gen/CitusInsertGenerator.java index 28334177f..58c72d620 100644 --- a/src/sqlancer/citus/gen/CitusInsertGenerator.java +++ b/src/sqlancer/citus/gen/CitusInsertGenerator.java @@ -1,7 +1,6 @@ package sqlancer.citus.gen; -import java.util.HashSet; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresInsertGenerator; @@ -13,7 +12,7 @@ private CitusInsertGenerator() { public static Query insert(PostgresGlobalState globalState) { Query insertQuery = PostgresInsertGenerator.insert(globalState); - HashSet errors = (HashSet) insertQuery.getExpectedErrors(); + ExpectedErrors errors = insertQuery.getExpectedErrors(); CitusCommon.addCitusErrors(errors); return insertQuery; } diff --git a/src/sqlancer/citus/gen/CitusSetGenerator.java b/src/sqlancer/citus/gen/CitusSetGenerator.java index 26baf43a9..6ed900b09 100644 --- a/src/sqlancer/citus/gen/CitusSetGenerator.java +++ b/src/sqlancer/citus/gen/CitusSetGenerator.java @@ -1,7 +1,6 @@ package sqlancer.citus.gen; -import java.util.Collection; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresSetGenerator; @@ -13,7 +12,7 @@ private CitusSetGenerator() { public static Query create(PostgresGlobalState globalState) { Query setQuery = PostgresSetGenerator.create(globalState); - Collection errors = setQuery.getExpectedErrors(); + ExpectedErrors errors = setQuery.getExpectedErrors(); CitusCommon.addCitusErrors(errors); return setQuery; } diff --git a/src/sqlancer/citus/gen/CitusUpdateGenerator.java b/src/sqlancer/citus/gen/CitusUpdateGenerator.java index 9604a12f0..32969d3a9 100644 --- a/src/sqlancer/citus/gen/CitusUpdateGenerator.java +++ b/src/sqlancer/citus/gen/CitusUpdateGenerator.java @@ -1,7 +1,6 @@ package sqlancer.citus.gen; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresUpdateGenerator; @@ -13,7 +12,7 @@ private CitusUpdateGenerator() { public static Query create(PostgresGlobalState globalState) { Query updateQuery = PostgresUpdateGenerator.create(globalState); - Set errors = (Set) updateQuery.getExpectedErrors(); + ExpectedErrors errors = updateQuery.getExpectedErrors(); CitusCommon.addCitusErrors(errors); return updateQuery; } diff --git a/src/sqlancer/citus/gen/CitusViewGenerator.java b/src/sqlancer/citus/gen/CitusViewGenerator.java index 65dc2fbea..fd0e9cafa 100644 --- a/src/sqlancer/citus/gen/CitusViewGenerator.java +++ b/src/sqlancer/citus/gen/CitusViewGenerator.java @@ -1,7 +1,6 @@ package sqlancer.citus.gen; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresViewGenerator; @@ -13,7 +12,7 @@ private CitusViewGenerator() { public static Query create(PostgresGlobalState globalState) { Query viewQuery = PostgresViewGenerator.create(globalState); - Set errors = (Set) viewQuery.getExpectedErrors(); + ExpectedErrors errors = viewQuery.getExpectedErrors(); CitusCommon.addCitusErrors(errors); return viewQuery; } diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 4aeafd6d5..c55a7e169 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -1,13 +1,13 @@ package sqlancer.clickhouse; -import java.util.Set; +import sqlancer.ExpectedErrors; public final class ClickHouseErrors { private ClickHouseErrors() { } - public static void addExpectedExpressionErrors(Set errors) { + public static void addExpectedExpressionErrors(ExpectedErrors errors) { // errors.add("Illegal type (String) of argument of function not"); // errors.add("Illegal type String of column for constant filter. Must be UInt8 or Nullable(UInt8)"); // errors.add("Illegal type Int32 of column for constant filter. Must be UInt8 or Nullable(UInt8)"); @@ -37,19 +37,19 @@ public static void addExpectedExpressionErrors(Set errors) { errors.add("argument of function"); } - public static void addExpressionHavingErrors(Set errors) { + public static void addExpressionHavingErrors(ExpectedErrors errors) { errors.add("Memory limit"); } - public static void addQueryErrors(Set errors) { + public static void addQueryErrors(ExpectedErrors errors) { errors.add("Memory limit"); } - public static void addGroupingErrors(Set errors) { + public static void addGroupingErrors(ExpectedErrors errors) { errors.add("Memory limit"); } - public static void addTableManipulationErrors(Set errors) { + public static void addTableManipulationErrors(ExpectedErrors errors) { errors.add("Memory limit"); errors.add("Directory for table data"); errors.add("Directory not empty"); diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 9d56b3c52..83e1fc997 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -1,11 +1,10 @@ package sqlancer.clickhouse.gen; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; @@ -17,7 +16,7 @@ public class ClickHouseInsertGenerator extends AbstractInsertGenerator { private final ClickHouseGlobalState globalState; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private final ClickHouseExpressionGenerator gen; public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index 3eb0e1537..de42ced6e 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -1,11 +1,10 @@ package sqlancer.clickhouse.gen; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import ru.yandex.clickhouse.domain.ClickHouseDataType; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -37,7 +36,7 @@ public ClickHouseTableGenerator(String tableName, ClickHouseProvider.ClickHouseG public static Query createTableStatement(String tableName, ClickHouseProvider.ClickHouseGlobalState globalState) { ClickHouseTableGenerator chTableGenerator = new ClickHouseTableGenerator(tableName, globalState); chTableGenerator.start(); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); ClickHouseErrors.addTableManipulationErrors(errors); return new QueryAdapter(chTableGenerator.sb.toString(), errors, true); } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 87cb561fd..a7993c04f 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -1,13 +1,13 @@ package sqlancer.cockroachdb; -import java.util.Set; +import sqlancer.ExpectedErrors; public final class CockroachDBErrors { private CockroachDBErrors() { } - public static void addExpressionErrors(Set errors) { + public static void addExpressionErrors(ExpectedErrors errors) { errors.add(" non-streaming operator encountered when vectorize=auto"); if (CockroachDBBugs.bug46915) { @@ -182,7 +182,7 @@ public static void addExpressionErrors(Set errors) { addArrayErrors(errors); } - private static void addArrayErrors(Set errors) { + private static void addArrayErrors(ExpectedErrors errors) { // arrays errors.add("cannot determine type of empty array"); errors.add("unknown signature: max(unknown[])"); @@ -232,16 +232,16 @@ private static void addArrayErrors(Set errors) { errors.add("to be of type unknown[]"); // IF with null array } - private static void addIntervalTypeErrors(Set errors) { + private static void addIntervalTypeErrors(ExpectedErrors errors) { errors.add("overflow during Encode"); errors.add("as type interval"); } - private static void addJoinTypes(Set errors) { + private static void addJoinTypes(ExpectedErrors errors) { errors.add("JOIN/USING types"); } - private static void addGroupByErrors(Set errors) { + private static void addGroupByErrors(ExpectedErrors errors) { errors.add("non-integer constant in GROUP BY"); // https://github.com/cockroachdb/cockroach/pull/46649 -> aggregates on NULL are @@ -265,7 +265,7 @@ private static void addGroupByErrors(Set errors) { } - private static void addFunctionErrors(Set errors) { + private static void addFunctionErrors(ExpectedErrors errors) { // functions errors.add("abs of min integer value (-9223372036854775808) not defined"); // ABS errors.add("the input string must not be empty"); // ASCII @@ -284,7 +284,7 @@ private static void addFunctionErrors(Set errors) { errors.add("must be greater than zero"); // split_part } - public static void addTransactionErrors(Set errors) { + public static void addTransactionErrors(ExpectedErrors errors) { errors.add("current transaction is aborted"); } diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index f10e6068d..6ed1efcfb 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -5,12 +5,10 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; @@ -58,12 +56,13 @@ public enum Action { TRANSACTION((g) -> { String s = Randomly.fromOptions("BEGIN", "ROLLBACK", "COMMIT"); return new QueryAdapter(s, - Arrays.asList("there is no transaction in progress", "there is already a transaction in progress", - "current transaction is aborted", "does not exist" /* interleaved indexes */)); + ExpectedErrors.from("there is no transaction in progress", + "there is already a transaction in progress", "current transaction is aborted", + "does not exist" /* interleaved indexes */)); }), // EXPLAIN((g) -> { StringBuilder sb = new StringBuilder("EXPLAIN "); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); if (Randomly.getBoolean()) { sb.append("("); sb.append(Randomly.nonEmptySubset("VERBOSE", "TYPES", "OPT", "DISTSQL", "VEC").stream() @@ -81,7 +80,7 @@ public enum Action { SCRUB((g) -> new QueryAdapter( "EXPERIMENTAL SCRUB table " + g.getSchema().getRandomTable(t -> !t.isView()).getName(), // https://github.com/cockroachdb/cockroach/issues/46401 - Arrays.asList("scrub-fk: column \"t.rowid\" does not exist", + ExpectedErrors.from("scrub-fk: column \"t.rowid\" does not exist", "check-constraint: cannot access temporary tables of other sessions" /* * https:// github. com/ * cockroachdb / cockroach @@ -98,7 +97,7 @@ public enum Action { } else { sb.append(" SPLIT AT VALUES (NULL);"); } - return new QueryAdapter(sb.toString(), Arrays.asList("must be of type")); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("must be of type")); }); private final QueryProvider queryProvider; diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java index 954a2de36..1eec56f09 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java @@ -1,9 +1,8 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -55,7 +54,7 @@ public static Query comment(CockroachDBGlobalState globalState) { sb.append(" IS '"); sb.append(globalState.getRandomly().getString().replace("'", "''")); sb.append("'"); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addTransactionErrors(errors); return new QueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java index 9def2dcbf..bc9b2cc2f 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java @@ -1,7 +1,6 @@ package sqlancer.cockroachdb.gen; -import java.util.Arrays; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -25,7 +24,7 @@ public static Query create(CockroachDBGlobalState globalState) { sb.append(randomTable.getName()); return new QueryAdapter(sb.toString(), - Arrays.asList("current transaction is aborted, commands ignored until end of transaction block", + ExpectedErrors.from("current transaction is aborted, commands ignored until end of transaction block", "ERROR: unable to encode table key: *tree.DArray" /* * https://github.com/cockroachdb/cockroach/ * issues/46964 diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index 3cae2cd19..28e3328e5 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -1,8 +1,6 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -18,7 +16,7 @@ private CockroachDBDeleteGenerator() { } public static Query delete(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append("DELETE FROM "); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java index 5cd9cf43e..50d9b558f 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java @@ -1,10 +1,9 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -20,7 +19,7 @@ private CockroachDBInsertGenerator() { } public static Query insert(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addExpressionErrors(errors); // e.g., caused by computed columns errors.add("violates not-null constraint"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 1d83d75f3..fc8f8bca3 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -1,9 +1,8 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; import java.util.function.Function; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -51,7 +50,7 @@ public static Query create(CockroachDBGlobalState globalState) { } else { sb.append(s.f.apply(globalState)); } - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addTransactionErrors(errors); errors.add("setting updated but timed out waiting to read new value"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index 341db2c42..49bc0907e 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -1,9 +1,8 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; import java.util.function.Function; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -52,7 +51,7 @@ public static Query create(CockroachDBGlobalState globalState) { sb.append(s); sb.append("="); sb.append(s.f.apply(globalState)); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addTransactionErrors(errors); return new QueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java index a2ca73ccc..5666a0603 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java @@ -1,8 +1,6 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -20,7 +18,7 @@ private enum Option { } public static Query show(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); switch (Randomly.fromOptions(Option.values())) { case EXPERIMENTAL_FINGERPRINTS: diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index d8ec86ba9..d1c13b360 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -1,8 +1,6 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -15,7 +13,7 @@ private CockroachDBTruncateGenerator() { // https://www.cockroachlabs.com/docs/v19.2/truncate.html public static Query truncate(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("is interleaved by table"); errors.add("is referenced by foreign key"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index 5adcf62f1..a964d16f0 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -1,9 +1,8 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -20,7 +19,7 @@ private CockroachDBUpdateGenerator() { } public static Query gen(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java index a2cadbeed..3740707f3 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java @@ -1,8 +1,6 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -29,7 +27,7 @@ public static Query generate(CockroachDBGlobalState globalState) { } sb.append(") AS "); sb.append(CockroachDBRandomQuerySynthesizer.generate(globalState, nrColumns).getQueryString()); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addExpressionErrors(errors); CockroachDBErrors.addTransactionErrors(errors); errors.add("value type unknown cannot be used for table columns"); diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 14f8e814e..b97fb73f2 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -4,9 +4,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.NoRECBase; @@ -98,7 +98,7 @@ public static List getJoins(List t } private int getOptimizedResult(CockroachDBExpression whereCondition, List tableList, - Set errors, List joinExpressions) throws SQLException { + ExpectedErrors errors, List joinExpressions) throws SQLException { CockroachDBSelect select = new CockroachDBSelect(); CockroachDBColumn c = new CockroachDBColumn("COUNT(*)", null, false, false); select.setFetchColumns(Arrays.asList(new CockroachDBColumnReference(c))); @@ -118,7 +118,7 @@ private int getOptimizedResult(CockroachDBExpression whereCondition, List tableList, - Set errors, List joinList) throws SQLException { + ExpectedErrors errors, List joinList) throws SQLException { String fromString = tableList.stream().map(t -> ((CockroachDBTableReference) t).getTable().getName()) .collect(Collectors.joining(", ")); if (!tableList.isEmpty() && !joinList.isEmpty()) { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index 45bb06930..947eefd53 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -3,14 +3,13 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import org.postgresql.util.PSQLException; import sqlancer.ComparatorHelper; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -39,7 +38,7 @@ public class CockroachDBTLPAggregateOracle implements TestOracle { private final CockroachDBGlobalState state; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private CockroachDBExpressionGenerator gen; private String firstResult; private String secondResult; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java index e65bff7ab..5f1200e7a 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java @@ -3,12 +3,11 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.ComparatorHelper; +import sqlancer.ExpectedErrors; import sqlancer.Randomly; import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBErrors; @@ -33,7 +32,7 @@ public class CockroachDBTLPJoinOracle implements TestOracle { final CockroachDBGlobalState state; - final Set errors = new HashSet<>(); + final ExpectedErrors errors = new ExpectedErrors(); CockroachDBSchema s; CockroachDBTables targetTables; diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 61c940f2f..d6b8685d8 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -1,13 +1,13 @@ package sqlancer.duckdb; -import java.util.Set; +import sqlancer.ExpectedErrors; public final class DuckDBErrors { private DuckDBErrors() { } - public static void addExpressionErrors(Set errors) { + public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Could not convert string"); errors.add("ORDER term out of range - should be between "); errors.add("You might need to add explicit type casts."); @@ -49,7 +49,7 @@ public static void addExpressionErrors(Set errors) { errors.add("Contents of view were altered: types don't match!"); } - private static void addRegexErrors(Set errors) { + private static void addRegexErrors(ExpectedErrors errors) { errors.add("missing ]"); errors.add("missing )"); errors.add("invalid escape sequence"); @@ -61,7 +61,7 @@ private static void addRegexErrors(Set errors) { errors.add("width is not integer"); } - private static void addFunctionErrors(Set errors) { + private static void addFunctionErrors(ExpectedErrors errors) { errors.add("SUBSTRING cannot handle negative offsets"); errors.add("is undefined outside [-1,1]"); // ACOS etc errors.add("invalid type specifier"); // PRINTF @@ -74,7 +74,7 @@ private static void addFunctionErrors(Set errors) { errors.add("Could not choose a best candidate function for the function call"); // monthname } - public static void addInsertErrors(Set errors) { + public static void addInsertErrors(ExpectedErrors errors) { errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); errors.add("duplicate key value violates primary key or unique constraint"); @@ -89,7 +89,7 @@ public static void addInsertErrors(Set errors) { // the table has a primary key } - public static void addGroupByErrors(Set errors) { + public static void addGroupByErrors(ExpectedErrors errors) { errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY term out of range"); } diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 6ff8b35b8..880f29e27 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -3,12 +3,11 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.HashSet; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.AbstractAction; import sqlancer.CompositeTestOracle; +import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; @@ -43,7 +42,7 @@ public enum Action implements AbstractAction { UPDATE(DuckDBUpdateGenerator::getQuery), // CREATE_VIEW(DuckDBViewGenerator::generate), // EXPLAIN((g) -> { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); DuckDBErrors.addExpressionErrors(errors); DuckDBErrors.addGroupByErrors(errors); return new QueryAdapter( diff --git a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java index 01d65213f..c6f612bce 100644 --- a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java @@ -1,8 +1,6 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -22,7 +20,7 @@ enum Action { } public static Query getQuery(DuckDBGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add(" does not have a column with name \"rowid\""); errors.add("Table does not contain column rowid referenced in alter statement"); StringBuilder sb = new StringBuilder("ALTER TABLE "); diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index 248bc40c2..8ff29312d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -1,8 +1,6 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -18,7 +16,7 @@ private DuckDBDeleteGenerator() { public static Query generate(DuckDBGlobalState globalState) { StringBuilder sb = new StringBuilder("DELETE FROM "); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 44bcae79a..6429e2ae4 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -1,9 +1,8 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -20,7 +19,7 @@ private DuckDBIndexGenerator() { } public static Query getQuery(DuckDBGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); sb.append("CREATE "); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 99369e9bf..e15b9c355 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -1,10 +1,9 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -18,7 +17,7 @@ public class DuckDBInsertGenerator extends AbstractInsertGenerator { private final DuckDBGlobalState globalState; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); public DuckDBInsertGenerator(DuckDBGlobalState globalState) { this.globalState = globalState; diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index a048f6646..eb394be55 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -1,11 +1,10 @@ package sqlancer.duckdb.gen; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -22,7 +21,7 @@ public class DuckDBTableGenerator { public Query getQuery(DuckDBGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); String tableName = globalState.getSchema().getFreeTableName(); sb.append("CREATE TABLE "); diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 5c0b3a27e..0a2e2ed19 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -1,9 +1,8 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -22,7 +21,7 @@ private DuckDBUpdateGenerator() { public static Query getQuery(DuckDBGlobalState globalState) { StringBuilder sb = new StringBuilder("UPDATE "); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); diff --git a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java index e2ac4ccaf..f019e444d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java @@ -1,8 +1,6 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -30,7 +28,7 @@ public static Query generate(DuckDBGlobalState globalState) { } sb.append(") AS "); sb.append(DuckDBToStringVisitor.asString(DuckDBRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); DuckDBErrors.addExpressionErrors(errors); DuckDBErrors.addGroupByErrors(errors); return new QueryAdapter(sb.toString(), errors, true); diff --git a/src/sqlancer/gen/AbstractGenerator.java b/src/sqlancer/gen/AbstractGenerator.java index 6ed0c68e9..d4fe512f6 100644 --- a/src/sqlancer/gen/AbstractGenerator.java +++ b/src/sqlancer/gen/AbstractGenerator.java @@ -1,14 +1,12 @@ package sqlancer.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; public abstract class AbstractGenerator { - protected final Set errors = new HashSet<>(); + protected final ExpectedErrors errors = new ExpectedErrors(); protected final StringBuilder sb = new StringBuilder(); protected boolean canAffectSchema; diff --git a/src/sqlancer/mariadb/MariaDBErrors.java b/src/sqlancer/mariadb/MariaDBErrors.java index f1e2f581f..e3751f05e 100644 --- a/src/sqlancer/mariadb/MariaDBErrors.java +++ b/src/sqlancer/mariadb/MariaDBErrors.java @@ -1,13 +1,13 @@ package sqlancer.mariadb; -import java.util.List; +import sqlancer.ExpectedErrors; public final class MariaDBErrors { private MariaDBErrors() { } - public static void addInsertErrors(List errors) { + public static void addInsertErrors(ExpectedErrors errors) { errors.add("Out of range"); errors.add("Duplicate entry"); // violates UNIQUE constraint errors.add("cannot be null"); // violates NOT NULL constraint diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index d4b396104..3728eb5ab 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -1,8 +1,8 @@ package sqlancer.mariadb.gen; -import java.util.ArrayList; import java.util.List; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -17,7 +17,7 @@ private MariaDBIndexGenerator() { } public static Query generate(MariaDBSchema s) { - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE "); errors.add("Key/Index cannot be defined on a virtual generated column"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java index b27e127c1..8345eea0d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java @@ -1,8 +1,6 @@ package sqlancer.mariadb.gen; -import java.util.ArrayList; -import java.util.List; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -34,7 +32,7 @@ public static Query insert(MariaDBSchema s, Randomly r) { } } sb.append(")"); - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); MariaDBErrors.addInsertErrors(errors); return new QueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java index 88ca4893b..e53df8862 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -22,7 +23,7 @@ public class MariaDBTableGenerator { private PrimaryKeyState primaryKeyState = Randomly.fromOptions(PrimaryKeyState.values()); private final List columnNames = new ArrayList<>(); private final Randomly r; - private final List errors = new ArrayList<>(); + private final ExpectedErrors errors = new ExpectedErrors(); public MariaDBTableGenerator(String tableName, Randomly r, MariaDBSchema newSchema) { this.tableName = tableName; diff --git a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java index 3e2e0f9ee..e000b7d80 100644 --- a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java @@ -1,8 +1,6 @@ package sqlancer.mariadb.gen; -import java.util.ArrayList; -import java.util.List; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -40,7 +38,7 @@ public static Query update(MariaDBSchema s, Randomly r) { } // [WHERE where_condition] [ORDER BY ...] [LIMIT row_count] } - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); MariaDBErrors.addInsertErrors(errors); return new QueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index c2893f72f..9fbaa3bc5 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -1,13 +1,13 @@ package sqlancer.mysql; -import java.util.Set; +import sqlancer.ExpectedErrors; public final class MySQLErrors { private MySQLErrors() { } - public static void addExpressionErrors(Set errors) { + public static void addExpressionErrors(ExpectedErrors errors) { errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) errors.add("is not valid for CHARACTER SET"); } diff --git a/src/sqlancer/mysql/gen/MySQLAlterTable.java b/src/sqlancer/mysql/gen/MySQLAlterTable.java index a6ef3855e..4b362f523 100644 --- a/src/sqlancer/mysql/gen/MySQLAlterTable.java +++ b/src/sqlancer/mysql/gen/MySQLAlterTable.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -61,9 +62,8 @@ PACK_KEYS, RENAME("doesn't exist", "already exists"), /* WITH_WITHOUT_VALIDATION } private Query create() { - List errors = new ArrayList<>( - Arrays.asList("does not support the create option", "doesn't have this option", - "is not supported for this operation", "Data truncation", "Specified key was too long")); + ExpectedErrors errors = ExpectedErrors.from("does not support the create option", "doesn't have this option", + "is not supported for this operation", "Data truncation", "Specified key was too long"); errors.add("Data truncated for functional index "); sb.append("ALTER TABLE "); MySQLTable table = schema.getRandomTable(); diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index 83c591d25..3234d102e 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -1,9 +1,8 @@ package sqlancer.mysql.gen; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -28,7 +27,7 @@ public static Query delete(MySQLGlobalState globalState) { private Query generate() { MySQLTable randomTable = globalState.getSchema().getRandomTable(); MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); sb.append("DELETE"); if (Randomly.getBoolean()) { sb.append(" LOW_PRIORITY"); diff --git a/src/sqlancer/mysql/gen/MySQLDropIndex.java b/src/sqlancer/mysql/gen/MySQLDropIndex.java index 9e3278fa5..14919a550 100644 --- a/src/sqlancer/mysql/gen/MySQLDropIndex.java +++ b/src/sqlancer/mysql/gen/MySQLDropIndex.java @@ -1,7 +1,6 @@ package sqlancer.mysql.gen; -import java.util.Arrays; - +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -45,8 +44,9 @@ public static Query generate(MySQLGlobalState globalState) { sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); } return new QueryAdapter(sb.toString(), - Arrays.asList("LOCK=NONE is not supported", "ALGORITHM=INPLACE is not supported", "Data truncation", - "Data truncated for functional index", "A primary key index cannot be invisible")); + ExpectedErrors.from("LOCK=NONE is not supported", "ALGORITHM=INPLACE is not supported", + "Data truncation", "Data truncated for functional index", + "A primary key index cannot be invisible")); } } diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 5ab1d3a3b..4dd478b3a 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -1,11 +1,10 @@ package sqlancer.mysql.gen; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -19,7 +18,7 @@ public class MySQLInsertGenerator { private final MySQLTable table; private final StringBuilder sb = new StringBuilder(); boolean canFail; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; public MySQLInsertGenerator(MySQLGlobalState globalState) { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 775353361..f5f01bdce 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -42,7 +43,7 @@ public static Query generate(String tableName, Randomly r, MySQLSchema schema) { } private Query create() { - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); sb.append("CREATE"); // TODO support temporary tables in the schema @@ -86,7 +87,7 @@ private Query create() { } - private void addCommonErrors(List list) { + private void addCommonErrors(ExpectedErrors list) { list.add("The storage engine for the table doesn't support"); list.add("doesn't have this option"); list.add("must include all columns"); diff --git a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java index 808622b29..b00c97d75 100644 --- a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java @@ -1,7 +1,6 @@ package sqlancer.mysql.gen; -import java.util.Arrays; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; @@ -14,7 +13,7 @@ private MySQLTruncateTableGenerator() { public static Query generate(MySQLGlobalState globalState) { StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); sb.append(globalState.getSchema().getRandomTable().getName()); - return new QueryAdapter(sb.toString(), Arrays.asList("doesn't have this option")); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("doesn't have this option")); } } diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index e935e7d9c..cce9d089c 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -1,9 +1,8 @@ package sqlancer.mysql.gen.datadef; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -38,7 +37,7 @@ public static Query create(MySQLGlobalState globalState) { } public Query create() { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); MySQLErrors.addExpressionErrors(errors); sb.append("CREATE "); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 51dca01f6..3bcc618df 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -1,9 +1,8 @@ package sqlancer.postgres.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -76,7 +75,7 @@ private enum Attribute { } }; - public List getActions(Set errors) { + public List getActions(ExpectedErrors errors) { PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonInsertUpdateErrors(errors); PostgresCommon.addCommonTableErrors(errors); @@ -117,7 +116,7 @@ public List getActions(Set errors) { } public Query generate() { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); int i = 0; List action = getActions(errors); StringBuilder sb = new StringBuilder(); diff --git a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java index 7a4a64427..0c3de03db 100644 --- a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java @@ -1,8 +1,8 @@ package sqlancer.postgres.gen; -import java.util.Arrays; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -37,7 +37,7 @@ public static Query create(PostgresGlobalState globalState) { } } // FIXME: bug in postgres? - return new QueryAdapter(sb.toString(), Arrays.asList("deadlock")); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("deadlock")); } } diff --git a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java index 00f364bdc..6bbb8e0b1 100644 --- a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java @@ -1,8 +1,6 @@ package sqlancer.postgres.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -15,7 +13,7 @@ private PostgresClusterGenerator() { } public static Query create(PostgresGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("there is no previously clustered index for table"); errors.add("cannot cluster a partitioned table"); errors.add("access method does not support clustering"); diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index f45180467..fd1aaa99c 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -3,11 +3,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Function; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; @@ -22,7 +22,7 @@ public final class PostgresCommon { private PostgresCommon() { } - public static void addCommonFetchErrors(Set errors) { + public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); errors.add("but it cannot be referenced from this part of the query"); errors.add("missing FROM-clause entry for table"); @@ -30,12 +30,12 @@ public static void addCommonFetchErrors(Set errors) { errors.add("canceling statement due to statement timeout"); } - public static void addCommonTableErrors(Set errors) { + public static void addCommonTableErrors(ExpectedErrors errors) { errors.add("is not commutative"); // exclude errors.add("operator requires run-time type coercion"); // exclude } - public static void addCommonExpressionErrors(Set errors) { + public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); @@ -66,7 +66,7 @@ public static void addCommonExpressionErrors(Set errors) { addCommonRegexExpressionErrors(errors); } - private static void addToCharFunctionErrors(Set errors) { + private static void addToCharFunctionErrors(ExpectedErrors errors) { errors.add("multiple decimal points"); errors.add("and decimal point together"); errors.add("multiple decimal points"); @@ -80,14 +80,14 @@ private static void addToCharFunctionErrors(Set errors) { errors.add("is not a number"); } - private static void addBitStringOperationErrors(Set errors) { + private static void addBitStringOperationErrors(ExpectedErrors errors) { errors.add("cannot XOR bit strings of different sizes"); errors.add("cannot AND bit strings of different sizes"); errors.add("cannot OR bit strings of different sizes"); errors.add("must be type boolean, not type text"); } - private static void addFunctionErrors(Set errors) { + private static void addFunctionErrors(ExpectedErrors errors) { errors.add("out of valid range"); // get_bit/get_byte errors.add("cannot take logarithm of a negative number"); errors.add("cannot take logarithm of zero"); @@ -101,18 +101,18 @@ private static void addFunctionErrors(Set errors) { errors.add("invalid mask length"); // set_masklen } - private static void addCommonRegexExpressionErrors(Set errors) { + private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { errors.add("is not a valid hexadecimal digit"); } - public static void addCommonRangeExpressionErrors(Set errors) { + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { errors.add("range lower bound must be less than or equal to range upper bound"); errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); } - public static void addCommonInsertUpdateErrors(Set errors) { + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { errors.add("value too long for type character"); errors.add("not found in view targetlist"); } @@ -219,7 +219,7 @@ private enum StorageParameters { } } - public static void generateWith(StringBuilder sb, PostgresGlobalState globalState, Set errors) { + public static void generateWith(StringBuilder sb, PostgresGlobalState globalState, ExpectedErrors errors) { if (Randomly.getBoolean()) { sb.append(" WITH ("); ArrayList values = new ArrayList<>(Arrays.asList(StorageParameters.values())); @@ -241,7 +241,7 @@ public static void generateWith(StringBuilder sb, PostgresGlobalState globalStat } public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder sb, PostgresTable table, - PostgresGlobalState globalState, Set errors) { + PostgresGlobalState globalState, ExpectedErrors errors) { // TODO constraint name List tableConstraints = Randomly.nonEmptySubset(TableConstraints.values()); if (excludePrimaryKey) { @@ -258,12 +258,12 @@ public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder } public static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, - Set errors) { + ExpectedErrors errors) { addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); } private static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, - TableConstraints t, Set errors) { + TableConstraints t, ExpectedErrors errors) { List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); List otherColumns; PostgresCommon.addCommonExpressionErrors(errors); @@ -362,7 +362,8 @@ private static void addTableConstraint(StringBuilder sb, PostgresTable table, Po } } - private static void appendIndexParameters(StringBuilder sb, PostgresGlobalState globalState, Set errors) { + private static void appendIndexParameters(StringBuilder sb, PostgresGlobalState globalState, + ExpectedErrors errors) { if (Randomly.getBoolean()) { generateWith(sb, globalState, errors); } @@ -403,7 +404,7 @@ private static void deleteOrUpdateAction(StringBuilder sb) { sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); } - public static void addGroupingErrors(Set errors) { + public static void addGroupingErrors(ExpectedErrors errors) { errors.add("non-integer constant in GROUP BY"); // TODO errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("is not in select list"); diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 1fc592235..3ea96aefc 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -1,8 +1,6 @@ package sqlancer.postgres.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -18,7 +16,7 @@ private PostgresDeleteGenerator() { public static Query create(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); diff --git a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java index 4dce13a48..349f92ade 100644 --- a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java @@ -1,7 +1,6 @@ package sqlancer.postgres.gen; -import java.util.Arrays; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -26,7 +25,7 @@ public static Query create(PostgresGlobalState globalState) { what = Randomly.fromOptions("PLANS", "SEQUENCES"); } sb.append(what); - return new QueryAdapter(sb.toString(), Arrays.asList("cannot run inside a transaction block")) { + return new QueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { @Override public boolean couldAffectSchema() { diff --git a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java index 79930f6cf..f454da256 100644 --- a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java @@ -1,8 +1,8 @@ package sqlancer.postgres.gen; -import java.util.Arrays; import java.util.List; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -44,8 +44,8 @@ public static Query create(PostgresGlobalState globalState) { sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } return new QueryAdapter(sb.toString(), - Arrays.asList("cannot drop desired object(s) because other objects depend on them", "cannot drop index", - "does not exist"), + ExpectedErrors.from("cannot drop desired object(s) because other objects depend on them", + "cannot drop index", "does not exist"), true); } diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 410a970f7..8aee16532 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -1,10 +1,9 @@ package sqlancer.postgres.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -27,7 +26,7 @@ public enum IndexType { } public static Query generate(PostgresGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); sb.append("CREATE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 759249326..a1c2329b9 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -1,10 +1,9 @@ package sqlancer.postgres.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -21,7 +20,7 @@ private PostgresInsertGenerator() { public static Query insert(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("cannot insert into column"); PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonInsertUpdateErrors(errors); diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index cd3480512..b443cdbe3 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -1,9 +1,9 @@ package sqlancer.postgres.gen; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -21,7 +21,7 @@ private enum Scope { } public static Query create(PostgresGlobalState globalState) { - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("could not create unique index"); // CONCURRENT INDEX StringBuilder sb = new StringBuilder(); sb.append("REINDEX"); diff --git a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java index e21d456d1..a5c384c0d 100644 --- a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java @@ -1,8 +1,6 @@ package sqlancer.postgres.gen; -import java.util.ArrayList; -import java.util.List; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -14,7 +12,7 @@ private PostgresSequenceGenerator() { } public static Query createSequence(PostgresGlobalState globalState) { - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE"); if (Randomly.getBoolean()) { sb.append(" "); diff --git a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java index 90b30d41e..d0a838a0a 100644 --- a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java @@ -1,9 +1,9 @@ package sqlancer.postgres.gen; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -44,7 +44,8 @@ public static Query insert(PostgresGlobalState globalState) { sb.append(randomColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); sb.append(" FROM "); sb.append(randomTable.getName()); - return new QueryAdapter(sb.toString(), Arrays.asList("cannot have more than 8 columns in statistics"), true); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("cannot have more than 8 columns in statistics"), + true); } public static Query remove(PostgresGlobalState globalState) { diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 78ae99907..a5edacc95 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -1,11 +1,10 @@ package sqlancer.postgres.gen; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -27,7 +26,7 @@ public class PostgresTableGenerator { private boolean isTemporaryTable; private final PostgresSchema newSchema; private final List columnsToBeAdded = new ArrayList<>(); - protected final Set errors = new HashSet<>(); + protected final ExpectedErrors errors = new ExpectedErrors(); private final PostgresTable table; private final boolean generateOnlyKnown; private final PostgresGlobalState globalState; diff --git a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java index 2c448e5d8..01dc28d6c 100644 --- a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java @@ -1,8 +1,6 @@ package sqlancer.postgres.gen; -import java.util.ArrayList; -import java.util.List; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -13,7 +11,7 @@ private PostgresTransactionGenerator() { } public static Query executeBegin() { - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("BEGIN"); if (Randomly.getBoolean()) { errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); diff --git a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java index ebcb22ad9..647a59ac4 100644 --- a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java @@ -1,8 +1,8 @@ package sqlancer.postgres.gen; -import java.util.Arrays; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -34,8 +34,8 @@ public static Query create(PostgresGlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } - return new QueryAdapter(sb.toString(), - Arrays.asList("cannot truncate a table referenced in a foreign key constraint", "is not a table")); + return new QueryAdapter(sb.toString(), ExpectedErrors + .from("cannot truncate a table referenced in a foreign key constraint", "is not a table")); } } diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 16d0cb9e3..3386a757e 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -1,10 +1,8 @@ package sqlancer.postgres.gen; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -26,12 +24,12 @@ public static Query create(PostgresGlobalState globalState) { sb.append("UPDATE "); sb.append(randomTable.getName()); sb.append(" SET "); - Set errors = new HashSet<>(Arrays.asList("conflicting key value violates exclusion constraint", + ExpectedErrors errors = ExpectedErrors.from("conflicting key value violates exclusion constraint", "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", " bit string too long", "can only be updated to DEFAULT", "division by zero", "You might need to add explicit type casts.", "invalid regular expression", - "View columns that are not columns of their base relation are not updatable")); + "View columns that are not columns of their base relation are not updatable"); errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced // table multiple times List columns = randomTable.getRandomNonEmptyColumnSubset(); diff --git a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java index 9571af351..a113472da 100644 --- a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java @@ -2,9 +2,9 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -53,7 +53,7 @@ public static Query create(PostgresGlobalState globalState) { } } } - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("VACUUM cannot run inside a transaction block"); errors.add("deadlock"); /* * "FULL" commented out due to https://www.postgresql.org/message-id/CA%2Bu7OA6pL% diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index afc91d341..ccc78f573 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -1,8 +1,6 @@ package sqlancer.postgres.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -17,7 +15,7 @@ private PostgresViewGenerator() { } public static Query create(PostgresGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE"); boolean materialized; boolean recursive = false; diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 69a9b8d9e..708f01df6 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -1,23 +1,22 @@ package sqlancer.sqlite3; import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Set; + +import sqlancer.ExpectedErrors; public final class SQLite3Errors { private SQLite3Errors() { } - public static void addDeleteErrors(List errors) { + public static void addDeleteErrors(ExpectedErrors errors) { // DELETE trigger for a view/table to which colomns were added or deleted errors.add("columns but"); // trigger with on conflict clause errors.add("ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint"); } - public static void addExpectedExpressionErrors(Collection errors) { + public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("FTS expression tree is too large"); errors.add("String or BLOB exceeds size limit"); errors.add("[SQLITE_ERROR] SQL error or missing database (integer overflow)"); @@ -64,7 +63,7 @@ public static void addExpectedExpressionErrors(Collection errors) { } - public static void addMatchQueryErrors(Collection errors) { + public static void addMatchQueryErrors(ExpectedErrors errors) { errors.add("unable to use function MATCH in the requested context"); errors.add("malformed MATCH expression"); errors.add("fts5: syntax error near"); @@ -75,18 +74,18 @@ public static void addMatchQueryErrors(Collection errors) { errors.add("unterminated string"); } - public static void addTableManipulationErrors(List errors) { + public static void addTableManipulationErrors(ExpectedErrors errors) { errors.add("unsupported frame specification"); errors.add("non-deterministic functions prohibited in CHECK constraints"); errors.addAll(Arrays.asList("subqueries prohibited in CHECK constraints", "generated columns cannot be part of the PRIMARY KEY", "must have at least one non-generated column")); } - public static void addQueryErrors(Set errors) { + public static void addQueryErrors(ExpectedErrors errors) { errors.add("ON clause references tables to its right"); } - public static void addInsertNowErrors(List errors) { + public static void addInsertNowErrors(ExpectedErrors errors) { errors.add("non-deterministic use of strftime()"); errors.add("non-deterministic use of time()"); errors.add("non-deterministic use of datetime()"); diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index e4d83cf50..2a49c8075 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -12,6 +12,7 @@ import java.util.List; import sqlancer.AbstractAction; +import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; @@ -154,7 +155,7 @@ public boolean canBeRetried() { sb.append(" noskipscan"); } sb.append("')"); - return new QueryAdapter(sb.toString(), Arrays.asList("no such table")); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("no such table")); } }); @@ -296,7 +297,7 @@ protected TestOracle getTestOracle(SQLite3GlobalState globalState) throws SQLExc private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws SQLException { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { Query q = new QueryAdapter("SELECT * FROM " + table.getName(), - Arrays.asList("needs an odd number of arguments", " requires an even number of arguments", + ExpectedErrors.from("needs an odd number of arguments", " requires an even number of arguments", "generated column loop", "integer overflow", "malformed JSON", "JSON cannot hold BLOB values", "JSON path error", "labels must be TEXT")); if (!q.execute(globalState)) { diff --git a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java index 2933214b1..9a37f7135 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java @@ -1,7 +1,6 @@ package sqlancer.sqlite3.gen; -import java.util.Arrays; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -37,7 +36,7 @@ public static Query generateAnalyze(SQLite3GlobalState globalState) { throw new AssertionError(); } } - return new QueryAdapter(sb.toString(), Arrays.asList("The database file is locked")); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("The database file is locked")); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index 31619e986..f1e46bb67 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -1,10 +1,9 @@ package sqlancer.sqlite3.gen; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -17,7 +16,7 @@ private SQLite3CreateVirtualRtreeTabelGenerator() { } public static Query createTableStatement(String rTreeTableName, SQLite3GlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); List columns = new ArrayList<>(); StringBuilder sb = new StringBuilder("CREATE VIRTUAL TABLE "); sb.append(rTreeTableName); diff --git a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java index 1c0a73115..3ca68b2f6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java @@ -1,10 +1,9 @@ package sqlancer.sqlite3.gen; import java.sql.SQLException; -import java.util.HashSet; -import java.util.Set; import java.util.function.Supplier; +import sqlancer.ExpectedErrors; import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -60,7 +59,7 @@ private enum PragmaAttribute { } private final StringBuilder sb = new StringBuilder(); - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); public void createPragma(String pragmaName, Supplier supplier) { boolean setSchema = Randomly.getBoolean(); diff --git a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java index 44b0fa86c..05f611bf0 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java @@ -1,8 +1,6 @@ package sqlancer.sqlite3.gen; -import java.util.ArrayList; -import java.util.List; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -24,7 +22,7 @@ private enum Target { public static Query executeReindex(SQLite3GlobalState globalState) { SQLite3Schema s = globalState.getSchema(); StringBuilder sb = new StringBuilder("REINDEX"); - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); Target t = Randomly.fromOptions(Target.values()); if (Randomly.getBoolean()) { sb.append(" "); diff --git a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java index 78787b31d..01f381d91 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java @@ -1,7 +1,6 @@ package sqlancer.sqlite3.gen; -import java.util.Arrays; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -18,8 +17,8 @@ public static Query generateCommit(SQLite3GlobalState globalState) { if (Randomly.getBoolean()) { sb.append(" TRANSACTION"); } - return new QueryAdapter(sb.toString(), Arrays.asList("no transaction is active", "The database file is locked", - "FOREIGN KEY constraint failed"), true); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("no transaction is active", + "The database file is locked", "FOREIGN KEY constraint failed"), true); } public static Query generateBeginTransaction(SQLite3GlobalState globalState) { @@ -30,13 +29,13 @@ public static Query generateBeginTransaction(SQLite3GlobalState globalState) { } sb.append(" TRANSACTION;"); return new QueryAdapter(sb.toString(), - Arrays.asList("cannot start a transaction within a transaction", "The database file is locked")); + ExpectedErrors.from("cannot start a transaction within a transaction", "The database file is locked")); } public static Query generateRollbackTransaction(SQLite3GlobalState globalState) { // TODO: could be extended by savepoint return new QueryAdapter("ROLLBACK TRANSACTION;", - Arrays.asList("no transaction is active", "The database file is locked"), true); + ExpectedErrors.from("no transaction is active", "The database file is locked"), true); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java index a6a2af842..d3d595a6b 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java @@ -1,7 +1,6 @@ package sqlancer.sqlite3.gen; -import java.util.Arrays; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -21,8 +20,8 @@ public static Query executeVacuum(SQLite3GlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("temp", "main")); } - return new QueryAdapter(sb.toString(), - Arrays.asList("cannot VACUUM from within a transaction", "cannot VACUUM - SQL statements in progress")); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("cannot VACUUM from within a transaction", + "cannot VACUUM - SQL statements in progress")); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java index 5c5cf8817..20e2e653a 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java @@ -1,8 +1,6 @@ package sqlancer.sqlite3.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -15,7 +13,7 @@ public class SQLite3VirtualFTSTableCommandGenerator { private final StringBuilder sb = new StringBuilder(); private final SQLite3Schema s; private final Randomly r; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); public static Query create(SQLite3GlobalState globalState) { return new SQLite3VirtualFTSTableCommandGenerator(globalState.getSchema(), globalState.getRandomly()) diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java index 9dd8de8c7..30d20be23 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java @@ -1,9 +1,8 @@ package sqlancer.sqlite3.gen.ddl; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -33,7 +32,7 @@ public SQLite3AlterTable(SQLite3GlobalState globalState) { } private Query getQuery(SQLite3Schema s, SQLite3AlterTable alterTable) throws AssertionError { - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("error in view"); errors.add("no such column"); // trigger errors.add("error in trigger"); // trigger diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java index fbc58ba76..6e4044a2d 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java @@ -2,9 +2,9 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -110,7 +110,7 @@ public static Query create(SQLite3GlobalState globalState) throws SQLException { sb.append("END"); return new QueryAdapter(sb.toString(), - Arrays.asList("parser stack overflow", "unsupported frame specification")); + ExpectedErrors.from("parser stack overflow", "unsupported frame specification")); } private static void appendTableNameAndWhen(SQLite3GlobalState globalState, StringBuilder sb, SQLite3Table table) { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java index b1d031918..0d87ded39 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.List; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -45,8 +46,8 @@ public Query create() { } else { createFts5Table(); } - return new QueryAdapter(sb.toString(), Arrays.asList("unrecognized parameter", "unknown tokenizer: ascii"), - true); + return new QueryAdapter(sb.toString(), + ExpectedErrors.from("unrecognized parameter", "unknown tokenizer: ascii"), true); } private void createFts4Table() { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java index 043d0dcbb..d35ba6d14 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java @@ -1,7 +1,6 @@ package sqlancer.sqlite3.gen.ddl; -import java.util.Arrays; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -23,7 +22,7 @@ public static Query dropIndex(SQLite3GlobalState globalState) { sb.append('"'); sb.append(indexName); sb.append('"'); - return new QueryAdapter(sb.toString(), Arrays.asList( + return new QueryAdapter(sb.toString(), ExpectedErrors.from( "[SQLITE_ERROR] SQL error or missing database (index associated with UNIQUE or PRIMARY KEY constraint cannot be dropped)"), true); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java index abb973057..099d1e1f8 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java @@ -1,7 +1,6 @@ package sqlancer.sqlite3.gen.ddl; -import java.util.Arrays; - +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -23,7 +22,7 @@ public static Query dropTable(SQLite3GlobalState globalState) { } sb.append(globalState.getSchema().getRandomTableOrBailout(t -> !t.isView()).getName()); return new QueryAdapter(sb.toString(), - Arrays.asList("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", + ExpectedErrors.from("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", "Abort due to constraint violation (FOREIGN KEY constraint failed)", "SQL error or missing database"), true); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index b05525265..f24da36b8 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -1,10 +1,9 @@ package sqlancer.sqlite3.gen.ddl; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -21,7 +20,7 @@ // see https://www.sqlite.org/lang_createindex.html public class SQLite3IndexGenerator { - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private final SQLite3GlobalState globalState; public static Query insertIndex(SQLite3GlobalState globalState) throws SQLException { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index f6ee0319c..6809fbb4a 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -47,7 +48,7 @@ public SQLite3TableGenerator(String tableName, SQLite3GlobalState globalState) { public static Query createTableStatement(String tableName, SQLite3GlobalState globalState) { SQLite3TableGenerator sqLite3TableGenerator = new SQLite3TableGenerator(tableName, globalState); sqLite3TableGenerator.start(); - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); SQLite3Errors.addTableManipulationErrors(errors); errors.add("second argument to likelihood() must be a constant between 0.0 and 1.0"); errors.add("non-deterministic functions prohibited in generated columns"); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index ccaf22106..feb9f5c12 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -1,9 +1,8 @@ package sqlancer.sqlite3.gen.ddl; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -42,7 +41,7 @@ public static Query generate(SQLite3GlobalState globalState) throws SQLException sb.append(" IF NOT EXISTS "); } sb.append(SQLite3Common.getFreeViewName(globalState.getSchema())); - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("is circularly defined"); errors.add("unsupported frame specification"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index d686e7fbc..a700f33b0 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -1,9 +1,8 @@ package sqlancer.sqlite3.gen.dml; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -32,7 +31,7 @@ public static Query deleteContent(SQLite3GlobalState globalState, SQLite3Table t sb.append(SQLite3Visitor.asString(new SQLite3ExpressionGenerator(globalState) .setColumns(tableName.getColumns()).generateExpression())); } - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); SQLite3Errors.addExpectedExpressionErrors(errors); errors.addAll(Arrays.asList("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", "[SQLITE_CONSTRAINT] Abort due to constraint violation ", diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java index 38e2ca594..071323e32 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java @@ -1,10 +1,10 @@ package sqlancer.sqlite3.gen.dml; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -21,13 +21,13 @@ public class SQLite3InsertGenerator { private final Randomly r; - private final List errors; + private final ExpectedErrors errors; private final SQLite3GlobalState globalState; public SQLite3InsertGenerator(SQLite3GlobalState globalState, Randomly r) { this.globalState = globalState; this.r = r; - errors = new ArrayList<>(); + errors = new ExpectedErrors(); } public static Query insertRow(SQLite3GlobalState globalState) throws SQLException { diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 8661ee99f..71269555e 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -1,9 +1,9 @@ package sqlancer.sqlite3.gen.dml; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -19,7 +19,7 @@ public class SQLite3UpdateGenerator { private final StringBuilder sb = new StringBuilder(); private final Randomly r; - private final List errors = new ArrayList<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private final SQLite3GlobalState globalState; public SQLite3UpdateGenerator(SQLite3GlobalState globalState, Randomly r) { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index fb0c65b8f..1b6b38227 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -53,7 +54,7 @@ public class SQLite3PivotedQuerySynthesisOracle implements TestOracle { private SQLite3StateToReproduce state; private SQLite3RowValue rw; private List fetchColumns; - private final List errors = new ArrayList<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private List colExpressions; private final SQLite3GlobalState globalState; @@ -85,7 +86,7 @@ public Query getQueryThatContainsAtLeastOneRow(SQLite3GlobalState state) throws return new QueryAdapter(queryString, errors); } - public static void addExpectedErrors(List errors) { + public static void addExpectedErrors(ExpectedErrors errors) { errors.add("no such index"); errors.add("no query solution"); errors.add( @@ -239,12 +240,11 @@ private boolean isContainedIn(Query query) throws SQLException { createStatement.close(); return isContainedIn; } catch (SQLException e) { - for (String exp : finalQuery.getExpectedErrors()) { - if (e.getMessage().contains(exp)) { - return true; - } + if (finalQuery.getExpectedErrors().errorIsExpected(e.getMessage())) { + return true; + } else { + throw e; } - throw e; } } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index df61d4d3c..e2287ee2b 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -1,11 +1,11 @@ package sqlancer.sqlite3.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import sqlancer.ComparatorHelper; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -31,7 +31,7 @@ public class SQLite3TLPAggregateOracle implements TestOracle { private final SQLite3GlobalState state; - private final List errors = new ArrayList<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private SQLite3ExpressionGenerator gen; public SQLite3TLPAggregateOracle(SQLite3GlobalState state) { diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index 14a3a37b2..004fd9574 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -3,10 +3,10 @@ import java.sql.SQLException; import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.ComparatorHelper; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.TestOracle; @@ -32,7 +32,7 @@ public class SQLite3TLPHavingOracle implements TestOracle { private final SQLite3GlobalState state; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); public SQLite3TLPHavingOracle(SQLite3GlobalState state) { this.state = state; diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 32bce6522..91a569094 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -14,6 +14,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -347,7 +348,7 @@ public String toString() { public static int getNrRows(SQLite3GlobalState globalState, String table) throws SQLException { String string = "SELECT COUNT(*) FROM " + table; - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("ORDER BY term out of range"); errors.addAll(Arrays.asList("second argument to nth_value must be a positive integer", "ON clause references tables to its right", "no such table", "no query solution", "no such index", diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 1c07fb7e3..26a69d581 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -1,13 +1,13 @@ package sqlancer.tidb; -import java.util.Set; +import sqlancer.ExpectedErrors; public final class TiDBErrors { private TiDBErrors() { } - public static void addExpressionErrors(Set errors) { + public static void addExpressionErrors(ExpectedErrors errors) { errors.add("DECIMAL value is out of range"); errors.add("error parsing regexp"); errors.add("BIGINT UNSIGNED value is out of range"); @@ -46,12 +46,12 @@ public static void addExpressionErrors(Set errors) { errors.add("Illegal mix of collations"); } - public static void addExpressionHavingErrors(Set errors) { + public static void addExpressionHavingErrors(ExpectedErrors errors) { errors.add("is not in GROUP BY clause and contains nonaggregated column"); errors.add("Unknown column"); } - public static void addInsertErrors(Set errors) { + public static void addInsertErrors(ExpectedErrors errors) { errors.add("Duplicate entry"); errors.add("cannot be null"); errors.add("doesn't have a default value"); diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index cec4caffd..b054ec823 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -4,13 +4,12 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.AbstractAction; import sqlancer.CompositeTestOracle; +import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; @@ -51,7 +50,7 @@ public enum Action implements AbstractAction { VIEW_GENERATOR(TiDBViewGenerator::getQuery), // ALTER_TABLE(TiDBAlterTableGenerator::getQuery), // EXPLAIN((g) -> { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); TiDBErrors.addExpressionErrors(errors); TiDBErrors.addExpressionHavingErrors(errors); return new QueryAdapter( diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 1878fa6d6..eb4b39749 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -1,9 +1,8 @@ package sqlancer.tidb.gen; -import java.util.HashSet; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -24,7 +23,7 @@ private enum Action { } public static Query getQuery(TiDBGlobalState globalState) { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 6d3b87430..5a3844ff3 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -1,8 +1,8 @@ package sqlancer.tidb.gen; import java.sql.SQLException; -import java.util.Arrays; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -32,7 +32,7 @@ public static Query getQuery(TiDBGlobalState globalState) throws SQLException { sb.append(Randomly.getNotCachedInteger(1, 1024)); sb.append(" BUCKETS"); } - return new QueryAdapter(sb.toString(), Arrays.asList("https://github.com/pingcap/tidb/issues/15993", + return new QueryAdapter(sb.toString(), ExpectedErrors.from("https://github.com/pingcap/tidb/issues/15993", /* https://github.com/pingcap/tidb/issues/15993 */ "doesn't have a default value" /* * https://github. * com/pingcap/tidb/ diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index 5a41e74cf..578ddaf59 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -1,10 +1,9 @@ package sqlancer.tidb.gen; import java.sql.SQLException; -import java.util.HashSet; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -20,7 +19,7 @@ private TiDBDeleteGenerator() { } public static Query getQuery(TiDBGlobalState globalState) throws SQLException { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); StringBuilder sb = new StringBuilder("DELETE "); diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index a8f063e2e..5c2135a66 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -1,10 +1,9 @@ package sqlancer.tidb.gen; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -18,7 +17,7 @@ private TiDBIndexGenerator() { } public static Query getQuery(TiDBGlobalState globalState) throws SQLException { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); TiDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); String indexName = globalState.getSchema().getFreeIndexName(); diff --git a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java index cf6bdbc12..31af8bbb7 100644 --- a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java @@ -1,11 +1,10 @@ package sqlancer.tidb.gen; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -19,7 +18,7 @@ public class TiDBInsertGenerator { private final TiDBGlobalState globalState; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private TiDBExpressionGenerator gen; public TiDBInsertGenerator(TiDBGlobalState globalState) { diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 5be1f4df1..f8526a8ed 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -2,11 +2,10 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; import sqlancer.QueryAdapter; @@ -25,7 +24,7 @@ public class TiDBTableGenerator { private boolean allowPrimaryKey; private final List columns = new ArrayList<>(); private boolean primaryKeyAsTableConstraints; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); public Query getQuery(TiDBGlobalState globalState) throws SQLException { errors.add("Information schema is changed during the execution of the statement"); diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 7cb4c81ca..3cee65bd0 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -1,10 +1,9 @@ package sqlancer.tidb.gen; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -21,7 +20,7 @@ private TiDBUpdateGenerator() { } public static Query getQuery(TiDBGlobalState globalState) throws SQLException { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); StringBuilder sb = new StringBuilder("UPDATE "); diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index 340673b6a..497b546c6 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -1,8 +1,6 @@ package sqlancer.tidb.gen; -import java.util.HashSet; -import java.util.Set; - +import sqlancer.ExpectedErrors; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; @@ -37,7 +35,7 @@ public static Query getQuery(TiDBGlobalState globalState) { } sb.append(") AS "); sb.append(TiDBRandomQuerySynthesizer.generate(globalState, nrColumns).getQueryString()); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); TiDBErrors.addExpressionErrors(errors); errors.add( "references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them"); diff --git a/test/sqlancer/TestExpectedErrors.java b/test/sqlancer/TestExpectedErrors.java new file mode 100644 index 000000000..c963c108b --- /dev/null +++ b/test/sqlancer/TestExpectedErrors.java @@ -0,0 +1,38 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class TestExpectedErrors { + + @Test + public void testEmpty() { + ExpectedErrors errors = new ExpectedErrors(); + assertFalse(errors.errorIsExpected("a")); + } + + @Test + public void testSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("a"); + errors.add("b"); + errors.add("c"); + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + + assertFalse(errors.errorIsExpected("d")); + } + + @Test + public void testRealistic() { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("violated"); + assertTrue(errors.errorIsExpected("UNIQUE constraint was violated!")); + assertTrue(errors.errorIsExpected("PRIMARY KEY constraint was violated!")); + } + +} From 15fb835dea87a253ea434ac70147f9b118dfc302 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Aug 2020 20:39:26 +0200 Subject: [PATCH 0096/1284] Add a marker interface for the DBMS-specific options --- src/sqlancer/DBMSSpecificOptions.java | 5 +++++ src/sqlancer/DatabaseProvider.java | 2 +- src/sqlancer/GlobalState.java | 2 +- src/sqlancer/Main.java | 4 ++-- src/sqlancer/ProviderAdapter.java | 3 ++- src/sqlancer/clickhouse/ClickHouseOptions.java | 4 ++-- src/sqlancer/cockroachdb/CockroachDBOptions.java | 3 ++- src/sqlancer/duckdb/DuckDBOptions.java | 3 ++- src/sqlancer/mariadb/MariaDBOptions.java | 4 +++- src/sqlancer/mysql/MySQLOptions.java | 4 +++- src/sqlancer/postgres/PostgresOptions.java | 3 ++- src/sqlancer/sqlite3/SQLite3Options.java | 3 ++- src/sqlancer/tidb/TiDBOptions.java | 3 ++- 13 files changed, 29 insertions(+), 14 deletions(-) create mode 100644 src/sqlancer/DBMSSpecificOptions.java diff --git a/src/sqlancer/DBMSSpecificOptions.java b/src/sqlancer/DBMSSpecificOptions.java new file mode 100644 index 000000000..85da277e4 --- /dev/null +++ b/src/sqlancer/DBMSSpecificOptions.java @@ -0,0 +1,5 @@ +package sqlancer; + +public interface DBMSSpecificOptions { + +} diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 9d45b30d2..0074914e4 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -4,7 +4,7 @@ import java.sql.Connection; import java.sql.SQLException; -public interface DatabaseProvider, O> { +public interface DatabaseProvider, O extends DBMSSpecificOptions> { /** * Gets the the {@link GlobalState} class. diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index f2e4e99ad..c0d634d0d 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -12,7 +12,7 @@ * @param * the option parameter. */ -public abstract class GlobalState { +public abstract class GlobalState { private Connection con; private Randomly r; diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index c2c328a60..db2e25324 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -288,7 +288,7 @@ public static void main(String[] args) { System.exit(executeMain(args)); } - public static class DBMSExecutor, O> { + public static class DBMSExecutor, O extends DBMSSpecificOptions> { private final DatabaseProvider provider; private final MainOptions options; @@ -363,7 +363,7 @@ public StateToReproduce getStateToReproduce() { } } - public static class DBMSExecutorFactory, O> { + public static class DBMSExecutorFactory, O extends DBMSSpecificOptions> { private final DatabaseProvider provider; private final MainOptions options; diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index f132627bd..26fd9b1cd 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -5,7 +5,8 @@ import sqlancer.StateToReproduce.OracleRunReproductionState; -public abstract class ProviderAdapter, O> implements DatabaseProvider { +public abstract class ProviderAdapter, O extends DBMSSpecificOptions> + implements DatabaseProvider { private final Class globalClass; private final Class optionClass; diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index d380049d0..b2bca31bb 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -7,7 +7,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.MainOptions; +import sqlancer.DBMSSpecificOptions; import sqlancer.TestOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; @@ -16,7 +16,7 @@ import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; @Parameters(separators = "=", commandDescription = "ClickHouse") -public class ClickHouseOptions extends MainOptions { +public class ClickHouseOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracle = Arrays.asList(ClickHouseOracle.TLPWhere); diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 75e92a776..dfa940939 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -8,6 +8,7 @@ import com.beust.jcommander.Parameters; import sqlancer.CompositeTestOracle; +import sqlancer.DBMSSpecificOptions; import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; @@ -20,7 +21,7 @@ import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPWhereOracle; @Parameters(separators = "=", commandDescription = "Test CockroachDB") -public class CockroachDBOptions { +public class CockroachDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public CockroachDBOracle oracle = CockroachDBOracle.NOREC; diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index 1c633cbc4..3751c7e31 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -9,6 +9,7 @@ import com.beust.jcommander.Parameters; import sqlancer.CompositeTestOracle; +import sqlancer.DBMSSpecificOptions; import sqlancer.TestOracle; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.test.DuckDBNoRECOracle; @@ -19,7 +20,7 @@ import sqlancer.duckdb.test.DuckDBQueryPartitioningWhereTester; @Parameters -public class DuckDBOptions { +public class DuckDBOptions implements DBMSSpecificOptions { @Parameter(names = "--test-collate", arity = 1) public boolean testCollate = true; diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 6290ddf66..8abbeb7cd 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -2,7 +2,9 @@ import com.beust.jcommander.Parameters; +import sqlancer.DBMSSpecificOptions; + @Parameters -public class MariaDBOptions { +public class MariaDBOptions implements DBMSSpecificOptions { } diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index c31b165a8..0a1d579b2 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -2,7 +2,9 @@ import com.beust.jcommander.Parameters; +import sqlancer.DBMSSpecificOptions; + @Parameters -public class MySQLOptions { +public class MySQLOptions implements DBMSSpecificOptions { } diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index e817e3f62..de1078363 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -9,6 +9,7 @@ import com.beust.jcommander.Parameters; import sqlancer.CompositeTestOracle; +import sqlancer.DBMSSpecificOptions; import sqlancer.TestOracle; import sqlancer.postgres.oracle.PostgresNoRECOracle; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; @@ -17,7 +18,7 @@ import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; @Parameters -public class PostgresOptions { +public class PostgresOptions implements DBMSSpecificOptions { @Parameter(names = "--bulk-insert") public boolean allowBulkInsert; diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 5d86d8baa..0465870f9 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -8,6 +8,7 @@ import com.beust.jcommander.Parameters; import sqlancer.CompositeTestOracle; +import sqlancer.DBMSSpecificOptions; import sqlancer.TestOracle; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.oracle.SQLite3Fuzzer; @@ -20,7 +21,7 @@ import sqlancer.sqlite3.oracle.tlp.SQLite3TLPWhereOracle; @Parameters(separators = "=", commandDescription = "SQLite3") -public class SQLite3Options { +public class SQLite3Options implements DBMSSpecificOptions { @Parameter(names = { "--test-fts" }, description = "Test the FTS extensions", arity = 1) public boolean testFts = true; diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index b24102f6a..ac6bc7733 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -9,13 +9,14 @@ import com.beust.jcommander.Parameters; import sqlancer.CompositeTestOracle; +import sqlancer.DBMSSpecificOptions; import sqlancer.TestOracle; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @Parameters -public class TiDBOptions { +public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracle = Arrays.asList(TiDBOracle.QUERY_PARTITIONING); From db6cab5c14701207be534049e4176a567d9c2d6d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Aug 2020 20:57:54 +0200 Subject: [PATCH 0097/1284] Add a OracleFactory interface --- src/sqlancer/OracleFactory.java | 9 +++++++++ src/sqlancer/citus/CitusOptions.java | 7 +++---- src/sqlancer/clickhouse/ClickHouseOptions.java | 7 ++++--- src/sqlancer/cockroachdb/CockroachDBOptions.java | 7 +++---- src/sqlancer/duckdb/DuckDBOptions.java | 7 +++---- src/sqlancer/postgres/PostgresOptions.java | 7 +++---- src/sqlancer/sqlite3/SQLite3Options.java | 7 +++---- src/sqlancer/tidb/TiDBOptions.java | 7 +++---- 8 files changed, 31 insertions(+), 27 deletions(-) create mode 100644 src/sqlancer/OracleFactory.java diff --git a/src/sqlancer/OracleFactory.java b/src/sqlancer/OracleFactory.java new file mode 100644 index 000000000..f8762a448 --- /dev/null +++ b/src/sqlancer/OracleFactory.java @@ -0,0 +1,9 @@ +package sqlancer; + +import java.sql.SQLException; + +public interface OracleFactory> { + + TestOracle create(G globalState) throws SQLException; + +} diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index ebea40e4a..e5768537c 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -8,6 +8,7 @@ import com.beust.jcommander.Parameter; import sqlancer.CompositeTestOracle; +import sqlancer.OracleFactory; import sqlancer.TestOracle; import sqlancer.citus.oracle.CitusNoRECOracle; import sqlancer.citus.oracle.tlp.CitusTLPAggregateOracle; @@ -23,9 +24,9 @@ public class CitusOptions extends PostgresOptions { public boolean repartition = true; @Parameter(names = "--citusoracle") - public List citusOracle = Arrays.asList(CitusOracle.QUERY_PARTITIONING); + public List citusOracle = Arrays.asList(CitusOracleFactory.QUERY_PARTITIONING); - public enum CitusOracle { + public enum CitusOracleFactory implements OracleFactory { NOREC { @Override public TestOracle create(PostgresGlobalState globalState) throws SQLException { @@ -60,8 +61,6 @@ public TestOracle create(PostgresGlobalState globalState) throws SQLException { } }; - public abstract TestOracle create(PostgresGlobalState globalState) throws SQLException; - } } diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index b2bca31bb..0c866a706 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -8,7 +8,9 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; import sqlancer.TestOracle; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; @@ -19,12 +21,12 @@ public class ClickHouseOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") - public List oracle = Arrays.asList(ClickHouseOracle.TLPWhere); + public List oracle = Arrays.asList(ClickHouseOracleFactory.TLPWhere); @Parameter(names = { "--test-joins" }, description = "Allow the generation of JOIN clauses", arity = 1) public boolean testJoins = true; - public enum ClickHouseOracle { + public enum ClickHouseOracleFactory implements OracleFactory { TLPWhere { @Override public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { @@ -56,6 +58,5 @@ public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) t } }; - public abstract TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException; } } diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index dfa940939..e36c30e8d 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -9,6 +9,7 @@ import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; @@ -24,9 +25,9 @@ public class CockroachDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") - public CockroachDBOracle oracle = CockroachDBOracle.NOREC; + public CockroachDBOracleFactory oracle = CockroachDBOracleFactory.NOREC; - public enum CockroachDBOracle { + public enum CockroachDBOracleFactory implements OracleFactory { NOREC { @Override public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { @@ -91,8 +92,6 @@ public TestOracle create(CockroachDBGlobalState globalState) throws SQLException } }; - public abstract TestOracle create(CockroachDBGlobalState globalState) throws SQLException; - } @Parameter(names = { diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index 3751c7e31..1b671265d 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -10,6 +10,7 @@ import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; import sqlancer.TestOracle; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.test.DuckDBNoRECOracle; @@ -89,9 +90,9 @@ public class DuckDBOptions implements DBMSSpecificOptions { public int maxNumUpdates = 5; @Parameter(names = "--oracle") - public List oracle = Arrays.asList(DuckDBOracle.QUERY_PARTITIONING); + public List oracle = Arrays.asList(DuckDBOracleFactory.QUERY_PARTITIONING); - public enum DuckDBOracle { + public enum DuckDBOracleFactory implements OracleFactory { NOREC { @Override @@ -145,8 +146,6 @@ public TestOracle create(DuckDBGlobalState globalState) throws SQLException { } }; - public abstract TestOracle create(DuckDBGlobalState globalState) throws SQLException; - } } diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index de1078363..ce223d783 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -10,6 +10,7 @@ import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; import sqlancer.TestOracle; import sqlancer.postgres.oracle.PostgresNoRECOracle; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; @@ -24,7 +25,7 @@ public class PostgresOptions implements DBMSSpecificOptions { public boolean allowBulkInsert; @Parameter(names = "--oracle") - public List oracle = Arrays.asList(PostgresOracle.QUERY_PARTITIONING); + public List oracle = Arrays.asList(PostgresOracleFactory.QUERY_PARTITIONING); @Parameter(names = "--test-collations", arity = 1) public boolean testCollations = true; @@ -32,7 +33,7 @@ public class PostgresOptions implements DBMSSpecificOptions { @Parameter(names = "--connection-url") public String connectionURL = "postgresql://localhost:5432/test"; - public enum PostgresOracle { + public enum PostgresOracleFactory implements OracleFactory { NOREC { @Override public TestOracle create(PostgresGlobalState globalState) throws SQLException { @@ -64,8 +65,6 @@ public TestOracle create(PostgresGlobalState globalState) throws SQLException { } }; - public abstract TestOracle create(PostgresGlobalState globalState) throws SQLException; - } } diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 0465870f9..98ffd90f4 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -9,6 +9,7 @@ import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; import sqlancer.TestOracle; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.oracle.SQLite3Fuzzer; @@ -71,7 +72,7 @@ public class SQLite3Options implements DBMSSpecificOptions { public boolean testDistinctInView; @Parameter(names = "--oracle") - public SQLite3Oracle oracle = SQLite3Oracle.NoREC; + public SQLite3OracleFactory oracle = SQLite3OracleFactory.NoREC; @Parameter(names = { "--delete-existing-databases" }, description = "Delete a database file if it already exists", arity = 1) @@ -85,7 +86,7 @@ public class SQLite3Options implements DBMSSpecificOptions { "--execute-queries" }, description = "Specifies whether the query in the fuzzer should be executed", arity = 1) public boolean executeQuery = true; - public enum SQLite3Oracle { + public enum SQLite3OracleFactory implements OracleFactory { PQS { @Override public TestOracle create(SQLite3GlobalState globalState) throws SQLException { @@ -151,8 +152,6 @@ public TestOracle create(SQLite3GlobalState globalState) throws SQLException { } }; - public abstract TestOracle create(SQLite3GlobalState globalState) throws SQLException; - } } diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index ac6bc7733..ea73a3614 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -10,6 +10,7 @@ import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; import sqlancer.TestOracle; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; @@ -19,9 +20,9 @@ public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") - public List oracle = Arrays.asList(TiDBOracle.QUERY_PARTITIONING); + public List oracle = Arrays.asList(TiDBOracleFactory.QUERY_PARTITIONING); - public enum TiDBOracle { + public enum TiDBOracleFactory implements OracleFactory { HAVING { @Override public TestOracle create(TiDBGlobalState globalState) throws SQLException { @@ -44,8 +45,6 @@ public TestOracle create(TiDBGlobalState globalState) throws SQLException { } }; - public abstract TestOracle create(TiDBGlobalState globalState) throws SQLException; - } } From 38bf58c7d18d3297d839c39beb0d66977c01c686 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Aug 2020 21:42:12 +0200 Subject: [PATCH 0098/1284] Add a method to get the test oracle factories from the DBMS-specific options --- src/sqlancer/DBMSSpecificOptions.java | 6 +++- src/sqlancer/DatabaseProvider.java | 2 +- src/sqlancer/GlobalState.java | 2 +- src/sqlancer/Main.java | 4 +-- src/sqlancer/ProviderAdapter.java | 2 +- .../clickhouse/ClickHouseOptions.java | 8 ++++- .../cockroachdb/CockroachDBOptions.java | 9 +++++- src/sqlancer/duckdb/DuckDBOptions.java | 10 ++++-- src/sqlancer/duckdb/DuckDBProvider.java | 2 +- src/sqlancer/mariadb/MariaDBOptions.java | 32 ++++++++++++++++++- src/sqlancer/mysql/MySQLOptions.java | 31 +++++++++++++++++- src/sqlancer/postgres/PostgresOptions.java | 8 ++++- src/sqlancer/sqlite3/SQLite3Options.java | 11 +++++-- src/sqlancer/sqlite3/SQLite3Provider.java | 2 +- src/sqlancer/tidb/TiDBOptions.java | 8 ++++- 15 files changed, 119 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/DBMSSpecificOptions.java b/src/sqlancer/DBMSSpecificOptions.java index 85da277e4..0cb7daf92 100644 --- a/src/sqlancer/DBMSSpecificOptions.java +++ b/src/sqlancer/DBMSSpecificOptions.java @@ -1,5 +1,9 @@ package sqlancer; -public interface DBMSSpecificOptions { +import java.util.List; + +public interface DBMSSpecificOptions>> { + + List getTestOracleFactory(); } diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 0074914e4..9e8642741 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -4,7 +4,7 @@ import java.sql.Connection; import java.sql.SQLException; -public interface DatabaseProvider, O extends DBMSSpecificOptions> { +public interface DatabaseProvider, O extends DBMSSpecificOptions> { /** * Gets the the {@link GlobalState} class. diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index c0d634d0d..a90bbd7bc 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -12,7 +12,7 @@ * @param * the option parameter. */ -public abstract class GlobalState { +public abstract class GlobalState, S> { private Connection con; private Randomly r; diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index db2e25324..443c4cb69 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -288,7 +288,7 @@ public static void main(String[] args) { System.exit(executeMain(args)); } - public static class DBMSExecutor, O extends DBMSSpecificOptions> { + public static class DBMSExecutor, O extends DBMSSpecificOptions> { private final DatabaseProvider provider; private final MainOptions options; @@ -363,7 +363,7 @@ public StateToReproduce getStateToReproduce() { } } - public static class DBMSExecutorFactory, O extends DBMSSpecificOptions> { + public static class DBMSExecutorFactory, O extends DBMSSpecificOptions> { private final DatabaseProvider provider; private final MainOptions options; diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 26fd9b1cd..5a8e817a0 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -5,7 +5,7 @@ import sqlancer.StateToReproduce.OracleRunReproductionState; -public abstract class ProviderAdapter, O extends DBMSSpecificOptions> +public abstract class ProviderAdapter, O extends DBMSSpecificOptions> implements DatabaseProvider { private final Class globalClass; diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 0c866a706..baeeb15d3 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -10,6 +10,7 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.TestOracle; +import sqlancer.clickhouse.ClickHouseOptions.ClickHouseOracleFactory; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; @@ -18,7 +19,7 @@ import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; @Parameters(separators = "=", commandDescription = "ClickHouse") -public class ClickHouseOptions implements DBMSSpecificOptions { +public class ClickHouseOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracle = Arrays.asList(ClickHouseOracleFactory.TLPWhere); @@ -59,4 +60,9 @@ public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) t }; } + + @Override + public List getTestOracleFactory() { + return oracle; + } } diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index e36c30e8d..ba125447f 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -2,6 +2,7 @@ import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameter; @@ -11,6 +12,7 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.TestOracle; +import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; @@ -22,7 +24,7 @@ import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPWhereOracle; @Parameters(separators = "=", commandDescription = "Test CockroachDB") -public class CockroachDBOptions implements DBMSSpecificOptions { +public class CockroachDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public CockroachDBOracleFactory oracle = CockroachDBOracleFactory.NOREC; @@ -105,4 +107,9 @@ public TestOracle create(CockroachDBGlobalState globalState) throws SQLException "--increased-vectorization" }, description = "Generate VECTORIZE=on with a higher probability (which found a number of bugs in the past)") public boolean makeVectorizationMoreLikely = true; + @Override + public List getTestOracleFactory() { + return Arrays.asList(oracle); + } + } diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index 1b671265d..6e2099122 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -12,6 +12,7 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.TestOracle; +import sqlancer.duckdb.DuckDBOptions.DuckDBOracleFactory; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.test.DuckDBNoRECOracle; import sqlancer.duckdb.test.DuckDBQueryPartitioningAggregateTester; @@ -21,7 +22,7 @@ import sqlancer.duckdb.test.DuckDBQueryPartitioningWhereTester; @Parameters -public class DuckDBOptions implements DBMSSpecificOptions { +public class DuckDBOptions implements DBMSSpecificOptions { @Parameter(names = "--test-collate", arity = 1) public boolean testCollate = true; @@ -90,7 +91,7 @@ public class DuckDBOptions implements DBMSSpecificOptions { public int maxNumUpdates = 5; @Parameter(names = "--oracle") - public List oracle = Arrays.asList(DuckDBOracleFactory.QUERY_PARTITIONING); + public List oracles = Arrays.asList(DuckDBOracleFactory.QUERY_PARTITIONING); public enum DuckDBOracleFactory implements OracleFactory { NOREC { @@ -148,4 +149,9 @@ public TestOracle create(DuckDBGlobalState globalState) throws SQLException { } + @Override + public List getTestOracleFactory() { + return oracles; + } + } diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 880f29e27..6474d4455 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -120,7 +120,7 @@ public void generateDatabase(DuckDBGlobalState globalState) throws SQLException @Override protected TestOracle getTestOracle(DuckDBGlobalState globalState) throws SQLException { - return new CompositeTestOracle(globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { + return new CompositeTestOracle(globalState.getDmbsSpecificOptions().oracles.stream().map(o -> { try { return o.create(globalState); } catch (SQLException e1) { diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 8abbeb7cd..86d485c8e 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -1,10 +1,40 @@ package sqlancer.mariadb; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.TestOracle; +import sqlancer.mariadb.MariaDBOptions.MariaDBOracleFactory; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.oracle.MariaDBNoRECOracle; @Parameters -public class MariaDBOptions implements DBMSSpecificOptions { +public class MariaDBOptions implements DBMSSpecificOptions { + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(MariaDBOracleFactory.NOREC); + + public enum MariaDBOracleFactory implements OracleFactory { + + NOREC { + + @Override + public TestOracle create(MariaDBGlobalState globalState) throws SQLException { + return new MariaDBNoRECOracle(globalState); + } + + } + } + + @Override + public List getTestOracleFactory() { + return oracles; + } } diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 0a1d579b2..4b65d798e 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -1,10 +1,39 @@ package sqlancer.mysql; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.TestOracle; +import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; +import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @Parameters -public class MySQLOptions implements DBMSSpecificOptions { +public class MySQLOptions implements DBMSSpecificOptions { + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(MySQLOracleFactory.TLP_WHERE); + + public enum MySQLOracleFactory implements OracleFactory { + + TLP_WHERE { + + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLTLPWhereOracle(globalState); + } + + } + } + + @Override + public List getTestOracleFactory() { + return oracles; + } } diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index ce223d783..5c318f1a8 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -12,6 +12,7 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.TestOracle; +import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; import sqlancer.postgres.oracle.PostgresNoRECOracle; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; @@ -19,7 +20,7 @@ import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; @Parameters -public class PostgresOptions implements DBMSSpecificOptions { +public class PostgresOptions implements DBMSSpecificOptions { @Parameter(names = "--bulk-insert") public boolean allowBulkInsert; @@ -67,4 +68,9 @@ public TestOracle create(PostgresGlobalState globalState) throws SQLException { } + @Override + public List getTestOracleFactory() { + return oracle; + } + } diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 98ffd90f4..e79a2d8af 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -2,6 +2,7 @@ import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameter; @@ -11,6 +12,7 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.TestOracle; +import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.oracle.SQLite3Fuzzer; import sqlancer.sqlite3.oracle.SQLite3NoRECOracle; @@ -22,7 +24,7 @@ import sqlancer.sqlite3.oracle.tlp.SQLite3TLPWhereOracle; @Parameters(separators = "=", commandDescription = "SQLite3") -public class SQLite3Options implements DBMSSpecificOptions { +public class SQLite3Options implements DBMSSpecificOptions { @Parameter(names = { "--test-fts" }, description = "Test the FTS extensions", arity = 1) public boolean testFts = true; @@ -72,7 +74,7 @@ public class SQLite3Options implements DBMSSpecificOptions { public boolean testDistinctInView; @Parameter(names = "--oracle") - public SQLite3OracleFactory oracle = SQLite3OracleFactory.NoREC; + public SQLite3OracleFactory oracles = SQLite3OracleFactory.NoREC; @Parameter(names = { "--delete-existing-databases" }, description = "Delete a database file if it already exists", arity = 1) @@ -154,4 +156,9 @@ public TestOracle create(SQLite3GlobalState globalState) throws SQLException { } + @Override + public List getTestOracleFactory() { + return Arrays.asList(oracles); + } + } diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 2a49c8075..6bd597752 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -282,7 +282,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws SQLException @Override protected TestOracle getTestOracle(SQLite3GlobalState globalState) throws SQLException { - TestOracle oracle = globalState.getDmbsSpecificOptions().oracle.create(globalState); + TestOracle oracle = globalState.getDmbsSpecificOptions().oracles.create(globalState); if (oracle.onlyWorksForNonEmptyTables()) { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { int nrRows = SQLite3Schema.getNrRows(globalState, table.getName()); diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index ea73a3614..2c5cc3c64 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -12,12 +12,13 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.TestOracle; +import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @Parameters -public class TiDBOptions implements DBMSSpecificOptions { +public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracle = Arrays.asList(TiDBOracleFactory.QUERY_PARTITIONING); @@ -47,4 +48,9 @@ public TestOracle create(TiDBGlobalState globalState) throws SQLException { } + @Override + public List getTestOracleFactory() { + return oracle; + } + } From 1d122072f210d8062dc863d6e62ab346226af5e8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Aug 2020 22:34:10 +0200 Subject: [PATCH 0099/1284] Move the getTestOracle method implementation to the provider --- src/sqlancer/ProviderAdapter.java | 20 +++++++++++++++++-- .../clickhouse/ClickHouseProvider.java | 15 -------------- .../cockroachdb/CockroachDBProvider.java | 6 ------ src/sqlancer/duckdb/DuckDBProvider.java | 14 ------------- src/sqlancer/mariadb/MariaDBProvider.java | 7 ------- src/sqlancer/mysql/MySQLProvider.java | 7 ------- src/sqlancer/postgres/PostgresProvider.java | 14 ------------- src/sqlancer/sqlite3/SQLite3Provider.java | 15 -------------- src/sqlancer/tidb/TiDBProvider.java | 16 --------------- 9 files changed, 18 insertions(+), 96 deletions(-) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 5a8e817a0..2cbf81693 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -2,10 +2,12 @@ import java.io.FileWriter; import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; import sqlancer.StateToReproduce.OracleRunReproductionState; -public abstract class ProviderAdapter, O extends DBMSSpecificOptions> +public abstract class ProviderAdapter, O extends DBMSSpecificOptions>> implements DatabaseProvider { private final Class globalClass; @@ -61,7 +63,21 @@ public void generateAndTestDatabase(G globalState) throws SQLException { } } - protected abstract TestOracle getTestOracle(G globalState) throws SQLException; + protected TestOracle getTestOracle(G globalState) throws SQLException { + List> testOracleFactory = globalState.getDmbsSpecificOptions() + .getTestOracleFactory(); + if (testOracleFactory.size() == 1) { + return testOracleFactory.get(0).create(globalState); + } else { + return new CompositeTestOracle(testOracleFactory.stream().map(o -> { + try { + return o.create(globalState); + } catch (SQLException e1) { + throw new AssertionError(e1); + } + }).collect(Collectors.toList()), globalState); + } + } public abstract void generateDatabase(G globalState) throws SQLException; diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 37e998ddb..2aab50e72 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -4,11 +4,9 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.List; import java.util.stream.Collectors; import sqlancer.AbstractAction; -import sqlancer.CompositeTestOracle; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; @@ -16,7 +14,6 @@ import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseInsertGenerator; @@ -102,18 +99,6 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws SQLExcept se.executeStatements(); } - @Override - protected TestOracle getTestOracle(ClickHouseGlobalState globalState) throws SQLException { - List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { - try { - return o.create(globalState); - } catch (SQLException e1) { - throw new AssertionError(e1); - } - }).collect(Collectors.toList()); - return new CompositeTestOracle(oracles, globalState); - } - @Override public Connection createDatabase(ClickHouseGlobalState globalState) throws SQLException { ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 6ed1efcfb..559d50be0 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -18,7 +18,6 @@ import sqlancer.QueryAdapter; import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.gen.CockroachDBCommentOnGenerator; @@ -247,11 +246,6 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws SQLExcep } } - @Override - protected TestOracle getTestOracle(CockroachDBGlobalState globalState) throws SQLException { - return globalState.getDmbsSpecificOptions().oracle.create(globalState); - } - @Override public Connection createDatabase(CockroachDBGlobalState globalState) throws SQLException { String databaseName = globalState.getDatabaseName(); diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 6474d4455..c93145c6c 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -3,10 +3,8 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.stream.Collectors; import sqlancer.AbstractAction; -import sqlancer.CompositeTestOracle; import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; @@ -16,7 +14,6 @@ import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.gen.DuckDBDeleteGenerator; import sqlancer.duckdb.gen.DuckDBIndexGenerator; @@ -118,17 +115,6 @@ public void generateDatabase(DuckDBGlobalState globalState) throws SQLException se.executeStatements(); } - @Override - protected TestOracle getTestOracle(DuckDBGlobalState globalState) throws SQLException { - return new CompositeTestOracle(globalState.getDmbsSpecificOptions().oracles.stream().map(o -> { - try { - return o.create(globalState); - } catch (SQLException e1) { - throw new AssertionError(e1); - } - }).collect(Collectors.toList()), globalState); - } - @Override public Connection createDatabase(DuckDBGlobalState globalState) throws SQLException { String url = "jdbc:duckdb:"; diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 8643d5aa1..733e30ccd 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -13,7 +13,6 @@ import sqlancer.ProviderAdapter; import sqlancer.Query; import sqlancer.Randomly; -import sqlancer.TestOracle; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.gen.MariaDBIndexGenerator; import sqlancer.mariadb.gen.MariaDBInsertGenerator; @@ -22,7 +21,6 @@ import sqlancer.mariadb.gen.MariaDBTableGenerator; import sqlancer.mariadb.gen.MariaDBTruncateGenerator; import sqlancer.mariadb.gen.MariaDBUpdateGenerator; -import sqlancer.mariadb.oracle.MariaDBNoRECOracle; import sqlancer.sqlite3.gen.SQLite3Common; public class MariaDBProvider extends ProviderAdapter { @@ -154,11 +152,6 @@ public void generateDatabase(MariaDBGlobalState globalState) throws SQLException } } - @Override - protected TestOracle getTestOracle(MariaDBGlobalState globalState) throws SQLException { - return new MariaDBNoRECOracle(globalState); - } - public static class MariaDBGlobalState extends GlobalState { @Override diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 23a7eeae5..899856652 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -20,7 +20,6 @@ import sqlancer.StateToReproduce; import sqlancer.StateToReproduce.MySQLStateToReproduce; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; @@ -38,7 +37,6 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLChecksum; import sqlancer.mysql.gen.tblmaintenance.MySQLOptimize; import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; -import sqlancer.mysql.oracle.MySQLTLPWhereOracle; import sqlancer.sqlite3.gen.SQLite3Common; public class MySQLProvider extends ProviderAdapter { @@ -159,11 +157,6 @@ public void generateDatabase(MySQLGlobalState globalState) throws SQLException { se.executeStatements(); } - @Override - protected TestOracle getTestOracle(MySQLGlobalState globalState) throws SQLException { - return new MySQLTLPWhereOracle(globalState); // FIXME: options for the other test oracles - } - public static int getNrRows(Connection con, MySQLTable table) throws SQLException { try (Statement s = con.createStatement()) { try (ResultSet query = s.executeQuery("SELECT COUNT(*) FROM " + table.getName())) { diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index b59f94e4e..9b1c84ca6 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -13,7 +13,6 @@ import java.util.stream.Collectors; import sqlancer.AbstractAction; -import sqlancer.CompositeTestOracle; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Query; @@ -24,7 +23,6 @@ import sqlancer.StateToReproduce; import sqlancer.StateToReproduce.PostgresStateToReproduce; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.ast.PostgresExpression; @@ -203,18 +201,6 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio prepareTables(globalState); } - @Override - protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { - List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { - try { - return o.create(globalState); - } catch (SQLException e1) { - throw new AssertionError(e1); - } - }).collect(Collectors.toList()); - return new CompositeTestOracle(oracles, globalState); - } - @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { username = globalState.getOptions().getUserName(); diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 6bd597752..451f55095 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -23,7 +23,6 @@ import sqlancer.StateToReproduce; import sqlancer.StateToReproduce.SQLite3StateToReproduce; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; import sqlancer.sqlite3.gen.SQLite3Common; @@ -280,20 +279,6 @@ public void generateDatabase(SQLite3GlobalState globalState) throws SQLException } } - @Override - protected TestOracle getTestOracle(SQLite3GlobalState globalState) throws SQLException { - TestOracle oracle = globalState.getDmbsSpecificOptions().oracles.create(globalState); - if (oracle.onlyWorksForNonEmptyTables()) { - for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { - int nrRows = SQLite3Schema.getNrRows(globalState, table.getName()); - if (nrRows == 0) { - throw new IgnoreMeException(); - } - } - } - return oracle; - } - private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws SQLException { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { Query q = new QueryAdapter("SELECT * FROM " + table.getName(), diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index b054ec823..bd147ddb1 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -4,11 +4,8 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.List; -import java.util.stream.Collectors; import sqlancer.AbstractAction; -import sqlancer.CompositeTestOracle; import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; @@ -18,7 +15,6 @@ import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; @@ -134,18 +130,6 @@ public void generateDatabase(TiDBGlobalState globalState) throws SQLException { } } - @Override - protected TestOracle getTestOracle(TiDBGlobalState globalState) throws SQLException { - List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { - try { - return o.create(globalState); - } catch (SQLException e1) { - throw new AssertionError(e1); - } - }).collect(Collectors.toList()); - return new CompositeTestOracle(oracles, globalState); - } - @Override public Connection createDatabase(TiDBGlobalState globalState) throws SQLException { String databaseName = globalState.getDatabaseName(); From cb1ba25c7bf05f88e91969d372aa8cd1833dd35b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Aug 2020 23:16:02 +0200 Subject: [PATCH 0100/1284] Introduce a new sqlancer.common package --- .../TernaryLogicPartitioningOracleBase.java | 2 +- src/sqlancer/ast/newast/Node.java | 5 ---- src/sqlancer/clickhouse/ClickHouseSchema.java | 8 +++--- .../clickhouse/ClickHouseToStringVisitor.java | 2 +- .../ClickHouseBinaryComparisonOperation.java | 2 +- .../clickhouse/ast/ClickHouseExpression.java | 2 +- .../ast/ClickHouseUnaryPostfixOperation.java | 4 +-- .../ast/ClickHouseUnaryPrefixOperation.java | 4 +-- .../gen/ClickHouseExpressionGenerator.java | 2 +- .../gen/ClickHouseInsertGenerator.java | 2 +- .../oracle/tlp/ClickHouseTLPBase.java | 2 +- .../cockroachdb/CockroachDBSchema.java | 10 +++---- .../CockroachDBToStringVisitor.java | 2 +- .../cockroachdb/ast/CockroachDBAlias.java | 2 +- .../CockroachDBBinaryArithmeticOperation.java | 4 +-- .../CockroachDBBinaryComparisonOperator.java | 4 +-- .../CockroachDBBinaryLogicalOperation.java | 4 +-- .../cockroachdb/ast/CockroachDBCast.java | 2 +- .../cockroachdb/ast/CockroachDBCollate.java | 2 +- .../ast/CockroachDBConcatOperation.java | 2 +- .../ast/CockroachDBIndexReference.java | 4 +-- .../ast/CockroachDBNotOperation.java | 2 +- .../ast/CockroachDBOrderingTerm.java | 2 +- .../ast/CockroachDBRegexOperation.java | 2 +- .../cockroachdb/ast/CockroachDBSelect.java | 2 +- .../ast/CockroachDBTypeAnnotation.java | 2 +- .../CockroachDBUnaryArithmeticOperation.java | 4 +-- .../ast/CockroachDBUnaryPostfixOperation.java | 4 +-- .../gen/CockroachDBCommentOnGenerator.java | 2 +- .../gen/CockroachDBExpressionGenerator.java | 2 +- .../cockroachdb/gen/CockroachDBGenerator.java | 2 +- .../oracle/tlp/CockroachDBTLPBase.java | 2 +- src/sqlancer/{ => common}/ast/BinaryNode.java | 4 +-- .../{ => common}/ast/BinaryOperatorNode.java | 4 +-- .../{ => common}/ast/FunctionNode.java | 2 +- src/sqlancer/{ => common}/ast/SelectBase.java | 2 +- .../{ => common}/ast/TernaryNode.java | 4 +-- src/sqlancer/{ => common}/ast/UnaryNode.java | 4 +-- .../{ => common}/ast/UnaryOperatorNode.java | 4 +-- .../ast/newast/ColumnReferenceNode.java | 4 +-- .../{ => common}/ast/newast/NewAliasNode.java | 2 +- .../ast/newast/NewBetweenOperatorNode.java | 2 +- .../ast/newast/NewBinaryOperatorNode.java | 4 +-- .../ast/newast/NewCaseOperatorNode.java | 2 +- .../ast/newast/NewFunctionNode.java | 2 +- .../ast/newast/NewInOperatorNode.java | 2 +- .../ast/newast/NewOrderingTerm.java | 2 +- .../ast/newast/NewPostfixTextNode.java | 2 +- .../ast/newast/NewTernaryNode.java | 2 +- .../ast/newast/NewToStringVisitor.java | 2 +- .../newast/NewUnaryPostfixOperatorNode.java | 4 +-- .../newast/NewUnaryPrefixOperatorNode.java | 4 +-- src/sqlancer/common/ast/newast/Node.java | 5 ++++ .../ast/newast/TableReferenceNode.java | 4 +-- .../{ => common}/gen/AbstractGenerator.java | 2 +- .../gen/AbstractInsertGenerator.java | 2 +- .../{ => common}/gen/ExpressionGenerator.java | 2 +- .../gen/TypedExpressionGenerator.java | 2 +- .../gen/UntypedExpressionGenerator.java | 2 +- .../{ => common}/schema/AbstractSchema.java | 2 +- .../{ => common}/schema/AbstractTable.java | 2 +- .../schema/AbstractTableColumn.java | 2 +- .../{ => common}/schema/AbstractTables.java | 2 +- .../{ => common}/schema/TableIndex.java | 2 +- .../{ => common}/visitor/BinaryOperation.java | 2 +- .../{ => common}/visitor/NodeVisitor.java | 2 +- .../visitor/TernaryOperation.java | 2 +- .../{ => common}/visitor/ToStringVisitor.java | 4 +-- .../{ => common}/visitor/UnaryOperation.java | 2 +- src/sqlancer/duckdb/DuckDBSchema.java | 10 +++---- .../duckdb/DuckDBToStringVisitor.java | 4 +-- src/sqlancer/duckdb/ast/DuckDBConstant.java | 2 +- src/sqlancer/duckdb/ast/DuckDBJoin.java | 4 +-- src/sqlancer/duckdb/ast/DuckDBSelect.java | 4 +-- .../duckdb/gen/DuckDBExpressionGenerator.java | 28 +++++++++---------- .../duckdb/gen/DuckDBIndexGenerator.java | 2 +- .../duckdb/gen/DuckDBInsertGenerator.java | 2 +- .../gen/DuckDBRandomQuerySynthesizer.java | 4 +-- .../duckdb/gen/DuckDBTableGenerator.java | 4 +-- .../duckdb/gen/DuckDBUpdateGenerator.java | 2 +- .../duckdb/test/DuckDBNoRECOracle.java | 8 +++--- ...uckDBQueryPartitioningAggregateTester.java | 12 ++++---- .../test/DuckDBQueryPartitioningBase.java | 8 +++--- .../DuckDBQueryPartitioningGroupByTester.java | 4 +-- .../DuckDBQueryPartitioningHavingTester.java | 2 +- src/sqlancer/mysql/MySQLSchema.java | 10 +++---- src/sqlancer/mysql/MySQLToStringVisitor.java | 2 +- src/sqlancer/mysql/ast/MySQLCollate.java | 2 +- src/sqlancer/mysql/ast/MySQLSelect.java | 2 +- .../mysql/ast/MySQLUnaryPrefixOperation.java | 4 +-- .../mysql/gen/MySQLExpressionGenerator.java | 2 +- .../oracle/MySQLQueryPartitioningBase.java | 2 +- src/sqlancer/postgres/PostgresSchema.java | 8 +++--- .../postgres/PostgresToStringVisitor.java | 2 +- .../postgres/ast/PostgresAggregate.java | 2 +- src/sqlancer/postgres/ast/PostgresAlias.java | 2 +- .../PostgresBinaryArithmeticOperation.java | 4 +-- .../ast/PostgresBinaryBitOperation.java | 4 +-- .../PostgresBinaryComparisonOperation.java | 4 +-- .../ast/PostgresBinaryLogicalOperation.java | 4 +-- .../ast/PostgresBinaryRangeOperation.java | 4 +-- .../postgres/ast/PostgresConcatOperation.java | 2 +- .../postgres/ast/PostgresLikeOperation.java | 2 +- .../ast/PostgresPOSIXRegularExpression.java | 2 +- .../ast/PostgresPostfixOperation.java | 2 +- .../postgres/ast/PostgresPrefixOperation.java | 2 +- src/sqlancer/postgres/ast/PostgresSelect.java | 2 +- .../gen/PostgresExpressionGenerator.java | 2 +- .../postgres/oracle/tlp/PostgresTLPBase.java | 2 +- .../sqlite3/SQLite3ToStringVisitor.java | 2 +- .../sqlite3/ast/SQLite3Expression.java | 4 +-- .../sqlite3/ast/SQLite3UnaryOperation.java | 2 +- .../gen/SQLite3ExpressionGenerator.java | 2 +- .../sqlite3/oracle/tlp/SQLite3TLPBase.java | 2 +- .../sqlite3/schema/SQLite3Schema.java | 6 ++-- .../tidb/TiDBExpressionGenerator.java | 2 +- src/sqlancer/tidb/TiDBSchema.java | 10 +++---- src/sqlancer/tidb/ast/TiDBAggregate.java | 2 +- .../ast/TiDBBinaryArithmeticOperation.java | 4 +-- .../tidb/ast/TiDBBinaryBitOperation.java | 4 +-- .../ast/TiDBBinaryComparisonOperation.java | 4 +-- .../tidb/ast/TiDBBinaryLogicalOperation.java | 4 +-- src/sqlancer/tidb/ast/TiDBCollate.java | 2 +- src/sqlancer/tidb/ast/TiDBOrderingTerm.java | 2 +- src/sqlancer/tidb/ast/TiDBRegexOperation.java | 4 +-- src/sqlancer/tidb/ast/TiDBSelect.java | 2 +- .../tidb/ast/TiDBUnaryPostfixOperation.java | 4 +-- .../tidb/ast/TiDBUnaryPrefixOperation.java | 4 +-- src/sqlancer/tidb/gen/TiDBHintGenerator.java | 2 +- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 2 +- .../tidb/visitor/TiDBToStringVisitor.java | 2 +- 131 files changed, 221 insertions(+), 221 deletions(-) delete mode 100644 src/sqlancer/ast/newast/Node.java rename src/sqlancer/{ => common}/ast/BinaryNode.java (82%) rename src/sqlancer/{ => common}/ast/BinaryOperatorNode.java (84%) rename src/sqlancer/{ => common}/ast/FunctionNode.java (92%) rename src/sqlancer/{ => common}/ast/SelectBase.java (98%) rename src/sqlancer/{ => common}/ast/TernaryNode.java (86%) rename src/sqlancer/{ => common}/ast/UnaryNode.java (75%) rename src/sqlancer/{ => common}/ast/UnaryOperatorNode.java (79%) rename src/sqlancer/{ => common}/ast/newast/ColumnReferenceNode.java (73%) rename src/sqlancer/{ => common}/ast/newast/NewAliasNode.java (90%) rename src/sqlancer/{ => common}/ast/newast/NewBetweenOperatorNode.java (94%) rename src/sqlancer/{ => common}/ast/newast/NewBinaryOperatorNode.java (85%) rename src/sqlancer/{ => common}/ast/newast/NewCaseOperatorNode.java (96%) rename src/sqlancer/{ => common}/ast/newast/NewFunctionNode.java (91%) rename src/sqlancer/{ => common}/ast/newast/NewInOperatorNode.java (94%) rename src/sqlancer/{ => common}/ast/newast/NewOrderingTerm.java (94%) rename src/sqlancer/{ => common}/ast/newast/NewPostfixTextNode.java (90%) rename src/sqlancer/{ => common}/ast/newast/NewTernaryNode.java (95%) rename src/sqlancer/{ => common}/ast/newast/NewToStringVisitor.java (99%) rename src/sqlancer/{ => common}/ast/newast/NewUnaryPostfixOperatorNode.java (81%) rename src/sqlancer/{ => common}/ast/newast/NewUnaryPrefixOperatorNode.java (81%) create mode 100644 src/sqlancer/common/ast/newast/Node.java rename src/sqlancer/{ => common}/ast/newast/TableReferenceNode.java (74%) rename src/sqlancer/{ => common}/gen/AbstractGenerator.java (94%) rename src/sqlancer/{ => common}/gen/AbstractInsertGenerator.java (96%) rename src/sqlancer/{ => common}/gen/ExpressionGenerator.java (96%) rename src/sqlancer/{ => common}/gen/TypedExpressionGenerator.java (98%) rename src/sqlancer/{ => common}/gen/UntypedExpressionGenerator.java (98%) rename src/sqlancer/{ => common}/schema/AbstractSchema.java (98%) rename src/sqlancer/{ => common}/schema/AbstractTable.java (98%) rename src/sqlancer/{ => common}/schema/AbstractTableColumn.java (98%) rename src/sqlancer/{ => common}/schema/AbstractTables.java (97%) rename src/sqlancer/{ => common}/schema/TableIndex.java (92%) rename src/sqlancer/{ => common}/visitor/BinaryOperation.java (78%) rename src/sqlancer/{ => common}/visitor/NodeVisitor.java (50%) rename src/sqlancer/{ => common}/visitor/TernaryOperation.java (75%) rename src/sqlancer/{ => common}/visitor/ToStringVisitor.java (95%) rename src/sqlancer/{ => common}/visitor/UnaryOperation.java (89%) diff --git a/src/sqlancer/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/TernaryLogicPartitioningOracleBase.java index 76bc9b31d..8b6f0e54b 100644 --- a/src/sqlancer/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/TernaryLogicPartitioningOracleBase.java @@ -1,6 +1,6 @@ package sqlancer; -import sqlancer.gen.ExpressionGenerator; +import sqlancer.common.gen.ExpressionGenerator; /** * This is the base class of the Ternary Logic Partitioning (TLP) oracles. The core idea of TLP is to partition a given diff --git a/src/sqlancer/ast/newast/Node.java b/src/sqlancer/ast/newast/Node.java deleted file mode 100644 index ae45e1e43..000000000 --- a/src/sqlancer/ast/newast/Node.java +++ /dev/null @@ -1,5 +0,0 @@ -package sqlancer.ast.newast; - -public interface Node { - -} diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index f42775470..efece12dd 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -18,10 +18,10 @@ import sqlancer.StateToReproduce; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ast.ClickHouseConstant; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.TableIndex; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; public class ClickHouseSchema extends AbstractSchema { diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index 877a6d5fa..beaa37b49 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -11,7 +11,7 @@ import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.visitor.ToStringVisitor; +import sqlancer.common.visitor.ToStringVisitor; public class ClickHouseToStringVisitor extends ToStringVisitor implements ClickHouseVisitor { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java index 76115c6d1..6696a3ee3 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java @@ -3,7 +3,7 @@ import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; -import sqlancer.visitor.BinaryOperation; +import sqlancer.common.visitor.BinaryOperation; public class ClickHouseBinaryComparisonOperation extends ClickHouseExpression implements BinaryOperation { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index c32e26131..c3642c208 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -1,7 +1,7 @@ package sqlancer.clickhouse.ast; import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public abstract class ClickHouseExpression { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java index 490980a02..553f85070 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java @@ -1,8 +1,8 @@ package sqlancer.clickhouse.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.visitor.UnaryOperation; public class ClickHouseUnaryPostfixOperation extends ClickHouseExpression implements UnaryOperation { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java index 41f91ef56..972136e64 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java @@ -2,8 +2,8 @@ import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.visitor.UnaryOperation; public class ClickHouseUnaryPrefixOperation extends ClickHouseExpression implements UnaryOperation { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 9943f455f..3bb29e75d 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -20,7 +20,7 @@ import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; -import sqlancer.gen.TypedExpressionGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; public class ClickHouseExpressionGenerator extends TypedExpressionGenerator { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 83e1fc997..af9c1b746 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -10,8 +10,8 @@ import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.clickhouse.ClickHouseToStringVisitor; -import sqlancer.gen.AbstractInsertGenerator; public class ClickHouseInsertGenerator extends AbstractInsertGenerator { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 39462589f..8df66fb9f 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -19,7 +19,7 @@ import sqlancer.clickhouse.ast.ClickHouseSelect; import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import sqlancer.gen.ExpressionGenerator; +import sqlancer.common.gen.ExpressionGenerator; public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 0379f7cb7..bb1b18664 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -9,11 +9,11 @@ import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; public class CockroachDBSchema extends AbstractSchema { diff --git a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java index 80e923d69..17a09a401 100644 --- a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java @@ -15,7 +15,7 @@ import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.visitor.ToStringVisitor; +import sqlancer.common.visitor.ToStringVisitor; public class CockroachDBToStringVisitor extends ToStringVisitor implements CockroachDBVisitor { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java b/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java index 9feb569af..3f083b5e3 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class CockroachDBAlias implements UnaryOperation, CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryArithmeticOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryArithmeticOperation.java index 0eb4da615..57e80dd61 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryArithmeticOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryArithmeticOperation.java @@ -1,9 +1,9 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; import sqlancer.cockroachdb.ast.CockroachDBBinaryArithmeticOperation.CockroachDBBinaryArithmeticOperator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class CockroachDBBinaryArithmeticOperation extends BinaryOperatorNode diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryComparisonOperator.java b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryComparisonOperator.java index 39ae85e7c..00a4ecf9c 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryComparisonOperator.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryComparisonOperator.java @@ -1,9 +1,9 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; import sqlancer.cockroachdb.ast.CockroachDBBinaryComparisonOperator.CockroachDBComparisonOperator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class CockroachDBBinaryComparisonOperator extends BinaryOperatorNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryLogicalOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryLogicalOperation.java index 352238afb..15695136a 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryLogicalOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryLogicalOperation.java @@ -1,9 +1,9 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation.CockroachDBBinaryLogicalOperator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class CockroachDBBinaryLogicalOperation extends BinaryOperatorNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBCast.java b/src/sqlancer/cockroachdb/ast/CockroachDBCast.java index 79fb19929..d4b54e4da 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBCast.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBCast.java @@ -1,7 +1,7 @@ package sqlancer.cockroachdb.ast; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBCompositeDataType; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class CockroachDBCast implements UnaryOperation, CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBCollate.java b/src/sqlancer/cockroachdb/ast/CockroachDBCollate.java index fe10a8c76..55ec5c99b 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBCollate.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBCollate.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.UnaryNode; +import sqlancer.common.ast.UnaryNode; public class CockroachDBCollate extends UnaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBConcatOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBConcatOperation.java index 9cab969fa..bb71c6ab4 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBConcatOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBConcatOperation.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.BinaryNode; +import sqlancer.common.ast.BinaryNode; public class CockroachDBConcatOperation extends BinaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBIndexReference.java b/src/sqlancer/cockroachdb/ast/CockroachDBIndexReference.java index e9410acce..d9f7f6874 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBIndexReference.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBIndexReference.java @@ -1,8 +1,8 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.schema.TableIndex; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.schema.TableIndex; +import sqlancer.common.visitor.UnaryOperation; public class CockroachDBIndexReference extends CockroachDBTableReference implements UnaryOperation { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBNotOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBNotOperation.java index 72f6d8e31..05ca2251e 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBNotOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBNotOperation.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.UnaryNode; +import sqlancer.common.ast.UnaryNode; public class CockroachDBNotOperation extends UnaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBOrderingTerm.java b/src/sqlancer/cockroachdb/ast/CockroachDBOrderingTerm.java index 1d23e9167..905bc7771 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBOrderingTerm.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBOrderingTerm.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class CockroachDBOrderingTerm implements UnaryOperation, CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBRegexOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBRegexOperation.java index a4609b63e..24a75a92f 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBRegexOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBRegexOperation.java @@ -1,7 +1,7 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryNode; +import sqlancer.common.ast.BinaryNode; public class CockroachDBRegexOperation extends BinaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java index f8d6c1a1a..2c7d33ff4 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.SelectBase; +import sqlancer.common.ast.SelectBase; public class CockroachDBSelect extends SelectBase implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBTypeAnnotation.java b/src/sqlancer/cockroachdb/ast/CockroachDBTypeAnnotation.java index acab86dcb..b47d2147f 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBTypeAnnotation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBTypeAnnotation.java @@ -1,7 +1,7 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.UnaryNode; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBCompositeDataType; +import sqlancer.common.ast.UnaryNode; public class CockroachDBTypeAnnotation extends UnaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java index aadc17626..777fc4205 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java @@ -1,9 +1,9 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; import sqlancer.cockroachdb.ast.CockroachDBUnaryArithmeticOperation.CockroachDBUnaryAritmeticOperator; +import sqlancer.common.ast.UnaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class CockroachDBUnaryArithmeticOperation extends UnaryOperatorNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java index 581a66088..140c0c9cc 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java @@ -1,9 +1,9 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; +import sqlancer.common.ast.UnaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class CockroachDBUnaryPostfixOperation extends UnaryOperatorNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java index 1eec56f09..95df4298a 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java @@ -11,7 +11,7 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.schema.TableIndex; +import sqlancer.common.schema.TableIndex; public final class CockroachDBCommentOnGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index 5fe4d6d0d..6326bcc95 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -39,7 +39,7 @@ import sqlancer.cockroachdb.ast.CockroachDBTypeAnnotation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; -import sqlancer.gen.TypedExpressionGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; public class CockroachDBExpressionGenerator extends TypedExpressionGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java index fe0e0fb55..e06cc8d9d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java @@ -7,7 +7,7 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.gen.AbstractGenerator; +import sqlancer.common.gen.AbstractGenerator; public abstract class CockroachDBGenerator extends AbstractGenerator { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index 4cf51ab62..e81e744f6 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -20,7 +20,7 @@ import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; -import sqlancer.gen.ExpressionGenerator; +import sqlancer.common.gen.ExpressionGenerator; public class CockroachDBTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { diff --git a/src/sqlancer/ast/BinaryNode.java b/src/sqlancer/common/ast/BinaryNode.java similarity index 82% rename from src/sqlancer/ast/BinaryNode.java rename to src/sqlancer/common/ast/BinaryNode.java index 5be42b654..dec1854c4 100644 --- a/src/sqlancer/ast/BinaryNode.java +++ b/src/sqlancer/common/ast/BinaryNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.visitor.BinaryOperation; +import sqlancer.common.visitor.BinaryOperation; public abstract class BinaryNode implements BinaryOperation { diff --git a/src/sqlancer/ast/BinaryOperatorNode.java b/src/sqlancer/common/ast/BinaryOperatorNode.java similarity index 84% rename from src/sqlancer/ast/BinaryOperatorNode.java rename to src/sqlancer/common/ast/BinaryOperatorNode.java index 002cab790..38d340a4b 100644 --- a/src/sqlancer/ast/BinaryOperatorNode.java +++ b/src/sqlancer/common/ast/BinaryOperatorNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public abstract class BinaryOperatorNode extends BinaryNode { diff --git a/src/sqlancer/ast/FunctionNode.java b/src/sqlancer/common/ast/FunctionNode.java similarity index 92% rename from src/sqlancer/ast/FunctionNode.java rename to src/sqlancer/common/ast/FunctionNode.java index 6527a3725..a841d2956 100644 --- a/src/sqlancer/ast/FunctionNode.java +++ b/src/sqlancer/common/ast/FunctionNode.java @@ -1,4 +1,4 @@ -package sqlancer.ast; +package sqlancer.common.ast; import java.util.List; diff --git a/src/sqlancer/ast/SelectBase.java b/src/sqlancer/common/ast/SelectBase.java similarity index 98% rename from src/sqlancer/ast/SelectBase.java rename to src/sqlancer/common/ast/SelectBase.java index 627bcd119..341b00209 100644 --- a/src/sqlancer/ast/SelectBase.java +++ b/src/sqlancer/common/ast/SelectBase.java @@ -1,4 +1,4 @@ -package sqlancer.ast; +package sqlancer.common.ast; import java.util.Collections; import java.util.List; diff --git a/src/sqlancer/ast/TernaryNode.java b/src/sqlancer/common/ast/TernaryNode.java similarity index 86% rename from src/sqlancer/ast/TernaryNode.java rename to src/sqlancer/common/ast/TernaryNode.java index fa96833f8..146a46474 100644 --- a/src/sqlancer/ast/TernaryNode.java +++ b/src/sqlancer/common/ast/TernaryNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.visitor.BinaryOperation; +import sqlancer.common.visitor.BinaryOperation; public abstract class TernaryNode implements BinaryOperation { diff --git a/src/sqlancer/ast/UnaryNode.java b/src/sqlancer/common/ast/UnaryNode.java similarity index 75% rename from src/sqlancer/ast/UnaryNode.java rename to src/sqlancer/common/ast/UnaryNode.java index 6b1e5a485..a6664c864 100644 --- a/src/sqlancer/ast/UnaryNode.java +++ b/src/sqlancer/common/ast/UnaryNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public abstract class UnaryNode implements UnaryOperation { diff --git a/src/sqlancer/ast/UnaryOperatorNode.java b/src/sqlancer/common/ast/UnaryOperatorNode.java similarity index 79% rename from src/sqlancer/ast/UnaryOperatorNode.java rename to src/sqlancer/common/ast/UnaryOperatorNode.java index e54231929..979479c12 100644 --- a/src/sqlancer/ast/UnaryOperatorNode.java +++ b/src/sqlancer/common/ast/UnaryOperatorNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public abstract class UnaryOperatorNode extends UnaryNode { diff --git a/src/sqlancer/ast/newast/ColumnReferenceNode.java b/src/sqlancer/common/ast/newast/ColumnReferenceNode.java similarity index 73% rename from src/sqlancer/ast/newast/ColumnReferenceNode.java rename to src/sqlancer/common/ast/newast/ColumnReferenceNode.java index cfb506431..18ff09db6 100644 --- a/src/sqlancer/ast/newast/ColumnReferenceNode.java +++ b/src/sqlancer/common/ast/newast/ColumnReferenceNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; -import sqlancer.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTableColumn; public class ColumnReferenceNode> implements Node { diff --git a/src/sqlancer/ast/newast/NewAliasNode.java b/src/sqlancer/common/ast/newast/NewAliasNode.java similarity index 90% rename from src/sqlancer/ast/newast/NewAliasNode.java rename to src/sqlancer/common/ast/newast/NewAliasNode.java index 924533214..9d7d84c43 100644 --- a/src/sqlancer/ast/newast/NewAliasNode.java +++ b/src/sqlancer/common/ast/newast/NewAliasNode.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; public class NewAliasNode implements Node { diff --git a/src/sqlancer/ast/newast/NewBetweenOperatorNode.java b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java similarity index 94% rename from src/sqlancer/ast/newast/NewBetweenOperatorNode.java rename to src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java index 91c54a5f1..c5a0acf27 100644 --- a/src/sqlancer/ast/newast/NewBetweenOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; public class NewBetweenOperatorNode implements Node { diff --git a/src/sqlancer/ast/newast/NewBinaryOperatorNode.java b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java similarity index 85% rename from src/sqlancer/ast/newast/NewBinaryOperatorNode.java rename to src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java index 3e703e8bd..2b640ed0c 100644 --- a/src/sqlancer/ast/newast/NewBinaryOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class NewBinaryOperatorNode implements Node { diff --git a/src/sqlancer/ast/newast/NewCaseOperatorNode.java b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java similarity index 96% rename from src/sqlancer/ast/newast/NewCaseOperatorNode.java rename to src/sqlancer/common/ast/newast/NewCaseOperatorNode.java index 3c0173536..04f6be4c5 100644 --- a/src/sqlancer/ast/newast/NewCaseOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; import java.util.List; diff --git a/src/sqlancer/ast/newast/NewFunctionNode.java b/src/sqlancer/common/ast/newast/NewFunctionNode.java similarity index 91% rename from src/sqlancer/ast/newast/NewFunctionNode.java rename to src/sqlancer/common/ast/newast/NewFunctionNode.java index fc54941a8..fae214713 100644 --- a/src/sqlancer/ast/newast/NewFunctionNode.java +++ b/src/sqlancer/common/ast/newast/NewFunctionNode.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; import java.util.List; diff --git a/src/sqlancer/ast/newast/NewInOperatorNode.java b/src/sqlancer/common/ast/newast/NewInOperatorNode.java similarity index 94% rename from src/sqlancer/ast/newast/NewInOperatorNode.java rename to src/sqlancer/common/ast/newast/NewInOperatorNode.java index 0eb88c81a..346156666 100644 --- a/src/sqlancer/ast/newast/NewInOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewInOperatorNode.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; import java.util.List; diff --git a/src/sqlancer/ast/newast/NewOrderingTerm.java b/src/sqlancer/common/ast/newast/NewOrderingTerm.java similarity index 94% rename from src/sqlancer/ast/newast/NewOrderingTerm.java rename to src/sqlancer/common/ast/newast/NewOrderingTerm.java index 6328070ae..5afe71821 100644 --- a/src/sqlancer/ast/newast/NewOrderingTerm.java +++ b/src/sqlancer/common/ast/newast/NewOrderingTerm.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; import sqlancer.Randomly; diff --git a/src/sqlancer/ast/newast/NewPostfixTextNode.java b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java similarity index 90% rename from src/sqlancer/ast/newast/NewPostfixTextNode.java rename to src/sqlancer/common/ast/newast/NewPostfixTextNode.java index 270d191ff..70e65b17b 100644 --- a/src/sqlancer/ast/newast/NewPostfixTextNode.java +++ b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; public class NewPostfixTextNode implements Node { diff --git a/src/sqlancer/ast/newast/NewTernaryNode.java b/src/sqlancer/common/ast/newast/NewTernaryNode.java similarity index 95% rename from src/sqlancer/ast/newast/NewTernaryNode.java rename to src/sqlancer/common/ast/newast/NewTernaryNode.java index 1ec9a6c35..f40c480ec 100644 --- a/src/sqlancer/ast/newast/NewTernaryNode.java +++ b/src/sqlancer/common/ast/newast/NewTernaryNode.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; public class NewTernaryNode implements Node { diff --git a/src/sqlancer/ast/newast/NewToStringVisitor.java b/src/sqlancer/common/ast/newast/NewToStringVisitor.java similarity index 99% rename from src/sqlancer/ast/newast/NewToStringVisitor.java rename to src/sqlancer/common/ast/newast/NewToStringVisitor.java index 6c275684a..3ad625440 100644 --- a/src/sqlancer/ast/newast/NewToStringVisitor.java +++ b/src/sqlancer/common/ast/newast/NewToStringVisitor.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; import java.util.List; diff --git a/src/sqlancer/ast/newast/NewUnaryPostfixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java similarity index 81% rename from src/sqlancer/ast/newast/NewUnaryPostfixOperatorNode.java rename to src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java index d02bfbaca..ebd17945f 100644 --- a/src/sqlancer/ast/newast/NewUnaryPostfixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class NewUnaryPostfixOperatorNode implements Node { diff --git a/src/sqlancer/ast/newast/NewUnaryPrefixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java similarity index 81% rename from src/sqlancer/ast/newast/NewUnaryPrefixOperatorNode.java rename to src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java index 87fa0b8f7..5274bd942 100644 --- a/src/sqlancer/ast/newast/NewUnaryPrefixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class NewUnaryPrefixOperatorNode implements Node { diff --git a/src/sqlancer/common/ast/newast/Node.java b/src/sqlancer/common/ast/newast/Node.java new file mode 100644 index 000000000..310e875c0 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Node.java @@ -0,0 +1,5 @@ +package sqlancer.common.ast.newast; + +public interface Node { + +} diff --git a/src/sqlancer/ast/newast/TableReferenceNode.java b/src/sqlancer/common/ast/newast/TableReferenceNode.java similarity index 74% rename from src/sqlancer/ast/newast/TableReferenceNode.java rename to src/sqlancer/common/ast/newast/TableReferenceNode.java index b1b59f016..b350a9af2 100644 --- a/src/sqlancer/ast/newast/TableReferenceNode.java +++ b/src/sqlancer/common/ast/newast/TableReferenceNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; -import sqlancer.schema.AbstractTable; +import sqlancer.common.schema.AbstractTable; public class TableReferenceNode> implements Node { diff --git a/src/sqlancer/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java similarity index 94% rename from src/sqlancer/gen/AbstractGenerator.java rename to src/sqlancer/common/gen/AbstractGenerator.java index d4fe512f6..17c412a7e 100644 --- a/src/sqlancer/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -1,4 +1,4 @@ -package sqlancer.gen; +package sqlancer.common.gen; import sqlancer.ExpectedErrors; import sqlancer.Query; diff --git a/src/sqlancer/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java similarity index 96% rename from src/sqlancer/gen/AbstractInsertGenerator.java rename to src/sqlancer/common/gen/AbstractInsertGenerator.java index e04def13b..61357fd10 100644 --- a/src/sqlancer/gen/AbstractInsertGenerator.java +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -1,4 +1,4 @@ -package sqlancer.gen; +package sqlancer.common.gen; import java.util.List; diff --git a/src/sqlancer/gen/ExpressionGenerator.java b/src/sqlancer/common/gen/ExpressionGenerator.java similarity index 96% rename from src/sqlancer/gen/ExpressionGenerator.java rename to src/sqlancer/common/gen/ExpressionGenerator.java index e3f2f05ec..4d5cff1e2 100644 --- a/src/sqlancer/gen/ExpressionGenerator.java +++ b/src/sqlancer/common/gen/ExpressionGenerator.java @@ -1,4 +1,4 @@ -package sqlancer.gen; +package sqlancer.common.gen; public interface ExpressionGenerator { diff --git a/src/sqlancer/gen/TypedExpressionGenerator.java b/src/sqlancer/common/gen/TypedExpressionGenerator.java similarity index 98% rename from src/sqlancer/gen/TypedExpressionGenerator.java rename to src/sqlancer/common/gen/TypedExpressionGenerator.java index a5c23d867..3ee7115fe 100644 --- a/src/sqlancer/gen/TypedExpressionGenerator.java +++ b/src/sqlancer/common/gen/TypedExpressionGenerator.java @@ -1,4 +1,4 @@ -package sqlancer.gen; +package sqlancer.common.gen; import java.util.ArrayList; import java.util.Collections; diff --git a/src/sqlancer/gen/UntypedExpressionGenerator.java b/src/sqlancer/common/gen/UntypedExpressionGenerator.java similarity index 98% rename from src/sqlancer/gen/UntypedExpressionGenerator.java rename to src/sqlancer/common/gen/UntypedExpressionGenerator.java index 8565b15e0..b52989624 100644 --- a/src/sqlancer/gen/UntypedExpressionGenerator.java +++ b/src/sqlancer/common/gen/UntypedExpressionGenerator.java @@ -1,4 +1,4 @@ -package sqlancer.gen; +package sqlancer.common.gen; import java.util.ArrayList; import java.util.List; diff --git a/src/sqlancer/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java similarity index 98% rename from src/sqlancer/schema/AbstractSchema.java rename to src/sqlancer/common/schema/AbstractSchema.java index df0e61f26..f0905df03 100644 --- a/src/sqlancer/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -1,4 +1,4 @@ -package sqlancer.schema; +package sqlancer.common.schema; import java.util.Collections; import java.util.List; diff --git a/src/sqlancer/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java similarity index 98% rename from src/sqlancer/schema/AbstractTable.java rename to src/sqlancer/common/schema/AbstractTable.java index fe052cc3c..594bde47f 100644 --- a/src/sqlancer/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -1,4 +1,4 @@ -package sqlancer.schema; +package sqlancer.common.schema; import java.util.Collections; import java.util.List; diff --git a/src/sqlancer/schema/AbstractTableColumn.java b/src/sqlancer/common/schema/AbstractTableColumn.java similarity index 98% rename from src/sqlancer/schema/AbstractTableColumn.java rename to src/sqlancer/common/schema/AbstractTableColumn.java index 978ef38a0..17ddace8f 100644 --- a/src/sqlancer/schema/AbstractTableColumn.java +++ b/src/sqlancer/common/schema/AbstractTableColumn.java @@ -1,4 +1,4 @@ -package sqlancer.schema; +package sqlancer.common.schema; public class AbstractTableColumn, U> implements Comparable> { diff --git a/src/sqlancer/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java similarity index 97% rename from src/sqlancer/schema/AbstractTables.java rename to src/sqlancer/common/schema/AbstractTables.java index 57dfcc903..c7f44e688 100644 --- a/src/sqlancer/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -1,4 +1,4 @@ -package sqlancer.schema; +package sqlancer.common.schema; import java.util.ArrayList; import java.util.List; diff --git a/src/sqlancer/schema/TableIndex.java b/src/sqlancer/common/schema/TableIndex.java similarity index 92% rename from src/sqlancer/schema/TableIndex.java rename to src/sqlancer/common/schema/TableIndex.java index 7d21fbf40..325fe1144 100644 --- a/src/sqlancer/schema/TableIndex.java +++ b/src/sqlancer/common/schema/TableIndex.java @@ -1,4 +1,4 @@ -package sqlancer.schema; +package sqlancer.common.schema; public class TableIndex { diff --git a/src/sqlancer/visitor/BinaryOperation.java b/src/sqlancer/common/visitor/BinaryOperation.java similarity index 78% rename from src/sqlancer/visitor/BinaryOperation.java rename to src/sqlancer/common/visitor/BinaryOperation.java index 26fb81c23..3e9ed9cbf 100644 --- a/src/sqlancer/visitor/BinaryOperation.java +++ b/src/sqlancer/common/visitor/BinaryOperation.java @@ -1,4 +1,4 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; public interface BinaryOperation { diff --git a/src/sqlancer/visitor/NodeVisitor.java b/src/sqlancer/common/visitor/NodeVisitor.java similarity index 50% rename from src/sqlancer/visitor/NodeVisitor.java rename to src/sqlancer/common/visitor/NodeVisitor.java index 393b012dd..68fd73c2a 100644 --- a/src/sqlancer/visitor/NodeVisitor.java +++ b/src/sqlancer/common/visitor/NodeVisitor.java @@ -1,4 +1,4 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; public class NodeVisitor { diff --git a/src/sqlancer/visitor/TernaryOperation.java b/src/sqlancer/common/visitor/TernaryOperation.java similarity index 75% rename from src/sqlancer/visitor/TernaryOperation.java rename to src/sqlancer/common/visitor/TernaryOperation.java index 9ac64e279..fbab64e95 100644 --- a/src/sqlancer/visitor/TernaryOperation.java +++ b/src/sqlancer/common/visitor/TernaryOperation.java @@ -1,4 +1,4 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; public interface TernaryOperation { diff --git a/src/sqlancer/visitor/ToStringVisitor.java b/src/sqlancer/common/visitor/ToStringVisitor.java similarity index 95% rename from src/sqlancer/visitor/ToStringVisitor.java rename to src/sqlancer/common/visitor/ToStringVisitor.java index 50168e4c7..c2385b94d 100644 --- a/src/sqlancer/visitor/ToStringVisitor.java +++ b/src/sqlancer/common/visitor/ToStringVisitor.java @@ -1,8 +1,8 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; import java.util.List; -import sqlancer.visitor.UnaryOperation.OperatorKind; +import sqlancer.common.visitor.UnaryOperation.OperatorKind; public abstract class ToStringVisitor extends NodeVisitor { diff --git a/src/sqlancer/visitor/UnaryOperation.java b/src/sqlancer/common/visitor/UnaryOperation.java similarity index 89% rename from src/sqlancer/visitor/UnaryOperation.java rename to src/sqlancer/common/visitor/UnaryOperation.java index c0b5fe9e4..9d937085e 100644 --- a/src/sqlancer/visitor/UnaryOperation.java +++ b/src/sqlancer/common/visitor/UnaryOperation.java @@ -1,4 +1,4 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; public interface UnaryOperation { diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 5daec973b..aea5839da 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -9,12 +9,12 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; public class DuckDBSchema extends AbstractSchema { diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index f35e3445e..f7c97f3e5 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -1,7 +1,7 @@ package sqlancer.duckdb; -import sqlancer.ast.newast.NewToStringVisitor; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; import sqlancer.duckdb.ast.DuckDBConstant; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; diff --git a/src/sqlancer/duckdb/ast/DuckDBConstant.java b/src/sqlancer/duckdb/ast/DuckDBConstant.java index d5b65e50d..b895a18e5 100644 --- a/src/sqlancer/duckdb/ast/DuckDBConstant.java +++ b/src/sqlancer/duckdb/ast/DuckDBConstant.java @@ -3,7 +3,7 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.Node; public class DuckDBConstant implements Node { diff --git a/src/sqlancer/duckdb/ast/DuckDBJoin.java b/src/sqlancer/duckdb/ast/DuckDBJoin.java index 42e0ea61f..c1c0c94a9 100644 --- a/src/sqlancer/duckdb/ast/DuckDBJoin.java +++ b/src/sqlancer/duckdb/ast/DuckDBJoin.java @@ -4,8 +4,8 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; -import sqlancer.ast.newast.TableReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; diff --git a/src/sqlancer/duckdb/ast/DuckDBSelect.java b/src/sqlancer/duckdb/ast/DuckDBSelect.java index 5e92aa685..9a2391aa3 100644 --- a/src/sqlancer/duckdb/ast/DuckDBSelect.java +++ b/src/sqlancer/duckdb/ast/DuckDBSelect.java @@ -1,7 +1,7 @@ package sqlancer.duckdb.ast; -import sqlancer.ast.SelectBase; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; public class DuckDBSelect extends SelectBase> implements Node { diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index eebee9d18..11ce516b4 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -6,26 +6,26 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.newast.ColumnReferenceNode; -import sqlancer.ast.newast.NewBetweenOperatorNode; -import sqlancer.ast.newast.NewBinaryOperatorNode; -import sqlancer.ast.newast.NewCaseOperatorNode; -import sqlancer.ast.newast.NewFunctionNode; -import sqlancer.ast.newast.NewInOperatorNode; -import sqlancer.ast.newast.NewOrderingTerm; -import sqlancer.ast.newast.NewOrderingTerm.Ordering; -import sqlancer.ast.newast.NewTernaryNode; -import sqlancer.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewCaseOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.NewTernaryNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; import sqlancer.duckdb.ast.DuckDBConstant; import sqlancer.duckdb.ast.DuckDBExpression; -import sqlancer.gen.UntypedExpressionGenerator; public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator, DuckDBColumn> { diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 6429e2ae4..66aec013d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.Node; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index e15b9c355..a5046eaf6 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -7,12 +7,12 @@ import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBToStringVisitor; -import sqlancer.gen.AbstractInsertGenerator; public class DuckDBInsertGenerator extends AbstractInsertGenerator { diff --git a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java index f48c4474d..29af42e10 100644 --- a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java +++ b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java @@ -5,8 +5,8 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; -import sqlancer.ast.newast.TableReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBSchema.DuckDBTables; diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index eb394be55..540749f86 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -8,7 +8,8 @@ import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; @@ -16,7 +17,6 @@ import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -import sqlancer.gen.UntypedExpressionGenerator; public class DuckDBTableGenerator { diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 0a2e2ed19..2c86a0a93 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.Node; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index f4fd87336..b4935df34 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -15,10 +15,10 @@ import sqlancer.Randomly; import sqlancer.SQLancerResultSet; import sqlancer.TestOracle; -import sqlancer.ast.newast.ColumnReferenceNode; -import sqlancer.ast.newast.NewPostfixTextNode; -import sqlancer.ast.newast.Node; -import sqlancer.ast.newast.TableReferenceNode; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index 6bcc0c233..61983a069 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -11,12 +11,12 @@ import sqlancer.Randomly; import sqlancer.SQLancerResultSet; import sqlancer.TestOracle; -import sqlancer.ast.newast.NewAliasNode; -import sqlancer.ast.newast.NewBinaryOperatorNode; -import sqlancer.ast.newast.NewFunctionNode; -import sqlancer.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 6e8cee2d8..8b64b95f0 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -8,9 +8,10 @@ import sqlancer.Randomly; import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; -import sqlancer.ast.newast.ColumnReferenceNode; -import sqlancer.ast.newast.Node; -import sqlancer.ast.newast.TableReferenceNode; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema; @@ -21,7 +22,6 @@ import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBSelect; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -import sqlancer.gen.ExpressionGenerator; public class DuckDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, DuckDBGlobalState> implements TestOracle { diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java index 81b6f28dd..3cd6356fa 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java @@ -7,8 +7,8 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.ast.newast.ColumnReferenceNode; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index 4215bf080..569bee295 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -8,7 +8,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.TestOracle; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.Node; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBToStringVisitor; diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 49b263248..715180003 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -13,14 +13,14 @@ import sqlancer.Randomly; import sqlancer.StateToReproduce.MySQLStateToReproduce; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; import sqlancer.mysql.ast.MySQLConstant; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; public class MySQLSchema extends AbstractSchema { diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index af34daaa4..06f13b008 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -4,6 +4,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.visitor.ToStringVisitor; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -22,7 +23,6 @@ import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; -import sqlancer.visitor.ToStringVisitor; public class MySQLToStringVisitor extends ToStringVisitor implements MySQLVisitor { diff --git a/src/sqlancer/mysql/ast/MySQLCollate.java b/src/sqlancer/mysql/ast/MySQLCollate.java index a0dc5c4e0..4512486f4 100644 --- a/src/sqlancer/mysql/ast/MySQLCollate.java +++ b/src/sqlancer/mysql/ast/MySQLCollate.java @@ -1,6 +1,6 @@ package sqlancer.mysql.ast; -import sqlancer.ast.UnaryNode; +import sqlancer.common.ast.UnaryNode; public class MySQLCollate extends UnaryNode implements MySQLExpression { diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index 8e48f7dac..7abd1f639 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -3,7 +3,7 @@ import java.util.Collections; import java.util.List; -import sqlancer.ast.SelectBase; +import sqlancer.common.ast.SelectBase; public class MySQLSelect extends SelectBase implements MySQLExpression { diff --git a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java index 6bf362f2e..0121deca0 100644 --- a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java +++ b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java @@ -2,8 +2,8 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; +import sqlancer.common.ast.UnaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; public class MySQLUnaryPrefixOperation extends UnaryOperatorNode diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 76af4a059..8f2ec6d6e 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -5,7 +5,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.gen.UntypedExpressionGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index c01b39aea..0026b1f39 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -7,7 +7,7 @@ import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; -import sqlancer.gen.ExpressionGenerator; +import sqlancer.common.gen.ExpressionGenerator; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 641acfcf0..170dfe78b 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -17,12 +17,12 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.StateToReproduce.PostgresStateToReproduce; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; import sqlancer.postgres.ast.PostgresConstant; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; public class PostgresSchema { diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 2da562f98..121fa1426 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -3,6 +3,7 @@ import java.util.Optional; import sqlancer.Randomly; +import sqlancer.common.visitor.ToStringVisitor; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresBetweenOperation; @@ -24,7 +25,6 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; -import sqlancer.visitor.ToStringVisitor; public final class PostgresToStringVisitor extends ToStringVisitor implements PostgresVisitor { diff --git a/src/sqlancer/postgres/ast/PostgresAggregate.java b/src/sqlancer/postgres/ast/PostgresAggregate.java index 9159cb191..bcf48d46e 100644 --- a/src/sqlancer/postgres/ast/PostgresAggregate.java +++ b/src/sqlancer/postgres/ast/PostgresAggregate.java @@ -5,7 +5,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.ast.FunctionNode; +import sqlancer.common.ast.FunctionNode; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate.PostgresAggregateFunction; diff --git a/src/sqlancer/postgres/ast/PostgresAlias.java b/src/sqlancer/postgres/ast/PostgresAlias.java index 38247aeec..9f46084b1 100644 --- a/src/sqlancer/postgres/ast/PostgresAlias.java +++ b/src/sqlancer/postgres/ast/PostgresAlias.java @@ -1,6 +1,6 @@ package sqlancer.postgres.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class PostgresAlias implements UnaryOperation, PostgresExpression { diff --git a/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java index 95639f34e..2c362efcd 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java @@ -3,8 +3,8 @@ import java.util.function.BinaryOperator; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresBinaryArithmeticOperation.PostgresBinaryOperator; diff --git a/src/sqlancer/postgres/ast/PostgresBinaryBitOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryBitOperation.java index 4ef9c690a..d111386da 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryBitOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryBitOperation.java @@ -1,8 +1,8 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresBinaryBitOperation.PostgresBinaryBitOperator; diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index ae4f9a16a..8f17be89c 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -1,8 +1,8 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator; diff --git a/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java index 63bc39ea6..5809ced09 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java @@ -1,8 +1,8 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresBinaryLogicalOperation.BinaryLogicalOperator; diff --git a/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java index 5f0f246b5..e3e522de1 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java @@ -1,8 +1,8 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresBinaryRangeOperation extends BinaryNode implements PostgresExpression { diff --git a/src/sqlancer/postgres/ast/PostgresConcatOperation.java b/src/sqlancer/postgres/ast/PostgresConcatOperation.java index b2e8f6542..b83633e1d 100644 --- a/src/sqlancer/postgres/ast/PostgresConcatOperation.java +++ b/src/sqlancer/postgres/ast/PostgresConcatOperation.java @@ -1,6 +1,6 @@ package sqlancer.postgres.ast; -import sqlancer.ast.BinaryNode; +import sqlancer.common.ast.BinaryNode; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresConcatOperation extends BinaryNode implements PostgresExpression { diff --git a/src/sqlancer/postgres/ast/PostgresLikeOperation.java b/src/sqlancer/postgres/ast/PostgresLikeOperation.java index 9504f3232..6971b1b11 100644 --- a/src/sqlancer/postgres/ast/PostgresLikeOperation.java +++ b/src/sqlancer/postgres/ast/PostgresLikeOperation.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; import sqlancer.LikeImplementationHelper; -import sqlancer.ast.BinaryNode; +import sqlancer.common.ast.BinaryNode; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresLikeOperation extends BinaryNode implements PostgresExpression { diff --git a/src/sqlancer/postgres/ast/PostgresPOSIXRegularExpression.java b/src/sqlancer/postgres/ast/PostgresPOSIXRegularExpression.java index a1e4a0362..b0affaf05 100644 --- a/src/sqlancer/postgres/ast/PostgresPOSIXRegularExpression.java +++ b/src/sqlancer/postgres/ast/PostgresPOSIXRegularExpression.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresPOSIXRegularExpression implements PostgresExpression { diff --git a/src/sqlancer/postgres/ast/PostgresPostfixOperation.java b/src/sqlancer/postgres/ast/PostgresPostfixOperation.java index 779945348..4eea1fe90 100644 --- a/src/sqlancer/postgres/ast/PostgresPostfixOperation.java +++ b/src/sqlancer/postgres/ast/PostgresPostfixOperation.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresPostfixOperation implements PostgresExpression { diff --git a/src/sqlancer/postgres/ast/PostgresPrefixOperation.java b/src/sqlancer/postgres/ast/PostgresPrefixOperation.java index 6e875b635..f517c57f2 100644 --- a/src/sqlancer/postgres/ast/PostgresPrefixOperation.java +++ b/src/sqlancer/postgres/ast/PostgresPrefixOperation.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; import sqlancer.IgnoreMeException; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresPrefixOperation implements PostgresExpression { diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index ef2f6e917..70a42650f 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -4,7 +4,7 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.ast.SelectBase; +import sqlancer.common.ast.SelectBase; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 9c1091bf7..a1b105141 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -10,7 +10,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.gen.ExpressionGenerator; +import sqlancer.common.gen.ExpressionGenerator; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 39c920704..2c5fea811 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -9,7 +9,7 @@ import sqlancer.Randomly; import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; -import sqlancer.gen.ExpressionGenerator; +import sqlancer.common.gen.ExpressionGenerator; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 2b3368634..379e24885 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -3,6 +3,7 @@ import java.util.Arrays; import sqlancer.Randomly; +import sqlancer.common.visitor.ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; import sqlancer.sqlite3.ast.SQLite3Case.CasePair; @@ -33,7 +34,6 @@ import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecBetween; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecTerm; -import sqlancer.visitor.ToStringVisitor; public class SQLite3ToStringVisitor extends ToStringVisitor implements SQLite3Visitor { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index fa53cdeed..6172c8dac 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -6,6 +6,8 @@ import sqlancer.IgnoreMeException; import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.UnaryOperation; import sqlancer.sqlite3.SQLite3CollateHelper; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.sqlite3.ast.SQLite3Expression.BinaryComparisonOperation.BinaryComparisonOperator; @@ -15,8 +17,6 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -import sqlancer.visitor.BinaryOperation; -import sqlancer.visitor.UnaryOperation; public abstract class SQLite3Expression { diff --git a/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java b/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java index 319b874e0..9247ef5cc 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java +++ b/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java @@ -3,10 +3,10 @@ import java.util.Optional; import sqlancer.Randomly; +import sqlancer.common.visitor.UnaryOperation; import sqlancer.sqlite3.SQLite3CollateHelper; import sqlancer.sqlite3.schema.SQLite3DataType; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; -import sqlancer.visitor.UnaryOperation; public class SQLite3UnaryOperation extends SQLite3Expression implements UnaryOperation { diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 8963e4366..e548679fd 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -7,7 +7,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.gen.ExpressionGenerator; +import sqlancer.common.gen.ExpressionGenerator; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index 7118c9df0..1c498035d 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; -import sqlancer.gen.ExpressionGenerator; +import sqlancer.common.gen.ExpressionGenerator; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Expression; diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 91a569094..2fb08d6cd 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -20,9 +20,9 @@ import sqlancer.Randomly; import sqlancer.SQLancerResultSet; import sqlancer.StateToReproduce.SQLite3StateToReproduce; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.TableIndex; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index 7440c0da3..dc378cc42 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -6,7 +6,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.gen.UntypedExpressionGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; import sqlancer.tidb.TiDBSchema.TiDBDataType; diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index c502cc479..a67930315 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -8,11 +8,11 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; import sqlancer.tidb.TiDBSchema.TiDBTable; public class TiDBSchema extends AbstractSchema { diff --git a/src/sqlancer/tidb/ast/TiDBAggregate.java b/src/sqlancer/tidb/ast/TiDBAggregate.java index 5f5190f11..21bd8eb0c 100644 --- a/src/sqlancer/tidb/ast/TiDBAggregate.java +++ b/src/sqlancer/tidb/ast/TiDBAggregate.java @@ -3,7 +3,7 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.ast.FunctionNode; +import sqlancer.common.ast.FunctionNode; import sqlancer.tidb.ast.TiDBAggregate.TiDBAggregateFunction; public class TiDBAggregate extends FunctionNode implements TiDBExpression { diff --git a/src/sqlancer/tidb/ast/TiDBBinaryArithmeticOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryArithmeticOperation.java index 1669bac6c..e26284f6e 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryArithmeticOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryArithmeticOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBBinaryArithmeticOperation.TiDBBinaryArithmeticOperator; public class TiDBBinaryArithmeticOperation extends BinaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBBinaryBitOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryBitOperation.java index a5127c99b..c97360dbd 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryBitOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryBitOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBBinaryBitOperation.TiDBBinaryBitOperator; public class TiDBBinaryBitOperation extends BinaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java index 4d3e13ea3..4efc554f0 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBBinaryComparisonOperation.TiDBComparisonOperator; public class TiDBBinaryComparisonOperation extends BinaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBBinaryLogicalOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryLogicalOperation.java index 732d4b3ee..51553a373 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryLogicalOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryLogicalOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; public class TiDBBinaryLogicalOperation extends BinaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBCollate.java b/src/sqlancer/tidb/ast/TiDBCollate.java index 1b74faa6d..08ae71bac 100644 --- a/src/sqlancer/tidb/ast/TiDBCollate.java +++ b/src/sqlancer/tidb/ast/TiDBCollate.java @@ -1,6 +1,6 @@ package sqlancer.tidb.ast; -import sqlancer.ast.UnaryNode; +import sqlancer.common.ast.UnaryNode; public class TiDBCollate extends UnaryNode implements TiDBExpression { diff --git a/src/sqlancer/tidb/ast/TiDBOrderingTerm.java b/src/sqlancer/tidb/ast/TiDBOrderingTerm.java index d11da943a..d735e142b 100644 --- a/src/sqlancer/tidb/ast/TiDBOrderingTerm.java +++ b/src/sqlancer/tidb/ast/TiDBOrderingTerm.java @@ -1,6 +1,6 @@ package sqlancer.tidb.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class TiDBOrderingTerm implements UnaryOperation, TiDBExpression { diff --git a/src/sqlancer/tidb/ast/TiDBRegexOperation.java b/src/sqlancer/tidb/ast/TiDBRegexOperation.java index 07ce00eb0..f5f011dd6 100644 --- a/src/sqlancer/tidb/ast/TiDBRegexOperation.java +++ b/src/sqlancer/tidb/ast/TiDBRegexOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBRegexOperation.TiDBRegexOperator; public class TiDBRegexOperation extends BinaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBSelect.java b/src/sqlancer/tidb/ast/TiDBSelect.java index 54ccf9aa5..f812ee8e0 100644 --- a/src/sqlancer/tidb/ast/TiDBSelect.java +++ b/src/sqlancer/tidb/ast/TiDBSelect.java @@ -1,6 +1,6 @@ package sqlancer.tidb.ast; -import sqlancer.ast.SelectBase; +import sqlancer.common.ast.SelectBase; public class TiDBSelect extends SelectBase implements TiDBExpression { diff --git a/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java b/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java index 9b22e579b..b4d04bf5a 100644 --- a/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java +++ b/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; +import sqlancer.common.ast.UnaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBUnaryPostfixOperation.TiDBUnaryPostfixOperator; public class TiDBUnaryPostfixOperation extends UnaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java b/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java index 52a81eb0a..4382e3e25 100644 --- a/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java +++ b/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; +import sqlancer.common.ast.UnaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; public class TiDBUnaryPrefixOperation extends UnaryOperatorNode diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index 7ee7505fa..f72eafe16 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -5,7 +5,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.schema.TableIndex; +import sqlancer.common.schema.TableIndex; import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.ast.TiDBSelect; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index 5e05fe9e2..a8be97d28 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.TernaryLogicPartitioningOracleBase; import sqlancer.TestOracle; -import sqlancer.gen.ExpressionGenerator; +import sqlancer.common.gen.ExpressionGenerator; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index ed396b591..5e0c56841 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -2,6 +2,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.visitor.ToStringVisitor; import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.ast.TiDBAggregate; import sqlancer.tidb.ast.TiDBCase; @@ -15,7 +16,6 @@ import sqlancer.tidb.ast.TiDBSelect; import sqlancer.tidb.ast.TiDBTableReference; import sqlancer.tidb.ast.TiDBText; -import sqlancer.visitor.ToStringVisitor; public class TiDBToStringVisitor extends ToStringVisitor implements TiDBVisitor { From 0c736fa880652acafaf852d4dc954ae2e7183118 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Aug 2020 23:21:24 +0200 Subject: [PATCH 0101/1284] Introduce a sqlancer.common.oracle package --- src/sqlancer/OracleFactory.java | 2 ++ src/sqlancer/ProviderAdapter.java | 2 ++ src/sqlancer/citus/CitusOptions.java | 4 ++-- src/sqlancer/citus/CitusProvider.java | 4 ++-- src/sqlancer/clickhouse/ClickHouseOptions.java | 2 +- src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java | 2 +- .../clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java | 2 +- src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java | 4 ++-- src/sqlancer/cockroachdb/CockroachDBOptions.java | 4 ++-- .../cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java | 2 +- .../cockroachdb/ast/CockroachDBUnaryPostfixOperation.java | 2 +- src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java | 4 ++-- .../oracle/tlp/CockroachDBTLPAggregateOracle.java | 2 +- src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java | 4 ++-- .../cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java | 2 +- src/sqlancer/{ => common/oracle}/CompositeTestOracle.java | 4 +++- src/sqlancer/{ => common/oracle}/NoRECBase.java | 5 ++++- .../oracle}/TernaryLogicPartitioningOracleBase.java | 3 ++- src/sqlancer/{ => common/oracle}/TestOracle.java | 2 +- src/sqlancer/duckdb/DuckDBOptions.java | 4 ++-- src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java | 2 +- src/sqlancer/duckdb/test/DuckDBNoRECOracle.java | 4 ++-- .../duckdb/test/DuckDBQueryPartitioningAggregateTester.java | 2 +- src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java | 4 ++-- .../duckdb/test/DuckDBQueryPartitioningHavingTester.java | 2 +- src/sqlancer/mariadb/MariaDBOptions.java | 2 +- src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java | 4 ++-- src/sqlancer/mysql/MySQLOptions.java | 2 +- src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java | 2 +- .../mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 2 +- src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java | 4 ++-- src/sqlancer/postgres/PostgresOptions.java | 4 ++-- src/sqlancer/postgres/oracle/PostgresNoRECOracle.java | 4 ++-- .../postgres/oracle/PostgresPivotedQuerySynthesisOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPAggregateOracle.java | 2 +- src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 4 ++-- src/sqlancer/sqlite3/SQLite3Options.java | 4 ++-- src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java | 2 +- src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java | 4 ++-- .../sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java | 2 +- .../sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java | 2 +- src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java | 4 ++-- src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java | 2 +- src/sqlancer/tidb/TiDBOptions.java | 4 ++-- src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java | 2 +- src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java | 2 +- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 4 ++-- src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java | 2 +- 48 files changed, 75 insertions(+), 65 deletions(-) rename src/sqlancer/{ => common/oracle}/CompositeTestOracle.java (92%) rename src/sqlancer/{ => common/oracle}/NoRECBase.java (84%) rename src/sqlancer/{ => common/oracle}/TernaryLogicPartitioningOracleBase.java (95%) rename src/sqlancer/{ => common/oracle}/TestOracle.java (85%) diff --git a/src/sqlancer/OracleFactory.java b/src/sqlancer/OracleFactory.java index f8762a448..424e271be 100644 --- a/src/sqlancer/OracleFactory.java +++ b/src/sqlancer/OracleFactory.java @@ -2,6 +2,8 @@ import java.sql.SQLException; +import sqlancer.common.oracle.TestOracle; + public interface OracleFactory> { TestOracle create(G globalState) throws SQLException; diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 2cbf81693..4d3101ea9 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -6,6 +6,8 @@ import java.util.stream.Collectors; import sqlancer.StateToReproduce.OracleRunReproductionState; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; public abstract class ProviderAdapter, O extends DBMSSpecificOptions>> implements DatabaseProvider { diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index e5768537c..718e43b91 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -7,13 +7,13 @@ import com.beust.jcommander.Parameter; -import sqlancer.CompositeTestOracle; import sqlancer.OracleFactory; -import sqlancer.TestOracle; import sqlancer.citus.oracle.CitusNoRECOracle; import sqlancer.citus.oracle.tlp.CitusTLPAggregateOracle; import sqlancer.citus.oracle.tlp.CitusTLPHavingOracle; import sqlancer.citus.oracle.tlp.CitusTLPWhereOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresOptions; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 152c84bbd..2f4e7562f 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -12,7 +12,6 @@ import java.util.stream.Collectors; import sqlancer.AbstractAction; -import sqlancer.CompositeTestOracle; import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Query; @@ -21,7 +20,6 @@ import sqlancer.Randomly; import sqlancer.SQLancerResultSet; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; import sqlancer.citus.gen.CitusAlterTableGenerator; import sqlancer.citus.gen.CitusCommon; import sqlancer.citus.gen.CitusDeleteGenerator; @@ -30,6 +28,8 @@ import sqlancer.citus.gen.CitusSetGenerator; import sqlancer.citus.gen.CitusUpdateGenerator; import sqlancer.citus.gen.CitusViewGenerator; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresOptions; import sqlancer.postgres.PostgresProvider; diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index baeeb15d3..5ca84871e 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -9,7 +9,6 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseOptions.ClickHouseOracleFactory; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; @@ -17,6 +16,7 @@ import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPHavingOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; +import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "ClickHouse") public class ClickHouseOptions implements DBMSSpecificOptions { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index af9c1b746..ee195392f 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -10,8 +10,8 @@ import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.clickhouse.ClickHouseToStringVisitor; +import sqlancer.common.gen.AbstractInsertGenerator; public class ClickHouseInsertGenerator extends AbstractInsertGenerator { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 0fef32786..a7a5d6333 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -10,7 +10,6 @@ import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; @@ -21,6 +20,7 @@ import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; +import sqlancer.common.oracle.TestOracle; public class ClickHouseTLPAggregateOracle implements TestOracle { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 8df66fb9f..b46199b1a 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -6,8 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TernaryLogicPartitioningOracleBase; -import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; @@ -20,6 +18,8 @@ import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index ba125447f..7772117b8 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -8,10 +8,8 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; @@ -22,6 +20,8 @@ import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPJoinOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPWhereOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "Test CockroachDB") public class CockroachDBOptions implements DBMSSpecificOptions { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java index 777fc4205..edd6b3c70 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java @@ -2,8 +2,8 @@ import sqlancer.Randomly; import sqlancer.cockroachdb.ast.CockroachDBUnaryArithmeticOperation.CockroachDBUnaryAritmeticOperator; -import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; public class CockroachDBUnaryArithmeticOperation extends UnaryOperatorNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java index 140c0c9cc..fddc2c315 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java @@ -2,8 +2,8 @@ import sqlancer.Randomly; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; -import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; public class CockroachDBUnaryPostfixOperation extends UnaryOperatorNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index b97fb73f2..8ab46d830 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -9,12 +9,10 @@ import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.NoRECBase; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -29,6 +27,8 @@ import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; public class CockroachDBNoRECOracle extends NoRECBase implements TestOracle { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index 947eefd53..374ce2110 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -14,7 +14,6 @@ import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -34,6 +33,7 @@ import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; +import sqlancer.common.oracle.TestOracle; public class CockroachDBTLPAggregateOracle implements TestOracle { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index e81e744f6..e08389bf5 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -6,8 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TernaryLogicPartitioningOracleBase; -import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema; @@ -21,6 +19,8 @@ import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; public class CockroachDBTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java index 5f1200e7a..229dff2a7 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java @@ -9,7 +9,6 @@ import sqlancer.ComparatorHelper; import sqlancer.ExpectedErrors; import sqlancer.Randomly; -import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema; @@ -28,6 +27,7 @@ import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.common.oracle.TestOracle; public class CockroachDBTLPJoinOracle implements TestOracle { diff --git a/src/sqlancer/CompositeTestOracle.java b/src/sqlancer/common/oracle/CompositeTestOracle.java similarity index 92% rename from src/sqlancer/CompositeTestOracle.java rename to src/sqlancer/common/oracle/CompositeTestOracle.java index e83194823..4f89815ac 100644 --- a/src/sqlancer/CompositeTestOracle.java +++ b/src/sqlancer/common/oracle/CompositeTestOracle.java @@ -1,8 +1,10 @@ -package sqlancer; +package sqlancer.common.oracle; import java.sql.SQLException; import java.util.List; +import sqlancer.GlobalState; + public class CompositeTestOracle implements TestOracle { private final TestOracle[] oracles; diff --git a/src/sqlancer/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java similarity index 84% rename from src/sqlancer/NoRECBase.java rename to src/sqlancer/common/oracle/NoRECBase.java index 55be742fd..f60069df8 100644 --- a/src/sqlancer/NoRECBase.java +++ b/src/sqlancer/common/oracle/NoRECBase.java @@ -1,8 +1,11 @@ -package sqlancer; +package sqlancer.common.oracle; import java.sql.Connection; +import sqlancer.ExpectedErrors; +import sqlancer.GlobalState; import sqlancer.Main.StateLogger; +import sqlancer.MainOptions; public abstract class NoRECBase> implements TestOracle { diff --git a/src/sqlancer/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java similarity index 95% rename from src/sqlancer/TernaryLogicPartitioningOracleBase.java rename to src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java index 8b6f0e54b..9b92ed637 100644 --- a/src/sqlancer/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java @@ -1,5 +1,6 @@ -package sqlancer; +package sqlancer.common.oracle; +import sqlancer.ExpectedErrors; import sqlancer.common.gen.ExpressionGenerator; /** diff --git a/src/sqlancer/TestOracle.java b/src/sqlancer/common/oracle/TestOracle.java similarity index 85% rename from src/sqlancer/TestOracle.java rename to src/sqlancer/common/oracle/TestOracle.java index 40586a0f7..4640b9185 100644 --- a/src/sqlancer/TestOracle.java +++ b/src/sqlancer/common/oracle/TestOracle.java @@ -1,4 +1,4 @@ -package sqlancer; +package sqlancer.common.oracle; import java.sql.SQLException; diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index 6e2099122..c163b5d0b 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -8,10 +8,10 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.TestOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.duckdb.DuckDBOptions.DuckDBOracleFactory; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.test.DuckDBNoRECOracle; diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 11ce516b4..99446ad41 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -14,11 +14,11 @@ import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.NewInOperatorNode; import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.ast.newast.NewTernaryNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index b4935df34..47bf9052c 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -9,16 +9,16 @@ import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.NoRECBase; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewPostfixTextNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index 61983a069..0b29a95f5 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -10,13 +10,13 @@ import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; import sqlancer.common.ast.newast.NewAliasNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 8b64b95f0..a8663b314 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -6,12 +6,12 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TernaryLogicPartitioningOracleBase; -import sqlancer.TestOracle; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index 569bee295..e11dcefb3 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -7,8 +7,8 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.TestOracle; import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBToStringVisitor; diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 86d485c8e..6037bec86 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -9,7 +9,7 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.mariadb.MariaDBOptions.MariaDBOracleFactory; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.oracle.MariaDBNoRECOracle; diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index eff65611d..d0cfe5d96 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -6,10 +6,10 @@ import java.util.List; import sqlancer.IgnoreMeException; -import sqlancer.NoRECBase; import sqlancer.QueryAdapter; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 4b65d798e..4e0f1fabf 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -9,7 +9,7 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; diff --git a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java index 0121deca0..c87048a4b 100644 --- a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java +++ b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java @@ -2,8 +2,8 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; public class MySQLUnaryPrefixOperation extends UnaryOperatorNode diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 37b205dc5..e061de2f4 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -10,7 +10,7 @@ import sqlancer.Randomly; import sqlancer.StateToReproduce.MySQLStateToReproduce; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLColumn; diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index 0026b1f39..8946136fe 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -5,9 +5,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.TernaryLogicPartitioningOracleBase; -import sqlancer.TestOracle; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 5c318f1a8..1f74e8cc1 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -8,10 +8,10 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.TestOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; import sqlancer.postgres.oracle.PostgresNoRECOracle; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index a12f10811..d32c43903 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -9,12 +9,12 @@ import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.NoRECBase; import sqlancer.Query; import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index aff969f45..8e400c4f1 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -14,7 +14,7 @@ import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.StateToReproduce.PostgresStateToReproduce; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index a5edbd894..d45248cf4 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -13,7 +13,7 @@ import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresVisitor; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 2c5fea811..34f5e602a 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -7,9 +7,9 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TernaryLogicPartitioningOracleBase; -import sqlancer.TestOracle; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index e79a2d8af..3e5d36ebf 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -8,10 +8,10 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.TestOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.oracle.SQLite3Fuzzer; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index a47fa4aed..c42eb3911 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -4,7 +4,7 @@ import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 1b0e06602..eac8402a4 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -6,11 +6,11 @@ import java.util.List; import sqlancer.IgnoreMeException; -import sqlancer.NoRECBase; import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 1b6b38227..3206f9fcc 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -16,7 +16,7 @@ import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.StateToReproduce.SQLite3StateToReproduce; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index e2287ee2b..dcad3a02a 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -10,7 +10,7 @@ import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.SQLancerResultSet; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index 1c498035d..a4518b6b6 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -6,9 +6,9 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TernaryLogicPartitioningOracleBase; -import sqlancer.TestOracle; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Expression; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index 004fd9574..d796939d8 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -9,7 +9,7 @@ import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 2c5cc3c64..9ce651232 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -8,10 +8,10 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.CompositeTestOracle; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.TestOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; diff --git a/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java b/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java index b4d04bf5a..3d7b1221d 100644 --- a/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java +++ b/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.tidb.ast.TiDBUnaryPostfixOperation.TiDBUnaryPostfixOperator; public class TiDBUnaryPostfixOperation extends UnaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java b/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java index 4382e3e25..c32c076ba 100644 --- a/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java +++ b/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; public class TiDBUnaryPrefixOperation extends UnaryOperatorNode diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index a8be97d28..ba2ec9df0 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -6,9 +6,9 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TernaryLogicPartitioningOracleBase; -import sqlancer.TestOracle; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java index 5e5126420..a9f64671b 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java @@ -6,7 +6,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.ast.TiDBExpression; From b7e4cf0d73fa57ccaeb7fe3879acbf8c6eb795c2 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Aug 2020 23:26:48 +0200 Subject: [PATCH 0102/1284] Introduce a sqlancer.common.query package --- src/sqlancer/AbstractAction.java | 2 ++ src/sqlancer/ComparatorHelper.java | 4 ++++ src/sqlancer/GlobalState.java | 2 ++ src/sqlancer/Main.java | 2 ++ src/sqlancer/StateToReproduce.java | 2 ++ src/sqlancer/StatementExecutor.java | 2 ++ src/sqlancer/citus/CitusProvider.java | 10 +++++----- src/sqlancer/citus/gen/CitusAlterTableGenerator.java | 4 ++-- src/sqlancer/citus/gen/CitusCommon.java | 2 +- src/sqlancer/citus/gen/CitusDeleteGenerator.java | 4 ++-- src/sqlancer/citus/gen/CitusIndexGenerator.java | 4 ++-- src/sqlancer/citus/gen/CitusInsertGenerator.java | 4 ++-- src/sqlancer/citus/gen/CitusSetGenerator.java | 4 ++-- src/sqlancer/citus/gen/CitusUpdateGenerator.java | 4 ++-- src/sqlancer/citus/gen/CitusViewGenerator.java | 4 ++-- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 +- src/sqlancer/clickhouse/ClickHouseProvider.java | 4 ++-- .../clickhouse/gen/ClickHouseInsertGenerator.java | 6 +++--- .../clickhouse/gen/ClickHouseTableGenerator.java | 6 +++--- .../oracle/tlp/ClickHouseTLPAggregateOracle.java | 4 ++-- src/sqlancer/cockroachdb/CockroachDBErrors.java | 2 +- src/sqlancer/cockroachdb/CockroachDBProvider.java | 8 ++++---- .../cockroachdb/gen/CockroachDBCommentOnGenerator.java | 6 +++--- .../gen/CockroachDBCreateStatisticsGenerator.java | 6 +++--- .../cockroachdb/gen/CockroachDBDeleteGenerator.java | 6 +++--- .../cockroachdb/gen/CockroachDBIndexGenerator.java | 2 +- .../cockroachdb/gen/CockroachDBInsertGenerator.java | 6 +++--- .../gen/CockroachDBRandomQuerySynthesizer.java | 4 ++-- .../gen/CockroachDBSetClusterSettingGenerator.java | 6 +++--- .../gen/CockroachDBSetSessionGenerator.java | 6 +++--- .../cockroachdb/gen/CockroachDBShowGenerator.java | 6 +++--- .../cockroachdb/gen/CockroachDBTableGenerator.java | 2 +- .../cockroachdb/gen/CockroachDBTruncateGenerator.java | 6 +++--- .../cockroachdb/gen/CockroachDBUpdateGenerator.java | 6 +++--- .../cockroachdb/gen/CockroachDBViewGenerator.java | 6 +++--- .../cockroachdb/oracle/CockroachDBNoRECOracle.java | 8 ++++---- .../oracle/tlp/CockroachDBTLPAggregateOracle.java | 6 +++--- .../oracle/tlp/CockroachDBTLPJoinOracle.java | 2 +- src/sqlancer/common/gen/AbstractGenerator.java | 6 +++--- src/sqlancer/common/oracle/NoRECBase.java | 2 +- .../oracle/TernaryLogicPartitioningOracleBase.java | 2 +- src/sqlancer/{ => common/query}/ExpectedErrors.java | 2 +- src/sqlancer/{ => common/query}/Query.java | 4 +++- src/sqlancer/{ => common/query}/QueryAdapter.java | 5 ++++- src/sqlancer/{ => common/query}/QueryProvider.java | 2 +- .../{ => common/query}/QueryResultCheckAdapter.java | 4 +++- src/sqlancer/{ => common/query}/SQLancerResultSet.java | 2 +- src/sqlancer/duckdb/DuckDBErrors.java | 2 +- src/sqlancer/duckdb/DuckDBProvider.java | 8 ++++---- src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java | 6 +++--- src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java | 6 +++--- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 6 +++--- src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java | 6 +++--- src/sqlancer/duckdb/gen/DuckDBTableGenerator.java | 6 +++--- src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java | 6 +++--- src/sqlancer/duckdb/gen/DuckDBViewGenerator.java | 6 +++--- src/sqlancer/duckdb/test/DuckDBNoRECOracle.java | 6 +++--- .../test/DuckDBQueryPartitioningAggregateTester.java | 4 ++-- src/sqlancer/mariadb/MariaDBErrors.java | 2 +- src/sqlancer/mariadb/MariaDBProvider.java | 2 +- src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java | 6 +++--- src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java | 6 +++--- src/sqlancer/mariadb/gen/MariaDBSetGenerator.java | 4 ++-- .../mariadb/gen/MariaDBTableAdminCommandGenerator.java | 6 +++--- src/sqlancer/mariadb/gen/MariaDBTableGenerator.java | 6 +++--- src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java | 4 ++-- src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java | 6 +++--- src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java | 4 ++-- src/sqlancer/mysql/MySQLErrors.java | 2 +- src/sqlancer/mysql/MySQLProvider.java | 6 +++--- src/sqlancer/mysql/gen/MySQLAlterTable.java | 6 +++--- src/sqlancer/mysql/gen/MySQLDeleteGenerator.java | 6 +++--- src/sqlancer/mysql/gen/MySQLDropIndex.java | 6 +++--- src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 6 +++--- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 4 ++-- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 6 +++--- .../mysql/gen/MySQLTruncateTableGenerator.java | 6 +++--- src/sqlancer/mysql/gen/admin/MySQLFlush.java | 4 ++-- src/sqlancer/mysql/gen/admin/MySQLReset.java | 4 ++-- .../mysql/gen/datadef/MySQLIndexGenerator.java | 6 +++--- .../mysql/gen/tblmaintenance/MySQLAnalyzeTable.java | 4 ++-- .../mysql/gen/tblmaintenance/MySQLCheckTable.java | 4 ++-- .../mysql/gen/tblmaintenance/MySQLChecksum.java | 4 ++-- .../mysql/gen/tblmaintenance/MySQLOptimize.java | 4 ++-- src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java | 4 ++-- src/sqlancer/postgres/PostgresProvider.java | 8 ++++---- .../postgres/gen/PostgresAlterTableGenerator.java | 6 +++--- .../postgres/gen/PostgresAnalyzeGenerator.java | 6 +++--- .../postgres/gen/PostgresClusterGenerator.java | 6 +++--- .../postgres/gen/PostgresCommentGenerator.java | 4 ++-- src/sqlancer/postgres/gen/PostgresCommon.java | 2 +- src/sqlancer/postgres/gen/PostgresDeleteGenerator.java | 6 +++--- .../postgres/gen/PostgresDiscardGenerator.java | 6 +++--- .../postgres/gen/PostgresDropIndexGenerator.java | 6 +++--- src/sqlancer/postgres/gen/PostgresIndexGenerator.java | 6 +++--- src/sqlancer/postgres/gen/PostgresInsertGenerator.java | 6 +++--- src/sqlancer/postgres/gen/PostgresNotifyGenerator.java | 4 ++-- .../postgres/gen/PostgresQueryCatalogGenerator.java | 4 ++-- .../postgres/gen/PostgresReindexGenerator.java | 6 +++--- .../postgres/gen/PostgresSequenceGenerator.java | 6 +++--- src/sqlancer/postgres/gen/PostgresSetGenerator.java | 4 ++-- .../postgres/gen/PostgresStatisticsGenerator.java | 6 +++--- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 6 +++--- .../postgres/gen/PostgresTransactionGenerator.java | 6 +++--- .../postgres/gen/PostgresTruncateGenerator.java | 6 +++--- src/sqlancer/postgres/gen/PostgresUpdateGenerator.java | 6 +++--- src/sqlancer/postgres/gen/PostgresVacuumGenerator.java | 6 +++--- src/sqlancer/postgres/gen/PostgresViewGenerator.java | 6 +++--- src/sqlancer/postgres/oracle/PostgresNoRECOracle.java | 6 +++--- .../oracle/tlp/PostgresTLPAggregateOracle.java | 4 ++-- src/sqlancer/sqlite3/SQLite3Errors.java | 2 +- src/sqlancer/sqlite3/SQLite3Provider.java | 8 ++++---- src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java | 6 +++--- .../gen/SQLite3CreateVirtualRtreeTabelGenerator.java | 6 +++--- src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java | 4 ++-- src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java | 4 ++-- src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java | 6 +++--- .../sqlite3/gen/SQLite3TransactionGenerator.java | 6 +++--- src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java | 6 +++--- .../gen/SQLite3VirtualFTSTableCommandGenerator.java | 6 +++--- src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java | 6 +++--- .../sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java | 6 +++--- .../gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java | 6 +++--- .../sqlite3/gen/ddl/SQLite3DropIndexGenerator.java | 6 +++--- .../sqlite3/gen/ddl/SQLite3DropTableGenerator.java | 6 +++--- .../sqlite3/gen/ddl/SQLite3IndexGenerator.java | 6 +++--- .../sqlite3/gen/ddl/SQLite3TableGenerator.java | 6 +++--- src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java | 6 +++--- .../sqlite3/gen/dml/SQLite3DeleteGenerator.java | 6 +++--- .../sqlite3/gen/dml/SQLite3InsertGenerator.java | 6 +++--- .../sqlite3/gen/dml/SQLite3UpdateGenerator.java | 6 +++--- src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java | 2 +- src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java | 4 ++-- .../oracle/SQLite3PivotedQuerySynthesisOracle.java | 6 +++--- .../sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java | 6 +++--- .../sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java | 2 +- src/sqlancer/sqlite3/schema/SQLite3Schema.java | 6 +++--- src/sqlancer/tidb/TiDBErrors.java | 2 +- src/sqlancer/tidb/TiDBProvider.java | 8 ++++---- src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java | 6 +++--- src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java | 6 +++--- src/sqlancer/tidb/gen/TiDBDeleteGenerator.java | 6 +++--- src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 6 +++--- src/sqlancer/tidb/gen/TiDBInsertGenerator.java | 6 +++--- src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java | 4 ++-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 4 ++-- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 6 +++--- src/sqlancer/tidb/gen/TiDBUpdateGenerator.java | 6 +++--- src/sqlancer/tidb/gen/TiDBViewGenerator.java | 6 +++--- test/sqlancer/TestExpectedErrors.java | 2 ++ 150 files changed, 382 insertions(+), 359 deletions(-) rename src/sqlancer/{ => common/query}/ExpectedErrors.java (98%) rename src/sqlancer/{ => common/query}/Query.java (95%) rename src/sqlancer/{ => common/query}/QueryAdapter.java (97%) rename src/sqlancer/{ => common/query}/QueryProvider.java (82%) rename src/sqlancer/{ => common/query}/QueryResultCheckAdapter.java (92%) rename src/sqlancer/{ => common/query}/SQLancerResultSet.java (97%) diff --git a/src/sqlancer/AbstractAction.java b/src/sqlancer/AbstractAction.java index 9238d8208..2ed0a7ed2 100644 --- a/src/sqlancer/AbstractAction.java +++ b/src/sqlancer/AbstractAction.java @@ -2,6 +2,8 @@ import java.sql.SQLException; +import sqlancer.common.query.Query; + public interface AbstractAction { Query getQuery(G globalState) throws SQLException; diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 97f47941c..e60c0a46c 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -8,6 +8,10 @@ import java.util.Set; import java.util.stream.Collectors; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; + public final class ComparatorHelper { private ComparatorHelper() { diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index a90bbd7bc..ab28029f1 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -5,6 +5,8 @@ import sqlancer.Main.QueryManager; import sqlancer.Main.StateLogger; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLancerResultSet; /** * Represents a global state that is valid for a testing session on a given database. diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 443c4cb69..98fb9b519 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -29,6 +29,8 @@ import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.cockroachdb.CockroachDBProvider; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.duckdb.DuckDBProvider; import sqlancer.mariadb.MariaDBProvider; import sqlancer.mysql.MySQLProvider; diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 67e2e62b3..853015181 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -9,6 +9,8 @@ import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ast.ClickHouseConstant; import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.ast.MySQLConstant; import sqlancer.mysql.ast.MySQLExpression; diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index edf46446a..88fe3c164 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -4,6 +4,8 @@ import java.util.ArrayList; import java.util.List; +import sqlancer.common.query.Query; + public class StatementExecutor, A extends AbstractAction> { private final G globalState; diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 2f4e7562f..6925dedf3 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -12,13 +12,8 @@ import java.util.stream.Collectors; import sqlancer.AbstractAction; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.StatementExecutor; import sqlancer.citus.gen.CitusAlterTableGenerator; import sqlancer.citus.gen.CitusCommon; @@ -30,6 +25,11 @@ import sqlancer.citus.gen.CitusViewGenerator; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresOptions; import sqlancer.postgres.PostgresProvider; diff --git a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java index 6271ae5cd..1142b02fb 100644 --- a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java @@ -2,9 +2,9 @@ import java.util.List; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.gen.PostgresAlterTableGenerator; diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 7e1a3f14a..b79edc4f8 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -1,6 +1,6 @@ package sqlancer.citus.gen; -import sqlancer.ExpectedErrors; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.gen.PostgresCommon; diff --git a/src/sqlancer/citus/gen/CitusDeleteGenerator.java b/src/sqlancer/citus/gen/CitusDeleteGenerator.java index 558372d23..e4f195410 100644 --- a/src/sqlancer/citus/gen/CitusDeleteGenerator.java +++ b/src/sqlancer/citus/gen/CitusDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.citus.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresDeleteGenerator; diff --git a/src/sqlancer/citus/gen/CitusIndexGenerator.java b/src/sqlancer/citus/gen/CitusIndexGenerator.java index 583895136..234af987f 100644 --- a/src/sqlancer/citus/gen/CitusIndexGenerator.java +++ b/src/sqlancer/citus/gen/CitusIndexGenerator.java @@ -1,7 +1,7 @@ package sqlancer.citus.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresInsertGenerator; diff --git a/src/sqlancer/citus/gen/CitusInsertGenerator.java b/src/sqlancer/citus/gen/CitusInsertGenerator.java index 58c72d620..13e46057c 100644 --- a/src/sqlancer/citus/gen/CitusInsertGenerator.java +++ b/src/sqlancer/citus/gen/CitusInsertGenerator.java @@ -1,7 +1,7 @@ package sqlancer.citus.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresInsertGenerator; diff --git a/src/sqlancer/citus/gen/CitusSetGenerator.java b/src/sqlancer/citus/gen/CitusSetGenerator.java index 6ed900b09..5b85fa0a7 100644 --- a/src/sqlancer/citus/gen/CitusSetGenerator.java +++ b/src/sqlancer/citus/gen/CitusSetGenerator.java @@ -1,7 +1,7 @@ package sqlancer.citus.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresSetGenerator; diff --git a/src/sqlancer/citus/gen/CitusUpdateGenerator.java b/src/sqlancer/citus/gen/CitusUpdateGenerator.java index 32969d3a9..ea237b6e8 100644 --- a/src/sqlancer/citus/gen/CitusUpdateGenerator.java +++ b/src/sqlancer/citus/gen/CitusUpdateGenerator.java @@ -1,7 +1,7 @@ package sqlancer.citus.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresUpdateGenerator; diff --git a/src/sqlancer/citus/gen/CitusViewGenerator.java b/src/sqlancer/citus/gen/CitusViewGenerator.java index fd0e9cafa..c946772c1 100644 --- a/src/sqlancer/citus/gen/CitusViewGenerator.java +++ b/src/sqlancer/citus/gen/CitusViewGenerator.java @@ -1,7 +1,7 @@ package sqlancer.citus.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.gen.PostgresViewGenerator; diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index c55a7e169..d6657dcf0 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -1,6 +1,6 @@ package sqlancer.clickhouse; -import sqlancer.ExpectedErrors; +import sqlancer.common.query.ExpectedErrors; public final class ClickHouseErrors { diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 2aab50e72..84360093b 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -10,14 +10,14 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StatementExecutor; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseInsertGenerator; import sqlancer.clickhouse.gen.ClickHouseTableGenerator; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryProvider; public class ClickHouseProvider extends ProviderAdapter { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index ee195392f..5cdc1260f 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -4,14 +4,14 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public class ClickHouseInsertGenerator extends AbstractInsertGenerator { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index de42ced6e..c0c5075b6 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -4,15 +4,15 @@ import java.util.List; import ru.yandex.clickhouse.domain.ClickHouseDataType; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public class ClickHouseTableGenerator { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index a7a5d6333..21ebd3553 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -7,9 +7,7 @@ import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; @@ -21,6 +19,8 @@ import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; public class ClickHouseTLPAggregateOracle implements TestOracle { diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index a7993c04f..14c74d039 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb; -import sqlancer.ExpectedErrors; +import sqlancer.common.query.ExpectedErrors; public final class CockroachDBErrors { diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 559d50be0..587c77d23 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -8,15 +8,11 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; import sqlancer.MainOptions; import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; @@ -33,6 +29,10 @@ import sqlancer.cockroachdb.gen.CockroachDBTruncateGenerator; import sqlancer.cockroachdb.gen.CockroachDBUpdateGenerator; import sqlancer.cockroachdb.gen.CockroachDBViewGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.QueryProvider; public class CockroachDBProvider extends ProviderAdapter { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java index 95df4298a..953414eed 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java @@ -2,15 +2,15 @@ import java.util.List; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.common.schema.TableIndex; public final class CockroachDBCommentOnGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java index bc9b2cc2f..cb670f420 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java @@ -1,11 +1,11 @@ package sqlancer.cockroachdb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBCreateStatisticsGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index 28e3328e5..69c918c72 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -1,14 +1,14 @@ package sqlancer.cockroachdb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBDeleteGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 4aae1ce78..106bf82f6 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -2,11 +2,11 @@ import java.util.List; -import sqlancer.Query; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.common.query.Query; // https://www.cockroachlabs.com/docs/stable/create-index.html public class CockroachDBIndexGenerator extends CockroachDBGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java index 50d9b558f..2ecf179df 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java @@ -3,15 +3,15 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBInsertGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java index 572b9abf3..945777148 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java @@ -4,8 +4,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -16,6 +14,8 @@ import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBRandomQuerySynthesizer { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index fc8f8bca3..2712a533d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -2,12 +2,12 @@ import java.util.function.Function; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBSetClusterSettingGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index 49bc0907e..c5d508725 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -2,12 +2,12 @@ import java.util.function.Function; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBSetSessionGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java index 5666a0603..ddd944711 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java @@ -1,11 +1,11 @@ package sqlancer.cockroachdb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBShowGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index f654fd7ee..ab3b7afc8 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -5,7 +5,6 @@ import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; @@ -15,6 +14,7 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.common.query.Query; public class CockroachDBTableGenerator extends CockroachDBGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index d1c13b360..6433f7225 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -1,10 +1,10 @@ package sqlancer.cockroachdb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBTruncateGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index a964d16f0..86c6777dd 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -2,9 +2,6 @@ import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -12,6 +9,9 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBUpdateGenerator { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java index 3740707f3..ef3c3dad9 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java @@ -1,11 +1,11 @@ package sqlancer.cockroachdb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class CockroachDBViewGenerator { diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 8ab46d830..966a37ab3 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -6,13 +6,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -29,6 +25,10 @@ import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; public class CockroachDBNoRECOracle extends NoRECBase implements TestOracle { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index 374ce2110..41f9c4ea7 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -9,11 +9,8 @@ import org.postgresql.util.PSQLException; import sqlancer.ComparatorHelper; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -34,6 +31,9 @@ import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; public class CockroachDBTLPAggregateOracle implements TestOracle { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java index 229dff2a7..988f82fba 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java @@ -7,7 +7,6 @@ import java.util.stream.Collectors; import sqlancer.ComparatorHelper; -import sqlancer.ExpectedErrors; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -28,6 +27,7 @@ import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; public class CockroachDBTLPJoinOracle implements TestOracle { diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java index 17c412a7e..c9acef68b 100644 --- a/src/sqlancer/common/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -1,8 +1,8 @@ package sqlancer.common.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public abstract class AbstractGenerator { diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java index f60069df8..ad15eeed7 100644 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ b/src/sqlancer/common/oracle/NoRECBase.java @@ -2,10 +2,10 @@ import java.sql.Connection; -import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.Main.StateLogger; import sqlancer.MainOptions; +import sqlancer.common.query.ExpectedErrors; public abstract class NoRECBase> implements TestOracle { diff --git a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java index 9b92ed637..b55853a0f 100644 --- a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java @@ -1,7 +1,7 @@ package sqlancer.common.oracle; -import sqlancer.ExpectedErrors; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.query.ExpectedErrors; /** * This is the base class of the Ternary Logic Partitioning (TLP) oracles. The core idea of TLP is to partition a given diff --git a/src/sqlancer/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java similarity index 98% rename from src/sqlancer/ExpectedErrors.java rename to src/sqlancer/common/query/ExpectedErrors.java index 9551956e8..20a7d97c8 100644 --- a/src/sqlancer/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -1,4 +1,4 @@ -package sqlancer; +package sqlancer.common.query; import java.util.Collection; import java.util.HashSet; diff --git a/src/sqlancer/Query.java b/src/sqlancer/common/query/Query.java similarity index 95% rename from src/sqlancer/Query.java rename to src/sqlancer/common/query/Query.java index ba535c223..eb358f80b 100644 --- a/src/sqlancer/Query.java +++ b/src/sqlancer/common/query/Query.java @@ -1,7 +1,9 @@ -package sqlancer; +package sqlancer.common.query; import java.sql.SQLException; +import sqlancer.GlobalState; + public abstract class Query { public abstract String getQueryString(); diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/common/query/QueryAdapter.java similarity index 97% rename from src/sqlancer/QueryAdapter.java rename to src/sqlancer/common/query/QueryAdapter.java index 9fd1b35e2..e7e7be4ee 100644 --- a/src/sqlancer/QueryAdapter.java +++ b/src/sqlancer/common/query/QueryAdapter.java @@ -1,10 +1,13 @@ -package sqlancer; +package sqlancer.common.query; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import sqlancer.GlobalState; +import sqlancer.Main; + public class QueryAdapter extends Query { private final String query; diff --git a/src/sqlancer/QueryProvider.java b/src/sqlancer/common/query/QueryProvider.java similarity index 82% rename from src/sqlancer/QueryProvider.java rename to src/sqlancer/common/query/QueryProvider.java index 99ee67d8c..628aaf323 100644 --- a/src/sqlancer/QueryProvider.java +++ b/src/sqlancer/common/query/QueryProvider.java @@ -1,4 +1,4 @@ -package sqlancer; +package sqlancer.common.query; import java.sql.SQLException; diff --git a/src/sqlancer/QueryResultCheckAdapter.java b/src/sqlancer/common/query/QueryResultCheckAdapter.java similarity index 92% rename from src/sqlancer/QueryResultCheckAdapter.java rename to src/sqlancer/common/query/QueryResultCheckAdapter.java index ba0dacfa8..a1626fda6 100644 --- a/src/sqlancer/QueryResultCheckAdapter.java +++ b/src/sqlancer/common/query/QueryResultCheckAdapter.java @@ -1,10 +1,12 @@ -package sqlancer; +package sqlancer.common.query; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.function.Consumer; +import sqlancer.GlobalState; + public class QueryResultCheckAdapter extends QueryAdapter { private final Consumer rsChecker; diff --git a/src/sqlancer/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java similarity index 97% rename from src/sqlancer/SQLancerResultSet.java rename to src/sqlancer/common/query/SQLancerResultSet.java index 896b4f7ce..f83cdd3a5 100644 --- a/src/sqlancer/SQLancerResultSet.java +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -1,4 +1,4 @@ -package sqlancer; +package sqlancer.common.query; import java.io.Closeable; import java.sql.ResultSet; diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index d6b8685d8..751c30ce0 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -1,6 +1,6 @@ package sqlancer.duckdb; -import sqlancer.ExpectedErrors; +import sqlancer.common.query.ExpectedErrors; public final class DuckDBErrors { diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index c93145c6c..5a8517375 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -5,15 +5,15 @@ import java.sql.SQLException; import sqlancer.AbstractAction; -import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.QueryProvider; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.gen.DuckDBDeleteGenerator; import sqlancer.duckdb.gen.DuckDBIndexGenerator; diff --git a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java index c6f612bce..fb59ff178 100644 --- a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java @@ -1,9 +1,9 @@ package sqlancer.duckdb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index 8ff29312d..ffe9af5a2 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -1,9 +1,9 @@ package sqlancer.duckdb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 66aec013d..51bba3e64 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -2,11 +2,11 @@ import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index a5046eaf6..f8c287522 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -3,11 +3,11 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index 540749f86..2ed1c22c0 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -4,12 +4,12 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 2c86a0a93..7e3fb6282 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -2,11 +2,11 @@ import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; diff --git a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java index f019e444d..1cf5ba023 100644 --- a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java @@ -1,9 +1,9 @@ package sqlancer.duckdb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBToStringVisitor; diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 47bf9052c..494eb2afb 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -9,16 +9,16 @@ import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewPostfixTextNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index 0b29a95f5..d8ed7a928 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -7,9 +7,7 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.common.ast.newast.NewAliasNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.NewFunctionNode; @@ -17,6 +15,8 @@ import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; diff --git a/src/sqlancer/mariadb/MariaDBErrors.java b/src/sqlancer/mariadb/MariaDBErrors.java index e3751f05e..ab1bec4cd 100644 --- a/src/sqlancer/mariadb/MariaDBErrors.java +++ b/src/sqlancer/mariadb/MariaDBErrors.java @@ -1,6 +1,6 @@ package sqlancer.mariadb; -import sqlancer.ExpectedErrors; +import sqlancer.common.query.ExpectedErrors; public final class MariaDBErrors { diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 733e30ccd..6f515dc72 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -11,8 +11,8 @@ import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.ProviderAdapter; -import sqlancer.Query; import sqlancer.Randomly; +import sqlancer.common.query.Query; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.gen.MariaDBIndexGenerator; import sqlancer.mariadb.gen.MariaDBInsertGenerator; diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 3728eb5ab..1dc100c37 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -2,10 +2,10 @@ import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; diff --git a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java index 8345eea0d..903089d3a 100644 --- a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java @@ -1,9 +1,9 @@ package sqlancer.mariadb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index 270862d76..e1a127bd6 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -6,9 +6,9 @@ import java.util.stream.Collectors; import sqlancer.MainOptions; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public class MariaDBSetGenerator { diff --git a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java index 62b2d3640..934fb1c7a 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java @@ -5,10 +5,10 @@ import java.util.function.Function; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryResultCheckAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.QueryResultCheckAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; diff --git a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java index e53df8862..e4cecfa28 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java @@ -4,10 +4,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mariadb.MariaDBBugs; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; diff --git a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java index 2b4da154c..a29178a85 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java @@ -1,7 +1,7 @@ package sqlancer.mariadb.gen; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mariadb.MariaDBSchema; public final class MariaDBTruncateGenerator { diff --git a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java index e000b7d80..171878b18 100644 --- a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java @@ -1,9 +1,9 @@ package sqlancer.mariadb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index d0cfe5d96..3fd8d997a 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -6,10 +6,10 @@ import java.util.List; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; -import sqlancer.SQLancerResultSet; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 9fbaa3bc5..39aa33e1e 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -1,6 +1,6 @@ package sqlancer.mysql; -import sqlancer.ExpectedErrors; +import sqlancer.common.query.ExpectedErrors; public final class MySQLErrors { diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 899856652..e92468b43 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -13,13 +13,13 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StateToReproduce; import sqlancer.StateToReproduce.MySQLStateToReproduce; import sqlancer.StatementExecutor; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.QueryProvider; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; diff --git a/src/sqlancer/mysql/gen/MySQLAlterTable.java b/src/sqlancer/mysql/gen/MySQLAlterTable.java index 4b362f523..ae322c4ba 100644 --- a/src/sqlancer/mysql/gen/MySQLAlterTable.java +++ b/src/sqlancer/mysql/gen/MySQLAlterTable.java @@ -5,10 +5,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index 3234d102e..c2506aa08 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -2,10 +2,10 @@ import java.util.Arrays; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; diff --git a/src/sqlancer/mysql/gen/MySQLDropIndex.java b/src/sqlancer/mysql/gen/MySQLDropIndex.java index 14919a550..800508b4e 100644 --- a/src/sqlancer/mysql/gen/MySQLDropIndex.java +++ b/src/sqlancer/mysql/gen/MySQLDropIndex.java @@ -1,10 +1,10 @@ package sqlancer.mysql.gen; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 4dd478b3a..d7597e7e2 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -4,10 +4,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index d00cf27f8..cadd1bb52 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -5,9 +5,9 @@ import java.util.stream.Collectors; import sqlancer.MainOptions; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; public class MySQLSetGenerator { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index f5f01bdce..8a6ee5482 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -6,11 +6,11 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLDataType; diff --git a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java index b00c97d75..b57354779 100644 --- a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java @@ -1,8 +1,8 @@ package sqlancer.mysql.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; public final class MySQLTruncateTableGenerator { diff --git a/src/sqlancer/mysql/gen/admin/MySQLFlush.java b/src/sqlancer/mysql/gen/admin/MySQLFlush.java index ee3699528..e5d048de5 100644 --- a/src/sqlancer/mysql/gen/admin/MySQLFlush.java +++ b/src/sqlancer/mysql/gen/admin/MySQLFlush.java @@ -3,9 +3,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; diff --git a/src/sqlancer/mysql/gen/admin/MySQLReset.java b/src/sqlancer/mysql/gen/admin/MySQLReset.java index a210b018e..70abef70c 100644 --- a/src/sqlancer/mysql/gen/admin/MySQLReset.java +++ b/src/sqlancer/mysql/gen/admin/MySQLReset.java @@ -2,9 +2,9 @@ import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; public final class MySQLReset { diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index cce9d089c..1fa0ba898 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -2,10 +2,10 @@ import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java index 6f34395fb..0cdaf120f 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java @@ -3,9 +3,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java index 6b1e5df1e..82a28cfbb 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java @@ -3,9 +3,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java index 55030a5bc..1c2034aa9 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java @@ -3,9 +3,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java index 73355ef7b..febdf6968 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java @@ -3,9 +3,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java index 5f8171961..bc67477c3 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java @@ -3,9 +3,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 9b1c84ca6..112b9257e 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -15,14 +15,14 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.StateToReproduce; import sqlancer.StateToReproduce.PostgresStateToReproduce; import sqlancer.StatementExecutor; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.ast.PostgresExpression; diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 3bcc618df..590973505 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -2,11 +2,11 @@ import java.util.List; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; diff --git a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java index 0c3de03db..5baa8695e 100644 --- a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java @@ -2,10 +2,10 @@ import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; diff --git a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java index 6bbb8e0b1..4adf47906 100644 --- a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java @@ -1,9 +1,9 @@ package sqlancer.postgres.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; diff --git a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java index df668c372..15a7fbfa0 100644 --- a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java @@ -1,9 +1,9 @@ package sqlancer.postgres.gen; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index fd1aaa99c..e92b9e0cd 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -7,9 +7,9 @@ import java.util.function.Function; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; import sqlancer.postgres.PostgresSchema.PostgresColumn; diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 3ea96aefc..131972345 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -1,9 +1,9 @@ package sqlancer.postgres.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; diff --git a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java index 349f92ade..75187220d 100644 --- a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java @@ -1,9 +1,9 @@ package sqlancer.postgres.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; diff --git a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java index f454da256..5a9233b60 100644 --- a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java @@ -2,10 +2,10 @@ import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresIndex; import sqlancer.sqlite3.gen.SQLite3Common; diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 8aee16532..11018cab1 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -3,10 +3,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index a1c2329b9..fd9328484 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -3,10 +3,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresTable; diff --git a/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java b/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java index a75b4e26a..cbb4f8f30 100644 --- a/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java @@ -1,8 +1,8 @@ package sqlancer.postgres.gen; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresNotifyGenerator { diff --git a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java index 9f4f79cc3..d68e4319f 100644 --- a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java @@ -5,9 +5,9 @@ import java.sql.Statement; import sqlancer.GlobalState; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class PostgresQueryCatalogGenerator { diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index b443cdbe3..c55b324ef 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -3,11 +3,11 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresIndex; diff --git a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java index a5c384c0d..61c401c14 100644 --- a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java @@ -1,9 +1,9 @@ package sqlancer.postgres.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresSequenceGenerator { diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index d403f0e0a..815c05971 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -4,9 +4,9 @@ import java.util.Arrays; import java.util.function.Function; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresSetGenerator { diff --git a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java index d0a838a0a..0ebb764ed 100644 --- a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java @@ -3,11 +3,11 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresStatisticsObject; diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index a5edacc95..9f474ceb9 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -4,10 +4,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; diff --git a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java index 01dc28d6c..aa582ea73 100644 --- a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java @@ -1,9 +1,9 @@ package sqlancer.postgres.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; public final class PostgresTransactionGenerator { diff --git a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java index 647a59ac4..374900232 100644 --- a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java @@ -2,10 +2,10 @@ import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresTruncateGenerator { diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 3386a757e..e184c144b 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -2,10 +2,10 @@ import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; diff --git a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java index a113472da..3f07e92b6 100644 --- a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java @@ -4,10 +4,10 @@ import java.util.Arrays; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index ccc78f573..a1b100bc2 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -1,9 +1,9 @@ package sqlancer.postgres.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresSelect; diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index d32c43903..58996bee6 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -9,12 +9,12 @@ import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index d45248cf4..72a3a4cd3 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -10,10 +10,10 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresVisitor; diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 708f01df6..0fa4058e3 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -2,7 +2,7 @@ import java.util.Arrays; -import sqlancer.ExpectedErrors; +import sqlancer.common.query.ExpectedErrors; public final class SQLite3Errors { diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 451f55095..696795424 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -12,17 +12,17 @@ import java.util.List; import sqlancer.AbstractAction; -import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StateToReproduce; import sqlancer.StateToReproduce.SQLite3StateToReproduce; import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.QueryProvider; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; import sqlancer.sqlite3.gen.SQLite3Common; diff --git a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java index 9a37f7135..94e3340d8 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java @@ -1,9 +1,9 @@ package sqlancer.sqlite3.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public final class SQLite3AnalyzeGenerator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index f1e46bb67..1234789d7 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -3,10 +3,10 @@ import java.util.ArrayList; import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java index a54a83069..f67c8cd03 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java @@ -2,9 +2,9 @@ import java.sql.SQLException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.sqlite3.SQLite3Provider.Action; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; diff --git a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java index 3ca68b2f6..f36f72379 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java @@ -3,9 +3,9 @@ import java.sql.SQLException; import java.util.function.Supplier; -import sqlancer.ExpectedErrors; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public class SQLite3PragmaGenerator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java index 05f611bf0..fbba42623 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java @@ -1,9 +1,9 @@ package sqlancer.sqlite3.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; diff --git a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java index 01f381d91..d2e6412a6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java @@ -1,9 +1,9 @@ package sqlancer.sqlite3.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public final class SQLite3TransactionGenerator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java index d3d595a6b..c3a16e781 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java @@ -1,9 +1,9 @@ package sqlancer.sqlite3.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; /** diff --git a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java index 20e2e653a..56616410c 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java @@ -1,9 +1,9 @@ package sqlancer.sqlite3.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java index 30d20be23..7cf0fb00d 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java @@ -2,10 +2,10 @@ import java.sql.SQLException; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; import sqlancer.sqlite3.gen.SQLite3Common; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java index 6e4044a2d..bea71997b 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java @@ -4,11 +4,11 @@ import java.util.ArrayList; import java.util.List; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java index 0d87ded39..5956aa63a 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java @@ -4,10 +4,10 @@ import java.util.Arrays; import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.gen.SQLite3Common; public class SQLite3CreateVirtualFTSTableGenerator { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java index d35ba6d14..cc96d1d49 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java @@ -1,9 +1,9 @@ package sqlancer.sqlite3.gen.ddl; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; // see https://www.sqlite.org/lang_dropindex.html diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java index 099d1e1f8..4cac79a87 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java @@ -1,10 +1,10 @@ package sqlancer.sqlite3.gen.ddl; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public final class SQLite3DropTableGenerator { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index f24da36b8..b6d2ca8dd 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -3,10 +3,10 @@ import java.sql.SQLException; import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index 6809fbb4a..f121c25e0 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -5,10 +5,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index feb9f5c12..50f69c365 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -2,11 +2,11 @@ import java.sql.SQLException; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index a700f33b0..0453535a6 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -2,10 +2,10 @@ import java.util.Arrays; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java index 071323e32..2e0d425e8 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java @@ -4,10 +4,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 71269555e..bde23fb1d 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -3,10 +3,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index c42eb3911..fad867944 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -2,9 +2,9 @@ import java.sql.SQLException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index eac8402a4..1edc97aad 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -6,11 +6,11 @@ import java.util.List; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 3206f9fcc..b2f2d7ef6 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -10,13 +10,13 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.StateToReproduce.SQLite3StateToReproduce; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index dcad3a02a..3b53cfbbe 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -5,12 +5,12 @@ import java.util.List; import sqlancer.ComparatorHelper; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index d796939d8..d82666e48 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -6,10 +6,10 @@ import java.util.stream.Collectors; import sqlancer.ComparatorHelper; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 2fb08d6cd..6c777efa8 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -14,12 +14,12 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.SQLancerResultSet; import sqlancer.StateToReproduce.SQLite3StateToReproduce; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.TableIndex; diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 26a69d581..fa09f11a9 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -1,6 +1,6 @@ package sqlancer.tidb; -import sqlancer.ExpectedErrors; +import sqlancer.common.query.ExpectedErrors; public final class TiDBErrors { diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index bd147ddb1..0d59d2bea 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -6,15 +6,15 @@ import java.sql.Statement; import sqlancer.AbstractAction; -import sqlancer.ExpectedErrors; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.QueryProvider; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index eb4b39749..515cd5955 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -2,11 +2,11 @@ import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 5a3844ff3..126a064d3 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -2,10 +2,10 @@ import java.sql.SQLException; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index 578ddaf59..c2b40bc2c 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -3,10 +3,10 @@ import java.sql.SQLException; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index 5c2135a66..db7e796a5 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -3,10 +3,10 @@ import java.sql.SQLException; import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; import sqlancer.tidb.TiDBSchema.TiDBTable; diff --git a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java index 31af8bbb7..253fb47a2 100644 --- a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java @@ -4,10 +4,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; diff --git a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java index a2252c998..730c78172 100644 --- a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java +++ b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java @@ -4,9 +4,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTables; diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 2cc2953e7..5bfffe13b 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -3,9 +3,9 @@ import java.sql.SQLException; import java.util.function.Function; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; public final class TiDBSetGenerator { diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index f8526a8ed..ae10f1b50 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -5,11 +5,11 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.ExpectedErrors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 3cee65bd0..9e9b7cf39 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -3,10 +3,10 @@ import java.sql.SQLException; import java.util.List; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index 497b546c6..084a5cfbf 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -1,9 +1,9 @@ package sqlancer.tidb.gen; -import sqlancer.ExpectedErrors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; diff --git a/test/sqlancer/TestExpectedErrors.java b/test/sqlancer/TestExpectedErrors.java index c963c108b..69a0f53e6 100644 --- a/test/sqlancer/TestExpectedErrors.java +++ b/test/sqlancer/TestExpectedErrors.java @@ -5,6 +5,8 @@ import org.junit.jupiter.api.Test; +import sqlancer.common.query.ExpectedErrors; + public class TestExpectedErrors { @Test From 2befbee07db7c3293c5369360e7a7ed6b7c931a9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 13 Aug 2020 23:10:55 +0200 Subject: [PATCH 0103/1284] Add PMD documentation rules --- configs/pmd-rules.xml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/configs/pmd-rules.xml b/configs/pmd-rules.xml index b9dbe1ce4..e5329b062 100644 --- a/configs/pmd-rules.xml +++ b/configs/pmd-rules.xml @@ -84,5 +84,11 @@ - + + 2 + + + + + From 8e2b623973add9d27099fa6738890f35b92e9df5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 13 Aug 2020 23:15:02 +0200 Subject: [PATCH 0104/1284] Address the PMD UncommentedEmptyConstructor rule --- configs/pmd-rules.xml | 1 - src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/configs/pmd-rules.xml b/configs/pmd-rules.xml index e5329b062..ed0a1f67a 100644 --- a/configs/pmd-rules.xml +++ b/configs/pmd-rules.xml @@ -89,6 +89,5 @@ - diff --git a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java index f36f72379..fffeb63ae 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java @@ -46,16 +46,6 @@ private enum Pragma { WAL_CHECKPOINT; // // WRITEABLE_SCHEMA - // VDBE_ADDOPTRACE(PragmaAttribute.DEBUG); // produces too much textual output directly on the console - // VDBE_LISTING(PragmaAttribute.DEBUG); // produces too much textual output directly on the console - - Pragma(PragmaAttribute... attrs) { - } - - private enum PragmaAttribute { - DEBUG /* only available in debug mode */ - } - } private final StringBuilder sb = new StringBuilder(); From 3b3ae42bacfad11a415196937c856e2e5a88eae5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 13 Aug 2020 23:21:38 +0200 Subject: [PATCH 0105/1284] Enforce the checkstyle JavadocType rule --- configs/checkstyle.xml | 2 +- src/sqlancer/GlobalState.java | 4 +++- .../common/oracle/TernaryLogicPartitioningOracleBase.java | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/configs/checkstyle.xml b/configs/checkstyle.xml index 7cbd93534..a5442df4b 100644 --- a/configs/checkstyle.xml +++ b/configs/checkstyle.xml @@ -81,7 +81,7 @@ - diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index ab28029f1..89c36f269 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -12,7 +12,9 @@ * Represents a global state that is valid for a testing session on a given database. * * @param - * the option parameter. + * the option parameter + * @param + * the schema parameter */ public abstract class GlobalState, S> { diff --git a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java index b55853a0f..6b5025806 100644 --- a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java @@ -1,5 +1,6 @@ package sqlancer.common.oracle; +import sqlancer.GlobalState; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.query.ExpectedErrors; @@ -10,8 +11,10 @@ * * @param * the expression type + * @param + * the global state type */ -public abstract class TernaryLogicPartitioningOracleBase implements TestOracle { +public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { protected E predicate; protected E negatedPredicate; From 51e16c892c24783d7479ec3477c379b3e38225f0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 13 Aug 2020 23:29:31 +0200 Subject: [PATCH 0106/1284] Enforce the JavadocStyle checkstyle rule --- configs/checkstyle.xml | 8 ++++++-- src/sqlancer/mysql/gen/admin/MySQLFlush.java | 2 +- src/sqlancer/sqlite3/ast/SQLite3Expression.java | 5 ++--- src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java | 4 +--- src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java | 2 +- src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java | 2 +- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/configs/checkstyle.xml b/configs/checkstyle.xml index a5442df4b..2d0dda3c1 100644 --- a/configs/checkstyle.xml +++ b/configs/checkstyle.xml @@ -81,8 +81,12 @@ - + + + + diff --git a/src/sqlancer/mysql/gen/admin/MySQLFlush.java b/src/sqlancer/mysql/gen/admin/MySQLFlush.java index e5d048de5..d826fa6f2 100644 --- a/src/sqlancer/mysql/gen/admin/MySQLFlush.java +++ b/src/sqlancer/mysql/gen/admin/MySQLFlush.java @@ -9,7 +9,7 @@ import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; -/** +/* * https://dev.mysql.com/doc/refman/8.0/en/flush.html#flush-tables-variants */ public class MySQLFlush { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 6172c8dac..df0dd25d4 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -92,17 +92,16 @@ public boolean isNumeric() { } } - /** + /* * See https://www.sqlite.org/datatype3.html 3.2 */ public TypeAffinity getAffinity() { return TypeAffinity.NONE; } - /** + /* * See https://www.sqlite.org/datatype3.html#assigning_collating_sequences_from_sql 7.1 * - * @return */ public abstract SQLite3CollateSequence getExplicitCollateSequence(); diff --git a/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java b/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java index 9247ef5cc..2c4c16e12 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java +++ b/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java @@ -34,9 +34,7 @@ public SQLite3CollateSequence getImplicitCollateSequence() { } /** - * Supported unary prefix operators are these: - * - * - + ~ NOT + * Supported unary prefix operators are these: -, +, ~, and NOT. * * @see https://www.sqlite.org/lang_expr.html * diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index e548679fd..f9e8dc029 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -327,7 +327,7 @@ private enum RowValueComparison { STANDARD_COMPARISON, BETWEEN, IN } - /** + /* * https://www.sqlite.org/rowvalue.html */ private SQLite3Expression getRowValueComparison(int depth) { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index b6d2ca8dd..3cbf3e61f 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -107,7 +107,7 @@ private String createIndex(SQLite3Table t, List columns) { } /** - * Appends ASC, DESC, or nothing + * Appends ASC, DESC, or nothing. */ private void appendPotentialOrdering(StringBuilder sb) { if (Randomly.getBoolean()) { From 967359401f63b610c42c6b9c2074101fe7e278b6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 13 Aug 2020 23:44:44 +0200 Subject: [PATCH 0107/1284] Enforce the JavadocMethod checkstyle rule --- configs/checkstyle.xml | 2 +- src/sqlancer/DatabaseProvider.java | 6 ++++++ src/sqlancer/StateToReproduce.java | 2 +- src/sqlancer/clickhouse/ast/ClickHouseCast.java | 2 +- src/sqlancer/common/query/Query.java | 10 +--------- src/sqlancer/mysql/ast/MySQLComputableFunction.java | 2 ++ src/sqlancer/mysql/gen/MySQLSetGenerator.java | 2 +- src/sqlancer/postgres/ast/PostgresFunction.java | 2 ++ src/sqlancer/sqlite3/ast/SQLite3Cast.java | 2 +- src/sqlancer/sqlite3/ast/SQLite3Function.java | 2 ++ .../sqlite3/gen/SQLite3ExpressionGenerator.java | 7 ++----- .../sqlite3/gen/ddl/SQLite3IndexGenerator.java | 2 +- src/sqlancer/sqlite3/schema/SQLite3Schema.java | 2 ++ 13 files changed, 23 insertions(+), 20 deletions(-) diff --git a/configs/checkstyle.xml b/configs/checkstyle.xml index 2d0dda3c1..530bd41d9 100644 --- a/configs/checkstyle.xml +++ b/configs/checkstyle.xml @@ -80,7 +80,7 @@ - + diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 9e8642741..d90c24ba6 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -8,11 +8,15 @@ public interface DatabaseProvider, O extends DBMSSpe /** * Gets the the {@link GlobalState} class. + * + * @return the class extending {@link GlobalState} */ Class getGlobalStateClass(); /** * Gets the JCommander option class. + * + * @return the class representing the DBMS-specific options. */ Class getOptionClass(); @@ -29,6 +33,8 @@ public interface DatabaseProvider, O extends DBMSSpe /** * The DBMS name is used to name the log directory and command to test the respective DBMS. + * + * @return the DBMS' name */ String getDBMSName(); diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 853015181..30ebaff2d 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -95,7 +95,7 @@ public long getSeedValue() { * Returns a local state in which a test oracle can save useful information about a single run. If the local state * is closed without indicating access to it, the local statements will be added to the global state. * - * @return + * @return the local state for logging */ public OracleRunReproductionState getLocalState() { return localState; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCast.java b/src/sqlancer/clickhouse/ast/ClickHouseCast.java index d0d1bc0b9..24e179c4f 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCast.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCast.java @@ -109,7 +109,7 @@ public static ClickHouseConstant castToNumericFromNumOperand(ClickHouseConstant return convertInternal(value, false, false, false); } - /** + /* * Applies numeric affinity to a value. */ public static ClickHouseConstant castToNumeric(ClickHouseConstant value) { diff --git a/src/sqlancer/common/query/Query.java b/src/sqlancer/common/query/Query.java index eb358f80b..1dad78f94 100644 --- a/src/sqlancer/common/query/Query.java +++ b/src/sqlancer/common/query/Query.java @@ -11,18 +11,10 @@ public abstract class Query { /** * Whether the query could affect the schema (i.e., by add/deleting columns or tables). * - * @return + * @return true if the query can affect the database's schema, false otherwise */ public abstract boolean couldAffectSchema(); - /** - * - * @param con - * - * @return true if the query was successful, false otherwise - * - * @throws SQLException - */ public abstract boolean execute(GlobalState globalState, String... fills) throws SQLException; public abstract ExpectedErrors getExpectedErrors(); diff --git a/src/sqlancer/mysql/ast/MySQLComputableFunction.java b/src/sqlancer/mysql/ast/MySQLComputableFunction.java index 8cfb5f70c..2adcae95b 100644 --- a/src/sqlancer/mysql/ast/MySQLComputableFunction.java +++ b/src/sqlancer/mysql/ast/MySQLComputableFunction.java @@ -177,6 +177,8 @@ private static MySQLConstant aggregate(MySQLConstant[] evaluatedArgs, BinaryOper /** * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number * of arguments is returned. + * + * @return the number of arguments */ public int getNrArgs() { return nrArgs; diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index cadd1bb52..1d3b5623a 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -128,7 +128,7 @@ private enum Action { this.scopes = scopes.clone(); } - /** + /* * @see https://dev.mysql.com/doc/refman/8.0/en/switchable-optimizations.html */ private static String getOptimizerSwitchConfiguration(Randomly r) { diff --git a/src/sqlancer/postgres/ast/PostgresFunction.java b/src/sqlancer/postgres/ast/PostgresFunction.java index a5bbc25b6..00c4c4810 100644 --- a/src/sqlancer/postgres/ast/PostgresFunction.java +++ b/src/sqlancer/postgres/ast/PostgresFunction.java @@ -248,6 +248,8 @@ public PostgresDataType[] getType(int nr, PostgresDataType type) { /** * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number * of arguments is returned. + * + * @return the number of arguments */ public int getNrArgs() { return nrArgs; diff --git a/src/sqlancer/sqlite3/ast/SQLite3Cast.java b/src/sqlancer/sqlite3/ast/SQLite3Cast.java index 5fb92d65c..c3ac74407 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Cast.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Cast.java @@ -114,7 +114,7 @@ public static SQLite3Constant castToNumericFromNumOperand(SQLite3Constant value) return convertInternal(value, false, false, false); } - /** + /* * Applies numeric affinity to a value. */ public static SQLite3Constant castToNumeric(SQLite3Constant value) { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Function.java b/src/sqlancer/sqlite3/ast/SQLite3Function.java index 79f0e157b..07bd2ae75 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Function.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Function.java @@ -275,6 +275,8 @@ public SQLite3Constant apply(SQLite3Constant... args) { /** * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number * of arguments is returned. + * + * @return the number of arguments */ public int getNrArgs() { return nrArgs; diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index f9e8dc029..b22caeba6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -157,11 +157,8 @@ public boolean omitBracketsWhenPrinting() { return expr; } - /** - * - * @see https://www.sqlite.org/syntax/literal-value.html - * - * @return + /* + * https://www.sqlite.org/syntax/literal-value.html */ private SQLite3Expression getRandomLiteralValueInternal(Randomly r) { LiteralValueType randomLiteral = Randomly.fromOptions(LiteralValueType.values()); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 3cbf3e61f..0eb01a678 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -106,7 +106,7 @@ private String createIndex(SQLite3Table t, List columns) { return sb.toString(); } - /** + /* * Appends ASC, DESC, or nothing. */ private void appendPotentialOrdering(StringBuilder sb) { diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 6c777efa8..886bd6609 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -98,6 +98,8 @@ public boolean isOnlyPrimaryKey() { /** * If a table has a single column primary key and the declared type of that column is "INTEGER" and the table is * not a WITHOUT ROWID table, then the column is known as an INTEGER PRIMARY KEY. + * + * @return whether the column is an INTEGER PRIMARY KEY */ public boolean isIntegerPrimaryKey() { return isInteger && isOnlyPrimaryKey() && !getTable().hasWithoutRowid(); From ac8f0930e9807169dbf174cd8ae92340fef4aff8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 13 Aug 2020 23:46:41 +0200 Subject: [PATCH 0108/1284] Remove the PMD LoosePackageCoupling rule This rule currently results in a warning "[WARNING] Removed misconfigured rule: LoosePackageCoupling cause: No packages or classes specified" being printed. --- configs/pmd-rules.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/pmd-rules.xml b/configs/pmd-rules.xml index ed0a1f67a..f89de4028 100644 --- a/configs/pmd-rules.xml +++ b/configs/pmd-rules.xml @@ -51,6 +51,7 @@ 2 + From 669569ebc2360300a4669b79defe58806823795a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 14 Aug 2020 11:46:25 +0200 Subject: [PATCH 0109/1284] Fix the random number generation Previously, the random number data generation consistently generated the same data sequence for every thread's database. This seems to have been introduced in commit 183b93e43b829f12bb47b95853379286b1491029. --- src/sqlancer/Main.java | 24 +++++++++++------------- src/sqlancer/Randomly.java | 10 ++++++++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 98fb9b519..65cc9e1a3 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -296,17 +296,17 @@ public static class DBMSExecutor, O extends DBMSSpec private final MainOptions options; private final O command; private final String databaseName; - private final long seed; private StateLogger logger; private StateToReproduce stateToRepro; + private final Randomly r; public DBMSExecutor(DatabaseProvider provider, MainOptions options, O dbmsSpecificOptions, - String databaseName, long seed) { + String databaseName, Randomly r) { this.provider = provider; this.options = options; this.databaseName = databaseName; - this.seed = seed; this.command = dbmsSpecificOptions; + this.r = r; } private G createGlobalState() { @@ -324,10 +324,9 @@ public O getCommand() { public void run() throws SQLException { G state = createGlobalState(); stateToRepro = provider.getStateToReproduce(databaseName); - stateToRepro.seedValue = seed; + stateToRepro.seedValue = r.getSeed(); state.setState(stateToRepro); logger = new StateLogger(databaseName, provider, options); - Randomly r = new Randomly(seed); state.setRandomly(r); state.setDatabaseName(databaseName); state.setMainOptions(options); @@ -390,10 +389,10 @@ public O getCommand() { } @SuppressWarnings("unchecked") - public DBMSExecutor getDBMSExecutor(String databaseName, long seed) { + public DBMSExecutor getDBMSExecutor(String databaseName, Randomly r) { try { return new DBMSExecutor(provider.getClass().getDeclaredConstructor().newInstance(), options, - command, databaseName, seed); + command, databaseName, r); } catch (Exception e) { throw new AssertionError(e); } @@ -461,7 +460,6 @@ private String formatInteger(long intValue) { } else { seed = options.getRandomSeed() + i; } - execService.execute(new Runnable() { @Override @@ -471,17 +469,17 @@ public void run() { } private void runThread(final String databaseName) { + Randomly r = new Randomly(seed); try { if (options.getMaxGeneratedDatabases() == -1) { // run without a limit boolean continueRunning = true; while (continueRunning) { - continueRunning = run(options, execService, executorFactory, seed, databaseName); + continueRunning = run(options, execService, executorFactory, r, databaseName); } } else { for (int i = 0; i < options.getMaxGeneratedDatabases(); i++) { - boolean continueRunning = run(options, execService, executorFactory, seed, - databaseName); + boolean continueRunning = run(options, execService, executorFactory, r, databaseName); if (!continueRunning) { break; } @@ -496,8 +494,8 @@ private void runThread(final String databaseName) { } private boolean run(MainOptions options, ExecutorService execService, - DBMSExecutorFactory executorFactory, final long seed, final String databaseName) { - DBMSExecutor executor = executorFactory.getDBMSExecutor(databaseName, seed); + DBMSExecutorFactory executorFactory, Randomly r, final String databaseName) { + DBMSExecutor executor = executorFactory.getDBMSExecutor(databaseName, r); try { executor.run(); return true; diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 75ae7b600..ada420a3b 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -20,6 +20,7 @@ public final class Randomly { private Supplier provider; private static final ThreadLocal THREAD_RANDOM = new ThreadLocal<>(); + private long seed; private void addToCache(long val) { if (USE_CACHING && cachedLongs.size() < CACHE_SIZE && !cachedLongs.contains(val)) { @@ -383,11 +384,12 @@ public Randomly(Supplier provider) { } public Randomly() { - getThreadRandom().set(new Random()); + THREAD_RANDOM.set(new Random()); } public Randomly(long seed) { - getThreadRandom().set(new Random(seed)); + this.seed = seed; + THREAD_RANDOM.set(new Random(seed)); } public static double getUncachedDouble() { @@ -429,4 +431,8 @@ private static int getNextInt(int lower, int upper) { return (int) getNextLong(lower, upper); } + public long getSeed() { + return seed; + } + } From d27c86ab68c2e8441c612d1b7c5cc73528c5b6db Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 14 Aug 2020 12:28:32 +0200 Subject: [PATCH 0110/1284] Address the newly-appearing expected errors --- .../cockroachdb/gen/CockroachDBIndexGenerator.java | 1 + src/sqlancer/mariadb/gen/MariaDBSetGenerator.java | 4 +++- src/sqlancer/postgres/gen/PostgresSetGenerator.java | 2 -- src/sqlancer/sqlite3/SQLite3Provider.java | 10 +++++++--- src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java | 1 + src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java | 2 +- .../sqlite3/gen/ddl/SQLite3IndexGenerator.java | 1 + src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java | 1 + src/sqlancer/sqlite3/schema/SQLite3Schema.java | 2 +- src/sqlancer/tidb/gen/TiDBViewGenerator.java | 9 ++++----- 10 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 106bf82f6..45831c31b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -28,6 +28,7 @@ public void buildStatement() { errors.add("schema change statement cannot follow a statement that has written in the same transaction"); errors.add("https://github.com/cockroachdb/cockroach/issues/35730"); // some array types are not indexable errors.add("cannot determine type of empty array. Consider annotating with the desired type"); + errors.add("incompatible IF expression"); // TODO: investigate; seems to be a bug CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append("CREATE "); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index e1a127bd6..f950eb474 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -7,6 +7,7 @@ import sqlancer.MainOptions; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; @@ -190,7 +191,8 @@ private Query get() { sb.append(a.name); sb.append(" = "); sb.append(a.prod.apply(r)); - return new QueryAdapter(sb.toString()); + return new QueryAdapter(sb.toString(), ExpectedErrors + .from("At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'")); } } diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 815c05971..56bc61bc5 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -66,8 +66,6 @@ private enum ConfigurationOption { // https://www.postgresql.org/docs/devel/runtime-config-query.html ENABLE_BITMAPSCAN("enable_bitmapscan", (r) -> Randomly.fromOptions(1, 0)), ENABLE_GATHERMERGE("enable_gathermerge", (r) -> Randomly.fromOptions(1, 0)), - ENABLE_HASHAGG("enable_hashagg", (r) -> Randomly.fromOptions(1, 0)), - ENABLE_HASHAGG_DISK("enable_hashagg_disk", (r) -> Randomly.fromOptions(1, 0)), ENABLE_HASHJOIN("enable_hashjoin", (r) -> Randomly.fromOptions(1, 0)), ENABLE_INDEXSCAN("enable_indexscan", (r) -> Randomly.fromOptions(1, 0)), ENABLE_INDEXONLYSCAN("enable_indexonlyscan", (r) -> Randomly.fromOptions(1, 0)), diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 696795424..29d319ed8 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -98,7 +98,7 @@ public boolean canBeRetried() { CHECK_RTREE_TABLE((g) -> { SQLite3Table table = g.getSchema().getRandomTableOrBailout(t -> t.getName().startsWith("r")); String format = String.format("SELECT rtreecheck('%s');", table.getName()); - return new QueryAdapter(format); + return new QueryAdapter(format, ExpectedErrors.from("The database file is locked")); }), // VIRTUAL_TABLE_ACTION(SQLite3VirtualFTSTableCommandGenerator::create), // CREATE_VIEW(SQLite3ViewGenerator::generate), // @@ -120,6 +120,8 @@ public boolean canBeRetried() { } indexName = rs.getString("name"); } + } catch (SQLException e) { + throw new IgnoreMeException(); } sb.append(" VALUES"); sb.append("('"); @@ -154,7 +156,8 @@ public boolean canBeRetried() { sb.append(" noskipscan"); } sb.append("')"); - return new QueryAdapter(sb.toString(), ExpectedErrors.from("no such table")); + return new QueryAdapter(sb.toString(), + ExpectedErrors.from("no such table", "The database file is locked")); } }); @@ -284,7 +287,8 @@ private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) Query q = new QueryAdapter("SELECT * FROM " + table.getName(), ExpectedErrors.from("needs an odd number of arguments", " requires an even number of arguments", "generated column loop", "integer overflow", "malformed JSON", - "JSON cannot hold BLOB values", "JSON path error", "labels must be TEXT")); + "JSON cannot hold BLOB values", "JSON path error", "labels must be TEXT", + "table does not support scanning")); if (!q.execute(globalState)) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java index fbba42623..9f4e319dd 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java @@ -23,6 +23,7 @@ public static Query executeReindex(SQLite3GlobalState globalState) { SQLite3Schema s = globalState.getSchema(); StringBuilder sb = new StringBuilder("REINDEX"); ExpectedErrors errors = new ExpectedErrors(); + errors.add("The database file is locked"); Target t = Randomly.fromOptions(Target.values()); if (Randomly.getBoolean()) { sb.append(" "); diff --git a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java index c3a16e781..32b462abd 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java @@ -21,7 +21,7 @@ public static Query executeVacuum(SQLite3GlobalState globalState) { sb.append(Randomly.fromOptions("temp", "main")); } return new QueryAdapter(sb.toString(), ExpectedErrors.from("cannot VACUUM from within a transaction", - "cannot VACUUM - SQL statements in progress")); + "cannot VACUUM - SQL statements in progress", "The database file is locked")); } } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 0eb01a678..f6e487e30 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -44,6 +44,7 @@ private Query create() throws SQLException { errors.add("non-deterministic use of julianday() in an index"); errors.add("non-deterministic use of date() in an index"); errors.add("non-deterministic use of datetime() in an index"); + errors.add("The database file is locked"); SQLite3Errors.addExpectedExpressionErrors(errors); if (!SQLite3Provider.mustKnowResult) { // can only happen when PRAGMA case_sensitive_like=ON; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 50f69c365..f4cf79136 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -44,6 +44,7 @@ public static Query generate(SQLite3GlobalState globalState) throws SQLException ExpectedErrors errors = new ExpectedErrors(); errors.add("is circularly defined"); errors.add("unsupported frame specification"); + errors.add("The database file is locked"); if (Randomly.getBoolean()) { SQLite3PivotedQuerySynthesisOracle queryGen = new SQLite3PivotedQuerySynthesisOracle(globalState); try { diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 886bd6609..f0c10a276 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -355,7 +355,7 @@ public static int getNrRows(SQLite3GlobalState globalState, String table) throws errors.addAll(Arrays.asList("second argument to nth_value must be a positive integer", "ON clause references tables to its right", "no such table", "no query solution", "no such index", "GROUP BY term", "is circularly defined", "misuse of aggregate", "no such column", - "misuse of window function")); + "misuse of window function", "table does not support scanning")); SQLite3Errors.addExpectedExpressionErrors(errors); QueryAdapter q = new QueryAdapter(string, errors); try (SQLancerResultSet query = q.executeAndGet(globalState)) { diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index 084a5cfbf..a77585758 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -1,5 +1,6 @@ package sqlancer.tidb.gen; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; @@ -40,11 +41,9 @@ public static Query getQuery(TiDBGlobalState globalState) { errors.add( "references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them"); errors.add("Unknown column "); - if (Randomly.getBoolean()) { - sb.append(" WITH "); - sb.append(Randomly.fromOptions("CASCADED", "LOCAL")); - sb.append(" "); - sb.append(" CHECK OPTION"); + if (sb.toString().contains("\\\\")) { + // TODO: CREATE VIEW v0(c0) AS SELECT '\\' FROM t0; causes an unexpected failure + throw new IgnoreMeException(); } return new QueryAdapter(sb.toString(), errors, true); } From f67e63450b53e1b733e610ef1e0004611999ce3a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 15 Aug 2020 15:29:19 +0200 Subject: [PATCH 0111/1284] Fix a race-condition issue --- src/sqlancer/Main.java | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 65cc9e1a3..a7ee59600 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -101,25 +101,27 @@ public StateLogger(String databaseName, DatabaseProvider provider, MainOpt } } - private synchronized void ensureExistsAndIsEmpty(File dir, DatabaseProvider provider) { + private void ensureExistsAndIsEmpty(File dir, DatabaseProvider provider) { if (INITIALIZED_PROVIDER_NAMES.contains(provider.getDBMSName())) { return; } - if (!dir.exists()) { - try { - Files.createDirectories(dir.toPath()); - } catch (IOException e) { - throw new AssertionError(e); + synchronized (INITIALIZED_PROVIDER_NAMES) { + if (!dir.exists()) { + try { + Files.createDirectories(dir.toPath()); + } catch (IOException e) { + throw new AssertionError(e); + } } - } - File[] listFiles = dir.listFiles(); - assert listFiles != null : "directory was just created, so it should exist"; - for (File file : listFiles) { - if (!file.isDirectory()) { - file.delete(); + File[] listFiles = dir.listFiles(); + assert listFiles != null : "directory was just created, so it should exist"; + for (File file : listFiles) { + if (!file.isDirectory()) { + file.delete(); + } } + INITIALIZED_PROVIDER_NAMES.add(provider.getDBMSName()); } - INITIALIZED_PROVIDER_NAMES.add(provider.getDBMSName()); } private FileWriter getLogFileWriter() { From a5da014d0444555b7d021d7c0b6e101e7910d1f0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2020 08:35:06 +0000 Subject: [PATCH 0112/1284] Bump plexus-compiler-eclipse from 2.8.6 to 2.8.7 Bumps plexus-compiler-eclipse from 2.8.6 to 2.8.7. Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 660ed886a..6e013920d 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.8.6 + 2.8.7 org.eclipse.jdt From f1817d477609a6366d323bc7db863cad551fa44f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Aug 2020 13:13:33 +0200 Subject: [PATCH 0113/1284] [H2] Add an initial H2 implementation. Currently, only a single table without any constraints is generated. BOOL and INT data types are considered. As expressions, constants, column references, simple comparisons, logical operators, and simple unary operations are supported. This is phase 1 of the systematic study on H2. --- pom.xml | 5 + src/sqlancer/Main.java | 2 + src/sqlancer/h2/H2Constant.java | 61 ++++++++ src/sqlancer/h2/H2Errors.java | 18 +++ src/sqlancer/h2/H2Expression.java | 5 + src/sqlancer/h2/H2ExpressionGenerator.java | 169 +++++++++++++++++++++ src/sqlancer/h2/H2InsertGenerator.java | 47 ++++++ src/sqlancer/h2/H2Options.java | 34 +++++ src/sqlancer/h2/H2Provider.java | 126 +++++++++++++++ src/sqlancer/h2/H2Schema.java | 139 +++++++++++++++++ src/sqlancer/h2/H2TableGenerator.java | 26 ++++ src/sqlancer/h2/H2ToStringVisitor.java | 27 ++++ 12 files changed, 659 insertions(+) create mode 100644 src/sqlancer/h2/H2Constant.java create mode 100644 src/sqlancer/h2/H2Errors.java create mode 100644 src/sqlancer/h2/H2Expression.java create mode 100644 src/sqlancer/h2/H2ExpressionGenerator.java create mode 100644 src/sqlancer/h2/H2InsertGenerator.java create mode 100644 src/sqlancer/h2/H2Options.java create mode 100644 src/sqlancer/h2/H2Provider.java create mode 100644 src/sqlancer/h2/H2Schema.java create mode 100644 src/sqlancer/h2/H2TableGenerator.java create mode 100644 src/sqlancer/h2/H2ToStringVisitor.java diff --git a/pom.xml b/pom.xml index 6e013920d..9d479e76d 100644 --- a/pom.xml +++ b/pom.xml @@ -229,6 +229,11 @@ clickhouse-jdbc 0.2.4 + + com.h2database + h2 + 1.4.200 + diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a7ee59600..940fd5b77 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -32,6 +32,7 @@ import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.duckdb.DuckDBProvider; +import sqlancer.h2.H2Provider; import sqlancer.mariadb.MariaDBProvider; import sqlancer.mysql.MySQLProvider; import sqlancer.postgres.PostgresProvider; @@ -548,6 +549,7 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new CitusProvider()); providers.add(new ClickHouseProvider()); providers.add(new DuckDBProvider()); + providers.add(new H2Provider()); return providers; } diff --git a/src/sqlancer/h2/H2Constant.java b/src/sqlancer/h2/H2Constant.java new file mode 100644 index 000000000..3a372eff5 --- /dev/null +++ b/src/sqlancer/h2/H2Constant.java @@ -0,0 +1,61 @@ +package sqlancer.h2; + +import sqlancer.common.ast.newast.Node; + +public class H2Constant implements Node { + + private H2Constant() { + } + + public static class H2NullConstant extends H2Constant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class H2IntConstant extends H2Constant { + + private final long value; + + public H2IntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static class H2BoolConstant extends H2Constant { + + private final boolean value; + + public H2BoolConstant(boolean value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static Node createIntConstant(long val) { + return new H2IntConstant(val); + } + + public static Node createNullConstant() { + return new H2NullConstant(); + } + + public static Node createBoolConstant(boolean val) { + return new H2BoolConstant(val); + } + +} diff --git a/src/sqlancer/h2/H2Errors.java b/src/sqlancer/h2/H2Errors.java new file mode 100644 index 000000000..85ca0f861 --- /dev/null +++ b/src/sqlancer/h2/H2Errors.java @@ -0,0 +1,18 @@ +package sqlancer.h2; + +import sqlancer.common.query.ExpectedErrors; + +public final class H2Errors { + + private H2Errors() { + } + + public static void addInsertErrors(ExpectedErrors errors) { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.add("Numeric value out of range"); + errors.add("are not comparable"); + } + +} diff --git a/src/sqlancer/h2/H2Expression.java b/src/sqlancer/h2/H2Expression.java new file mode 100644 index 000000000..61b8c9658 --- /dev/null +++ b/src/sqlancer/h2/H2Expression.java @@ -0,0 +1,5 @@ +package sqlancer.h2; + +public interface H2Expression { + +} diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java new file mode 100644 index 000000000..d67ad67f7 --- /dev/null +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -0,0 +1,169 @@ +package sqlancer.h2; + +import java.util.Random; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2DataType; + +public class H2ExpressionGenerator extends UntypedExpressionGenerator, H2Column> { + + private static final Random RANDOM = new Random(); + + private final H2GlobalState globalState; + + public H2ExpressionGenerator(H2GlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX; + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + Expression expr = Randomly.fromOptions(Expression.values()); + switch (expr) { + case BINARY_COMPARISON: + Operator op = H2BinaryComparisonOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), + op); + case BINARY_LOGICAL: + op = H2BinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), + op); + case UNARY_POSTFIX: + op = H2UnaryPostfixOperator.getRandom(); + return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), op); + case UNARY_PREFIX: + return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), + H2UnaryPrefixOperator.getRandom()); + default: + throw new AssertionError(); + } + } + + @Override + protected Node generateColumn() { + return new ColumnReferenceNode(Randomly.fromList(columns)); + } + + @Override + public Node generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return H2Constant.createNullConstant(); + } + switch (H2DataType.getRandom()) { + case INT: + return H2Constant.createIntConstant(getUncachedInt()); + case BOOL: + return H2Constant.createBoolConstant(Randomly.getBoolean()); + default: + throw new AssertionError(); + } + } + + public static int getUncachedInt() { + return RANDOM.nextInt(); + } + + public enum H2UnaryPostfixOperator implements Operator { + + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + H2UnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static H2UnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum H2UnaryPrefixOperator implements Operator { + + NOT("NOT"), PLUS("+"), MINUS("-"); + + private String textRepr; + + H2UnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static H2UnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum H2BinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum H2BinaryComparisonOperator implements Operator { + + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); + + private String textRepr; + + H2BinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, H2UnaryPrefixOperator.NOT); + } + + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, H2UnaryPostfixOperator.IS_NULL); + } + +} diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java new file mode 100644 index 000000000..9105528b6 --- /dev/null +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -0,0 +1,47 @@ +package sqlancer.h2; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; + +public class H2InsertGenerator extends AbstractInsertGenerator { + + private final H2GlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + private final H2ExpressionGenerator gen; + + public H2InsertGenerator(H2GlobalState globalState) { + this.globalState = globalState; + gen = new H2ExpressionGenerator(globalState); + } + + public static Query getQuery(H2GlobalState globalState) { + return new H2InsertGenerator(globalState).generate(); + } + + private Query generate() { + sb.append("INSERT INTO "); + H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + H2Errors.addInsertErrors(errors); + return new QueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertValue(H2Column tiDBColumn) { + sb.append(H2ToStringVisitor.asString(gen.generateConstant())); + } +} diff --git a/src/sqlancer/h2/H2Options.java b/src/sqlancer/h2/H2Options.java new file mode 100644 index 000000000..631c22266 --- /dev/null +++ b/src/sqlancer/h2/H2Options.java @@ -0,0 +1,34 @@ +package sqlancer.h2; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.h2.H2Options.H2OracleFactory; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Provider.H2TLPWhereOracle; + +public class H2Options implements DBMSSpecificOptions { + + public enum H2OracleFactory implements OracleFactory { + + TLP_WHERE { + + @Override + public TestOracle create(H2GlobalState globalState) throws SQLException { + return new H2TLPWhereOracle(globalState); + } + + }; + + } + + @Override + public List getTestOracleFactory() { + return Arrays.asList(H2OracleFactory.TLP_WHERE); + } + +} diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java new file mode 100644 index 000000000..b4a333e5e --- /dev/null +++ b/src/sqlancer/h2/H2Provider.java @@ -0,0 +1,126 @@ +package sqlancer.h2; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.AbstractAction; +import sqlancer.ComparatorHelper; +import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; +import sqlancer.ProviderAdapter; +import sqlancer.Randomly; +import sqlancer.StatementExecutor; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryProvider; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; + +public class H2Provider extends ProviderAdapter { + + public H2Provider() { + super(H2GlobalState.class, H2Options.class); + } + + public enum Action implements AbstractAction { + + INSERT(H2InsertGenerator::getQuery); + + private final QueryProvider queryProvider; + + Action(QueryProvider queryProvider) { + this.queryProvider = queryProvider; + } + + @Override + public Query getQuery(H2GlobalState state) throws SQLException { + return queryProvider.getQuery(state); + } + } + + private static int mapActions(H2GlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + default: + throw new AssertionError(a); + } + } + + public static class H2GlobalState extends GlobalState { + + @Override + protected void updateSchema() throws SQLException { + setSchema(H2Schema.fromConnection(getConnection(), getDatabaseName())); + } + + } + + @Override + public void generateDatabase(H2GlobalState globalState) throws SQLException { + boolean success = false; + do { // create exactly one table + Query qt = new H2TableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + H2Provider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public Connection createDatabase(H2GlobalState globalState) throws SQLException { + String connectionString = "jdbc:h2:~/" + globalState.getDatabaseName() + ";DB_CLOSE_ON_EXIT=FALSE"; + Connection connection = DriverManager.getConnection(connectionString, "sa", ""); + connection.createStatement().execute("DROP ALL OBJECTS DELETE FILES"); + connection.close(); + connection = DriverManager.getConnection(connectionString, "sa", ""); + return connection; + } + + @Override + public String getDBMSName() { + return "h2"; + } + + public static class H2TLPWhereOracle implements TestOracle { + + private final H2GlobalState globalState; + + public H2TLPWhereOracle(H2GlobalState globalState) { + this.globalState = globalState; + } + + @Override + public void check() throws SQLException { + List columns = globalState.getSchema().getRandomTableNonEmptyTables().getColumns(); + String predicate = H2ToStringVisitor + .asString(new H2ExpressionGenerator(globalState).setColumns(columns).generateExpression()); + String original = "SELECT * FROM t0;"; + ExpectedErrors errors = new ExpectedErrors(); + H2Errors.addExpressionErrors(errors); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(original, errors, globalState); + + String nonNegated = "SELECT * FROM t0 WHERE " + predicate; + String negated = "SELECT * FROM t0 WHERE NOT " + predicate; + String isNull = "SELECT * FROM t0 WHERE " + predicate + " IS NULL"; + List combinedString = new ArrayList<>(); + + List secondResultSet = ComparatorHelper.getCombinedResultSet(nonNegated, negated, isNull, + combinedString, true, globalState, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, original, combinedString, + globalState); + } + + } + +} diff --git a/src/sqlancer/h2/H2Schema.java b/src/sqlancer/h2/H2Schema.java new file mode 100644 index 000000000..b78ac2783 --- /dev/null +++ b/src/sqlancer/h2/H2Schema.java @@ -0,0 +1,139 @@ +package sqlancer.h2; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.h2.H2Schema.H2Table; + +public class H2Schema extends AbstractSchema { + + public enum H2DataType { + + INT, BOOL; + + public static H2DataType getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public static class H2CompositeDataType { + + private final H2DataType dataType; + + public H2CompositeDataType(H2DataType dataType) { + this.dataType = dataType; + } + + public H2DataType getPrimitiveDataType() { + return dataType; + } + + public static H2CompositeDataType getRandom() { + return new H2CompositeDataType(H2DataType.getRandom()); + } + + @Override + public String toString() { + return dataType.toString(); + } + + } + + public static class H2Column extends AbstractTableColumn { + + public H2Column(String name, H2CompositeDataType columnType) { + super(name, null, columnType); + } + + } + + public static class H2Tables extends AbstractTables { + + public H2Tables(List tables) { + super(tables); + } + + } + + public H2Schema(List databaseTables) { + super(databaseTables); + } + + public H2Tables getRandomTableNonEmptyTables() { + return new H2Tables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public static class H2Table extends AbstractTable { + + public H2Table(String tableName, List columns) { + super(tableName, columns, Collections.emptyList(), false); + } + + } + + public static H2Schema fromConnection(Connection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + H2Table t = new H2Table(tableName, databaseColumns); + for (H2Column c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new H2Schema(databaseTables); + } + + private static List getTableNames(Connection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES")) { + while (rs.next()) { + tableNames.add(rs.getString("TABLE_NAME")); + } + } + } + return tableNames; + } + + private static List getTableColumns(Connection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("SHOW COLUMNS FROM %s;", tableName))) { + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + String columnType = rs.getString("TYPE"); + H2DataType primitiveType = getColumnType(columnType); + H2Column c = new H2Column(columnName, new H2CompositeDataType(primitiveType)); + columns.add(c); + } + } + } + return columns; + } + + private static H2DataType getColumnType(String columnType) { + if (columnType.startsWith("INTEGER")) { + return H2DataType.INT; + } else if (columnType.startsWith("BOOLEAN")) { + return H2DataType.BOOL; + } else { + throw new AssertionError(columnType); + } + } + +} diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java new file mode 100644 index 000000000..f77b37f30 --- /dev/null +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -0,0 +1,26 @@ +package sqlancer.h2; + +import sqlancer.Randomly; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2CompositeDataType; + +public class H2TableGenerator { + + public Query getQuery(H2GlobalState globalState) { + StringBuilder sb = new StringBuilder("CREATE TABLE t0("); + for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + sb.append(" "); + sb.append(H2CompositeDataType.getRandom()); + } + sb.append(")"); + return new QueryAdapter(sb.toString(), true); + } + +} diff --git a/src/sqlancer/h2/H2ToStringVisitor.java b/src/sqlancer/h2/H2ToStringVisitor.java new file mode 100644 index 000000000..ff2f479a5 --- /dev/null +++ b/src/sqlancer/h2/H2ToStringVisitor.java @@ -0,0 +1,27 @@ +package sqlancer.h2; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; + +public class H2ToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof H2Constant) { + visit((H2Constant) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(H2Constant constant) { + sb.append(constant.toString()); + } + + public static String asString(Node expr) { + H2ToStringVisitor visitor = new H2ToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} From 61e6f1c3c68e0f411fcadcd3699ea52ac037fc62 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 14 Aug 2020 22:42:36 +0200 Subject: [PATCH 0114/1284] [H2] Extend the H2 implementation Generate up to 3 tables, and additionally consider PRIMARY KEY, UNIQUE, and NOT NULL constraints. Add the VARCHAR and DOUBLE types. Add the CREATE INDEX and ANALYZE statements. Add additional operators including IN, BETWEEN, CASE, LIKE, IS DISTINCT FROM, and REGEXP. This is phase 2 of the systematic study on H2. --- .../common/schema/AbstractSchema.java | 2 +- src/sqlancer/h2/H2Constant.java | 38 +++++++++++++ src/sqlancer/h2/H2Errors.java | 5 ++ src/sqlancer/h2/H2ExpressionGenerator.java | 28 ++++++++- src/sqlancer/h2/H2IndexGenerator.java | 57 +++++++++++++++++++ src/sqlancer/h2/H2Provider.java | 32 +++++++---- src/sqlancer/h2/H2Schema.java | 8 ++- src/sqlancer/h2/H2TableGenerator.java | 28 +++++++-- 8 files changed, 179 insertions(+), 19 deletions(-) create mode 100644 src/sqlancer/h2/H2IndexGenerator.java diff --git a/src/sqlancer/common/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java index f0905df03..07814ebfd 100644 --- a/src/sqlancer/common/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -68,7 +68,7 @@ public String getFreeTableName() { } do { String tableName = String.format("t%d", i++); - if (databaseTables.stream().noneMatch(t -> t.getName().contentEquals(tableName))) { + if (databaseTables.stream().noneMatch(t -> t.getName().equalsIgnoreCase(tableName))) { return tableName; } } while (true); diff --git a/src/sqlancer/h2/H2Constant.java b/src/sqlancer/h2/H2Constant.java index 3a372eff5..38eb74673 100644 --- a/src/sqlancer/h2/H2Constant.java +++ b/src/sqlancer/h2/H2Constant.java @@ -46,6 +46,36 @@ public String toString() { } + public static class H2StringConstant extends H2Constant { + + private final String value; + + public H2StringConstant(String value) { + this.value = value; + } + + @Override + public String toString() { + return String.format("'%s'", value); + } + + } + + public static class H2DoubleConstant extends H2Constant { + + private final double value; + + public H2DoubleConstant(double value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + public static Node createIntConstant(long val) { return new H2IntConstant(val); } @@ -58,4 +88,12 @@ public static Node createBoolConstant(boolean val) { return new H2BoolConstant(val); } + public static Node createStringConstant(String val) { + return new H2StringConstant(val); + } + + public static Node createDoubleConstant(double val) { + return new H2DoubleConstant(val); + } + } diff --git a/src/sqlancer/h2/H2Errors.java b/src/sqlancer/h2/H2Errors.java index 85ca0f861..0e7a9701b 100644 --- a/src/sqlancer/h2/H2Errors.java +++ b/src/sqlancer/h2/H2Errors.java @@ -8,11 +8,16 @@ private H2Errors() { } public static void addInsertErrors(ExpectedErrors errors) { + errors.add("NULL not allowed for column"); + errors.add("Unique index or primary key violation"); + errors.add("Data conversion error"); } public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Numeric value out of range"); errors.add("are not comparable"); + errors.add("Data conversion error converting"); + errors.add("Feature not supported"); } } diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index d67ad67f7..c64623bd1 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -5,7 +5,10 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewCaseOperatorNode; +import sqlancer.common.ast.newast.NewInOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; @@ -25,7 +28,7 @@ public H2ExpressionGenerator(H2GlobalState globalState) { } private enum Expression { - BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX; + BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX, IN, BETWEEN, CASE; } @Override @@ -49,6 +52,17 @@ protected Node generateExpression(int depth) { case UNARY_PREFIX: return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), H2UnaryPrefixOperator.getRandom()); + case IN: + return new NewInOperatorNode(generateExpression(depth + 1), + generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); + case BETWEEN: + return new NewBetweenOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new NewCaseOperatorNode(generateExpression(depth + 1), + generateExpressions(depth + 1, nr), generateExpressions(depth + 1, nr), + generateExpression(depth + 1)); default: throw new AssertionError(); } @@ -69,6 +83,10 @@ public Node generateConstant() { return H2Constant.createIntConstant(getUncachedInt()); case BOOL: return H2Constant.createBoolConstant(Randomly.getBoolean()); + case VARCHAR: + return H2Constant.createStringConstant(Character.toString((char) (RANDOM.nextInt('z' - 'a') + 'a'))); + case DOUBLE: + return H2Constant.createDoubleConstant(getUncachedDouble()); default: throw new AssertionError(); } @@ -78,6 +96,10 @@ public static int getUncachedInt() { return RANDOM.nextInt(); } + public static double getUncachedDouble() { + return RANDOM.nextDouble(); + } + public enum H2UnaryPostfixOperator implements Operator { IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); @@ -137,7 +159,9 @@ public static Operator getRandom() { public enum H2BinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + IS_DISTINCT_FROM("IS DISTINCT FROM"), IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), LIKE("LIKE"), + NOT_LIKE("NOT LIKE"), REGEXP("REGEXP"), NOT_REGEXP("NOT REGEXP"); private String textRepr; diff --git a/src/sqlancer/h2/H2IndexGenerator.java b/src/sqlancer/h2/H2IndexGenerator.java new file mode 100644 index 000000000..0f68a7aff --- /dev/null +++ b/src/sqlancer/h2/H2IndexGenerator.java @@ -0,0 +1,57 @@ +package sqlancer.h2; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Table; + +public class H2IndexGenerator { + + private final H2GlobalState globalState; + + public H2IndexGenerator(H2GlobalState globalState) { + this.globalState = globalState; + } + + public static Query getQuery(H2GlobalState globalState) { + return new H2IndexGenerator(globalState).generate(); + } + + private Query generate() { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + if (Randomly.getBoolean()) { + sb.append("UNIQUE "); + errors.add("Unique index or primary key violation"); + } + if (Randomly.getBoolean()) { + sb.append("HASH "); + } + sb.append("INDEX IF NOT EXISTS "); + sb.append(globalState.getSchema().getFreeIndexName()); + sb.append(" ON "); + H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append('('); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(table.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(' '); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBoolean()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + sb.append(')'); + return new QueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index b4a333e5e..296168ebe 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -16,9 +16,11 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Tables; public class H2Provider extends ProviderAdapter { @@ -28,7 +30,9 @@ public H2Provider() { public enum Action implements AbstractAction { - INSERT(H2InsertGenerator::getQuery); + INSERT(H2InsertGenerator::getQuery), // + INDEX(H2IndexGenerator::getQuery), // + ANALYZE((g) -> new QueryAdapter("ANALYZE")); private final QueryProvider queryProvider; @@ -47,6 +51,10 @@ private static int mapActions(H2GlobalState globalState, Action a) { switch (a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case ANALYZE: + return r.getInteger(0, 5); + case INDEX: + return r.getInteger(0, 5); default: throw new AssertionError(a); } @@ -64,10 +72,12 @@ protected void updateSchema() throws SQLException { @Override public void generateDatabase(H2GlobalState globalState) throws SQLException { boolean success = false; - do { // create exactly one table - Query qt = new H2TableGenerator().getQuery(globalState); - success = globalState.executeStatement(qt); - } while (!success); + for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { + do { + Query qt = new H2TableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), H2Provider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { @@ -102,17 +112,19 @@ public H2TLPWhereOracle(H2GlobalState globalState) { @Override public void check() throws SQLException { - List columns = globalState.getSchema().getRandomTableNonEmptyTables().getColumns(); + H2Tables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + String tablesString = tables.tableNamesAsString(); + List columns = tables.getColumns(); String predicate = H2ToStringVisitor .asString(new H2ExpressionGenerator(globalState).setColumns(columns).generateExpression()); - String original = "SELECT * FROM t0;"; + String original = "SELECT * FROM " + tablesString; ExpectedErrors errors = new ExpectedErrors(); H2Errors.addExpressionErrors(errors); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(original, errors, globalState); - String nonNegated = "SELECT * FROM t0 WHERE " + predicate; - String negated = "SELECT * FROM t0 WHERE NOT " + predicate; - String isNull = "SELECT * FROM t0 WHERE " + predicate + " IS NULL"; + String nonNegated = "SELECT * FROM " + tablesString + " WHERE " + predicate; + String negated = "SELECT * FROM " + tablesString + " WHERE NOT " + predicate; + String isNull = "SELECT * FROM " + tablesString + " WHERE " + predicate + " IS NULL"; List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(nonNegated, negated, isNull, diff --git a/src/sqlancer/h2/H2Schema.java b/src/sqlancer/h2/H2Schema.java index b78ac2783..f98542346 100644 --- a/src/sqlancer/h2/H2Schema.java +++ b/src/sqlancer/h2/H2Schema.java @@ -20,7 +20,7 @@ public class H2Schema extends AbstractSchema { public enum H2DataType { - INT, BOOL; + INT, BOOL, VARCHAR, DOUBLE; public static H2DataType getRandom() { return Randomly.fromOptions(values()); @@ -41,7 +41,7 @@ public H2DataType getPrimitiveDataType() { } public static H2CompositeDataType getRandom() { - return new H2CompositeDataType(H2DataType.getRandom()); + return new H2CompositeDataType(Randomly.fromOptions(H2DataType.INT, H2DataType.BOOL, H2DataType.DOUBLE)); } @Override @@ -131,6 +131,10 @@ private static H2DataType getColumnType(String columnType) { return H2DataType.INT; } else if (columnType.startsWith("BOOLEAN")) { return H2DataType.BOOL; + } else if (columnType.startsWith("CHARACTER VARYING")) { + return H2DataType.VARCHAR; + } else if (columnType.startsWith("DOUBLE")) { + return H2DataType.DOUBLE; } else { throw new AssertionError(columnType); } diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java index f77b37f30..28c910636 100644 --- a/src/sqlancer/h2/H2TableGenerator.java +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -1,6 +1,11 @@ package sqlancer.h2; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; @@ -9,18 +14,33 @@ public class H2TableGenerator { public Query getQuery(H2GlobalState globalState) { - StringBuilder sb = new StringBuilder("CREATE TABLE t0("); + StringBuilder sb = new StringBuilder("CREATE TABLE " + globalState.getSchema().getFreeTableName() + "("); + List columnNames = new ArrayList<>(); for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { + columnNames.add("c" + i); + } + + for (int i = 0; i < columnNames.size(); i++) { if (i != 0) { sb.append(", "); } - sb.append("c"); - sb.append(i); + sb.append(columnNames.get(i)); sb.append(" "); sb.append(H2CompositeDataType.getRandom()); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" UNIQUE"); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(", PRIMARY KEY("); + sb.append(Randomly.nonEmptySubset(columnNames).stream().collect(Collectors.joining(", "))); + sb.append(")"); } sb.append(")"); - return new QueryAdapter(sb.toString(), true); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("already exists"), true); } } From f53a88e8bbc5ec3e5077ac82cb70106f71f01106 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 18 Aug 2020 13:35:21 +0200 Subject: [PATCH 0115/1284] [H2] Extend the H2 implementation Create views, and introduce a random query generator. Consider additional keywords (such as ORDER BY) in the TLP WHERE oracle. Generate JOINs. Add generated columns. Add binary arithmetic operators and || for string concatenation. Add the BINARY data type. This is phase 3 of the systematic study on H2. --- src/sqlancer/h2/H2Constant.java | 26 +++++ src/sqlancer/h2/H2Errors.java | 12 ++ src/sqlancer/h2/H2ExpressionGenerator.java | 27 ++++- src/sqlancer/h2/H2InsertGenerator.java | 1 + src/sqlancer/h2/H2Join.java | 106 ++++++++++++++++++ src/sqlancer/h2/H2Options.java | 3 +- src/sqlancer/h2/H2Provider.java | 44 +------- src/sqlancer/h2/H2QueryPartitioningBase.java | 60 ++++++++++ .../h2/H2QueryPartitioningWhereTester.java | 42 +++++++ src/sqlancer/h2/H2RandomQuerySynthesizer.java | 57 ++++++++++ src/sqlancer/h2/H2Schema.java | 13 ++- src/sqlancer/h2/H2Select.java | 8 ++ src/sqlancer/h2/H2TableGenerator.java | 21 +++- src/sqlancer/h2/H2ToStringVisitor.java | 53 +++++++++ src/sqlancer/h2/H2ViewGenerator.java | 34 ++++++ 15 files changed, 457 insertions(+), 50 deletions(-) create mode 100644 src/sqlancer/h2/H2Join.java create mode 100644 src/sqlancer/h2/H2QueryPartitioningBase.java create mode 100644 src/sqlancer/h2/H2QueryPartitioningWhereTester.java create mode 100644 src/sqlancer/h2/H2RandomQuerySynthesizer.java create mode 100644 src/sqlancer/h2/H2Select.java create mode 100644 src/sqlancer/h2/H2ViewGenerator.java diff --git a/src/sqlancer/h2/H2Constant.java b/src/sqlancer/h2/H2Constant.java index 38eb74673..d4bb19423 100644 --- a/src/sqlancer/h2/H2Constant.java +++ b/src/sqlancer/h2/H2Constant.java @@ -76,6 +76,28 @@ public String toString() { } + public static class H2BinaryConstant extends H2Constant { + + private String value; + + public H2BinaryConstant(long value) { + this.value = Long.toHexString(value); + if (this.value.length() % 2 == 1) { + this.value = '0' + this.value; // pad with leading zero if needed + } + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "X'" + value + "'"; + } + + } + public static Node createIntConstant(long val) { return new H2IntConstant(val); } @@ -96,4 +118,8 @@ public static Node createDoubleConstant(double val) { return new H2DoubleConstant(val); } + public static Node createBinaryConstant(long val) { + return new H2BinaryConstant(val); + } + } diff --git a/src/sqlancer/h2/H2Errors.java b/src/sqlancer/h2/H2Errors.java index 0e7a9701b..014176077 100644 --- a/src/sqlancer/h2/H2Errors.java +++ b/src/sqlancer/h2/H2Errors.java @@ -11,6 +11,8 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("NULL not allowed for column"); errors.add("Unique index or primary key violation"); errors.add("Data conversion error"); + errors.add("Generated column"); + errors.add("Value too long for column"); } public static void addExpressionErrors(ExpectedErrors errors) { @@ -18,6 +20,16 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("are not comparable"); errors.add("Data conversion error converting"); errors.add("Feature not supported"); + + errors.add("must be in the GROUP BY list"); + errors.add("must be in the result list in this case"); // ORDER BY + errors.add("Division by zero"); + errors.add("for parameter \"numeric\";"); // Invalid value "CHARACTER VARYING(1)" for parameter "numeric"; SQL + // statement: + + // regexp + errors.add("Unclosed group near index"); + errors.add("Error in LIKE ESCAPE"); } } diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index c64623bd1..b2f51f578 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -28,7 +28,7 @@ public H2ExpressionGenerator(H2GlobalState globalState) { } private enum Expression { - BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX, IN, BETWEEN, CASE; + BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX, IN, BETWEEN, CASE, BINARY_ARITHMETIC; } @Override @@ -63,6 +63,9 @@ protected Node generateExpression(int depth) { return new NewCaseOperatorNode(generateExpression(depth + 1), generateExpressions(depth + 1, nr), generateExpressions(depth + 1, nr), generateExpression(depth + 1)); + case BINARY_ARITHMETIC: + return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), + H2BinaryArithmeticOperator.getRandom()); default: throw new AssertionError(); } @@ -87,6 +90,8 @@ public Node generateConstant() { return H2Constant.createStringConstant(Character.toString((char) (RANDOM.nextInt('z' - 'a') + 'a'))); case DOUBLE: return H2Constant.createDoubleConstant(getUncachedDouble()); + case BINARY: + return H2Constant.createBinaryConstant(getUncachedInt()); default: throw new AssertionError(); } @@ -142,6 +147,26 @@ public static H2UnaryPrefixOperator getRandom() { } + public enum H2BinaryArithmeticOperator implements Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"); + + private String textRepr; + + H2BinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + public enum H2BinaryLogicalOperator implements Operator { AND, OR; diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index 9105528b6..c61353751 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -37,6 +37,7 @@ private Query generate() { sb.append(" VALUES "); insertColumns(columns); H2Errors.addInsertErrors(errors); + H2Errors.addExpressionErrors(errors); // generated columns return new QueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/h2/H2Join.java b/src/sqlancer/h2/H2Join.java new file mode 100644 index 000000000..1b566c5f0 --- /dev/null +++ b/src/sqlancer/h2/H2Join.java @@ -0,0 +1,106 @@ +package sqlancer.h2; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; + +public class H2Join implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + + public enum JoinType { + INNER, CROSS, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public H2Join(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + public static List> getJoins(List> tableList, + H2GlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + H2ExpressionGenerator joinGen = new H2ExpressionGenerator(globalState).setColumns(columns); + JoinType random = H2Join.JoinType.getRandom(); + switch (random) { + case INNER: + joinExpressions.add(H2Join.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + joinExpressions.add(H2Join.createNaturalJoin(leftTable, rightTable)); + break; + case LEFT: + joinExpressions.add(H2Join.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions.add(H2Join.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case CROSS: + joinExpressions.add(new H2Join(leftTable, rightTable, JoinType.CROSS, null)); + break; + default: + throw new AssertionError(random); + } + } + return joinExpressions; + } + + public static H2Join createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new H2Join(left, right, JoinType.RIGHT, predicate); + } + + public static H2Join createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new H2Join(left, right, JoinType.LEFT, predicate); + } + + public static H2Join createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new H2Join(left, right, JoinType.INNER, predicate); + } + + public static Node createNaturalJoin(TableReferenceNode left, + TableReferenceNode right) { + return new H2Join(left, right, JoinType.NATURAL, null); + } + +} diff --git a/src/sqlancer/h2/H2Options.java b/src/sqlancer/h2/H2Options.java index 631c22266..3ae576e1c 100644 --- a/src/sqlancer/h2/H2Options.java +++ b/src/sqlancer/h2/H2Options.java @@ -9,7 +9,6 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.h2.H2Options.H2OracleFactory; import sqlancer.h2.H2Provider.H2GlobalState; -import sqlancer.h2.H2Provider.H2TLPWhereOracle; public class H2Options implements DBMSSpecificOptions { @@ -19,7 +18,7 @@ public enum H2OracleFactory implements OracleFactory { @Override public TestOracle create(H2GlobalState globalState) throws SQLException { - return new H2TLPWhereOracle(globalState); + return new H2QueryPartitioningWhereTester(globalState); } }; diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 296168ebe..7195766ec 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -3,24 +3,17 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; import sqlancer.AbstractAction; -import sqlancer.ComparatorHelper; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Randomly; import sqlancer.StatementExecutor; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.h2.H2Provider.H2GlobalState; -import sqlancer.h2.H2Schema.H2Column; -import sqlancer.h2.H2Schema.H2Tables; public class H2Provider extends ProviderAdapter { @@ -32,7 +25,7 @@ public enum Action implements AbstractAction { INSERT(H2InsertGenerator::getQuery), // INDEX(H2IndexGenerator::getQuery), // - ANALYZE((g) -> new QueryAdapter("ANALYZE")); + ANALYZE((g) -> new QueryAdapter("ANALYZE")), CREATE_VIEW(H2ViewGenerator::getQuery); private final QueryProvider queryProvider; @@ -55,6 +48,8 @@ private static int mapActions(H2GlobalState globalState, Action a) { return r.getInteger(0, 5); case INDEX: return r.getInteger(0, 5); + case CREATE_VIEW: + return r.getInteger(0, 2); default: throw new AssertionError(a); } @@ -102,37 +97,4 @@ public String getDBMSName() { return "h2"; } - public static class H2TLPWhereOracle implements TestOracle { - - private final H2GlobalState globalState; - - public H2TLPWhereOracle(H2GlobalState globalState) { - this.globalState = globalState; - } - - @Override - public void check() throws SQLException { - H2Tables tables = globalState.getSchema().getRandomTableNonEmptyTables(); - String tablesString = tables.tableNamesAsString(); - List columns = tables.getColumns(); - String predicate = H2ToStringVisitor - .asString(new H2ExpressionGenerator(globalState).setColumns(columns).generateExpression()); - String original = "SELECT * FROM " + tablesString; - ExpectedErrors errors = new ExpectedErrors(); - H2Errors.addExpressionErrors(errors); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(original, errors, globalState); - - String nonNegated = "SELECT * FROM " + tablesString + " WHERE " + predicate; - String negated = "SELECT * FROM " + tablesString + " WHERE NOT " + predicate; - String isNull = "SELECT * FROM " + tablesString + " WHERE " + predicate + " IS NULL"; - List combinedString = new ArrayList<>(); - - List secondResultSet = ComparatorHelper.getCombinedResultSet(nonNegated, negated, isNull, - combinedString, true, globalState, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, original, combinedString, - globalState); - } - - } - } diff --git a/src/sqlancer/h2/H2QueryPartitioningBase.java b/src/sqlancer/h2/H2QueryPartitioningBase.java new file mode 100644 index 000000000..b15f1c312 --- /dev/null +++ b/src/sqlancer/h2/H2QueryPartitioningBase.java @@ -0,0 +1,60 @@ +package sqlancer.h2; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; +import sqlancer.h2.H2Schema.H2Tables; + +public class H2QueryPartitioningBase extends TernaryLogicPartitioningOracleBase, H2GlobalState> + implements TestOracle { + + H2Schema s; + H2Tables targetTables; + H2ExpressionGenerator gen; + H2Select select; + + public H2QueryPartitioningBase(H2GlobalState state) { + super(state); + H2Errors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new H2ExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new H2Select(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = H2Join.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + columns.add(new ColumnReferenceNode<>(new H2Column("*", null))); + return columns; + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + +} diff --git a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java new file mode 100644 index 000000000..8de4ecd37 --- /dev/null +++ b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java @@ -0,0 +1,42 @@ +package sqlancer.h2; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.h2.H2Provider.H2GlobalState; + +public class H2QueryPartitioningWhereTester extends H2QueryPartitioningBase { + + public H2QueryPartitioningWhereTester(H2GlobalState state) { + super(state); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = H2ToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setWhereClause(predicate); + String firstQueryString = H2ToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = H2ToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = H2ToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + +} diff --git a/src/sqlancer/h2/H2RandomQuerySynthesizer.java b/src/sqlancer/h2/H2RandomQuerySynthesizer.java new file mode 100644 index 000000000..37942f0e5 --- /dev/null +++ b/src/sqlancer/h2/H2RandomQuerySynthesizer.java @@ -0,0 +1,57 @@ +package sqlancer.h2; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Table; +import sqlancer.h2.H2Schema.H2Tables; + +public final class H2RandomQuerySynthesizer { + + private H2RandomQuerySynthesizer() { + } + + public static H2Select generateSelect(H2GlobalState globalState, int nrColumns) { + H2Tables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState).setColumns(targetTables.getColumns()); + H2Select select = new H2Select(); + List> columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + Node expression = gen.generateExpression(); + columns.add(expression); + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = H2Join.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + + if (Randomly.getBoolean()) { + select.setLimitClause(H2Constant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(H2Constant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + return select; + } + +} diff --git a/src/sqlancer/h2/H2Schema.java b/src/sqlancer/h2/H2Schema.java index f98542346..44a6953c1 100644 --- a/src/sqlancer/h2/H2Schema.java +++ b/src/sqlancer/h2/H2Schema.java @@ -20,7 +20,7 @@ public class H2Schema extends AbstractSchema { public enum H2DataType { - INT, BOOL, VARCHAR, DOUBLE; + INT, BOOL, VARCHAR, DOUBLE, BINARY; public static H2DataType getRandom() { return Randomly.fromOptions(values()); @@ -41,7 +41,8 @@ public H2DataType getPrimitiveDataType() { } public static H2CompositeDataType getRandom() { - return new H2CompositeDataType(Randomly.fromOptions(H2DataType.INT, H2DataType.BOOL, H2DataType.DOUBLE)); + return new H2CompositeDataType( + Randomly.fromOptions(H2DataType.INT, H2DataType.BOOL, H2DataType.DOUBLE, H2DataType.BINARY)); } @Override @@ -78,7 +79,7 @@ public H2Tables getRandomTableNonEmptyTables() { public static class H2Table extends AbstractTable { public H2Table(String tableName, List columns) { - super(tableName, columns, Collections.emptyList(), false); + super(tableName, columns, Collections.emptyList(), tableName.startsWith("V")); } } @@ -135,6 +136,12 @@ private static H2DataType getColumnType(String columnType) { return H2DataType.VARCHAR; } else if (columnType.startsWith("DOUBLE")) { return H2DataType.DOUBLE; + } else if (columnType.startsWith("NUMERIC")) { + return H2DataType.INT; + } else if (columnType.contentEquals("NULL")) { + return H2DataType.INT; // for a NULL view column + } else if (columnType.startsWith("BINARY")) { + return H2DataType.BINARY; } else { throw new AssertionError(columnType); } diff --git a/src/sqlancer/h2/H2Select.java b/src/sqlancer/h2/H2Select.java new file mode 100644 index 000000000..07191659d --- /dev/null +++ b/src/sqlancer/h2/H2Select.java @@ -0,0 +1,8 @@ +package sqlancer.h2; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class H2Select extends SelectBase> implements Node { + +} diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java index 28c910636..d76bdde1f 100644 --- a/src/sqlancer/h2/H2TableGenerator.java +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -9,11 +9,14 @@ import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2CompositeDataType; public class H2TableGenerator { public Query getQuery(H2GlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("already exists"); StringBuilder sb = new StringBuilder("CREATE TABLE " + globalState.getSchema().getFreeTableName() + "("); List columnNames = new ArrayList<>(); for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { @@ -24,15 +27,27 @@ public Query getQuery(H2GlobalState globalState) { if (i != 0) { sb.append(", "); } - sb.append(columnNames.get(i)); + String c = columnNames.get(i); + sb.append(c); sb.append(" "); sb.append(H2CompositeDataType.getRandom()); - if (Randomly.getBooleanWithRatherLowProbability()) { + boolean generated = Randomly.getBooleanWithRatherLowProbability(); + if (Randomly.getBooleanWithRatherLowProbability() && !generated) { sb.append(" UNIQUE"); } if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); } + if (generated) { + sb.append(" AS ("); + List columns = columnNames.stream().filter(cName -> !cName.contentEquals(c)) + .map(c2 -> new H2Column(c2, null)).collect(Collectors.toList()); + H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState).setColumns(columns); + sb.append(H2ToStringVisitor.asString(gen.generateExpression())); + H2Errors.addExpressionErrors(errors); + errors.add("not found"); // generated column cycles + sb.append(')'); + } } if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(", PRIMARY KEY("); @@ -40,7 +55,7 @@ public Query getQuery(H2GlobalState globalState) { sb.append(")"); } sb.append(")"); - return new QueryAdapter(sb.toString(), ExpectedErrors.from("already exists"), true); + return new QueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/h2/H2ToStringVisitor.java b/src/sqlancer/h2/H2ToStringVisitor.java index ff2f479a5..adf043ed0 100644 --- a/src/sqlancer/h2/H2ToStringVisitor.java +++ b/src/sqlancer/h2/H2ToStringVisitor.java @@ -9,6 +9,10 @@ public class H2ToStringVisitor extends NewToStringVisitor { public void visitSpecific(Node expr) { if (expr instanceof H2Constant) { visit((H2Constant) expr); + } else if (expr instanceof H2Select) { + visit((H2Select) expr); + } else if (expr instanceof H2Join) { + visit((H2Join) expr); } else { throw new AssertionError(expr.getClass()); } @@ -18,6 +22,55 @@ private void visit(H2Constant constant) { sb.append(constant.toString()); } + private void visit(H2Join join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + public void visit(H2Select select) { + sb.append("SELECT "); + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + public static String asString(Node expr) { H2ToStringVisitor visitor = new H2ToStringVisitor(); visitor.visit(expr); diff --git a/src/sqlancer/h2/H2ViewGenerator.java b/src/sqlancer/h2/H2ViewGenerator.java new file mode 100644 index 000000000..0ab9fb0f1 --- /dev/null +++ b/src/sqlancer/h2/H2ViewGenerator.java @@ -0,0 +1,34 @@ +package sqlancer.h2; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; + +public final class H2ViewGenerator { + + private H2ViewGenerator() { + } + + public static Query getQuery(H2GlobalState globalState) { + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(H2ToStringVisitor.asString(H2RandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + H2Errors.addExpressionErrors(errors); + return new QueryAdapter(sb.toString(), errors, true); + } + +} From e81df01f843e736ea66356d7301fa624e9b5ba41 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 18 Aug 2020 20:05:56 +0200 Subject: [PATCH 0116/1284] [H2] Extend the H2 implementation Generate UPDATE and DELETE statements. Consider the full range of existing types and their aliases. Add foreign keys, CHECK constraints, and DEFAULT values. Add the CAST, IS TRUE, IS FALSE, and UNKNOWN operators. This is phase 4 of the systematic study on H2. --- src/sqlancer/h2/H2CastNode.java | 24 ++++++ src/sqlancer/h2/H2DeleteGenerator.java | 34 ++++++++ src/sqlancer/h2/H2Errors.java | 9 +++ src/sqlancer/h2/H2ExpressionGenerator.java | 8 +- src/sqlancer/h2/H2Provider.java | 7 +- src/sqlancer/h2/H2Schema.java | 93 ++++++++++++++++++++-- src/sqlancer/h2/H2TableGenerator.java | 54 ++++++++++++- src/sqlancer/h2/H2ToStringVisitor.java | 10 +++ src/sqlancer/h2/H2UpdateGenerator.java | 44 ++++++++++ 9 files changed, 270 insertions(+), 13 deletions(-) create mode 100644 src/sqlancer/h2/H2CastNode.java create mode 100644 src/sqlancer/h2/H2DeleteGenerator.java create mode 100644 src/sqlancer/h2/H2UpdateGenerator.java diff --git a/src/sqlancer/h2/H2CastNode.java b/src/sqlancer/h2/H2CastNode.java new file mode 100644 index 000000000..9d279c8dd --- /dev/null +++ b/src/sqlancer/h2/H2CastNode.java @@ -0,0 +1,24 @@ +package sqlancer.h2; + +import sqlancer.common.ast.newast.Node; +import sqlancer.h2.H2Schema.H2CompositeDataType; + +public class H2CastNode implements Node { + + private final Node expression; + private final H2CompositeDataType type; + + public H2CastNode(Node expression, H2CompositeDataType type) { + this.expression = expression; + this.type = type; + } + + public Node getExpression() { + return expression; + } + + public H2CompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java new file mode 100644 index 000000000..973399eb2 --- /dev/null +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -0,0 +1,34 @@ +package sqlancer.h2; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Table; + +public final class H2DeleteGenerator { + + private H2DeleteGenerator() { + } + + public static Query getQuery(H2GlobalState globalState) { + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(H2ToStringVisitor.asString( + new H2ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); + } + if (Randomly.getBoolean()) { + sb.append(" LIMIT "); + sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); + } + H2Errors.addExpressionErrors(errors); + H2Errors.addDeleteErrors(errors); + return new QueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/h2/H2Errors.java b/src/sqlancer/h2/H2Errors.java index 014176077..8f87b2661 100644 --- a/src/sqlancer/h2/H2Errors.java +++ b/src/sqlancer/h2/H2Errors.java @@ -13,6 +13,9 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Data conversion error"); errors.add("Generated column"); errors.add("Value too long for column"); + errors.add("Referential integrity constraint violation"); + errors.add("Check constraint invalid"); + errors.add("Check constraint violation"); } public static void addExpressionErrors(ExpectedErrors errors) { @@ -32,4 +35,10 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Error in LIKE ESCAPE"); } + public static void addDeleteErrors(ExpectedErrors errors) { + errors.add("No default value is set for column"); // referential actions + errors.add("Referential integrity constraint violation"); + errors.add("NULL not allowed for column"); + } + } diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index b2f51f578..41aa06578 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -15,6 +15,7 @@ import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2CompositeDataType; import sqlancer.h2.H2Schema.H2DataType; public class H2ExpressionGenerator extends UntypedExpressionGenerator, H2Column> { @@ -28,7 +29,7 @@ public H2ExpressionGenerator(H2GlobalState globalState) { } private enum Expression { - BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX, IN, BETWEEN, CASE, BINARY_ARITHMETIC; + BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX, IN, BETWEEN, CASE, BINARY_ARITHMETIC, CAST; } @Override @@ -66,6 +67,8 @@ protected Node generateExpression(int depth) { case BINARY_ARITHMETIC: return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), H2BinaryArithmeticOperator.getRandom()); + case CAST: + return new H2CastNode(generateExpression(depth + 1), H2CompositeDataType.getRandom()); default: throw new AssertionError(); } @@ -107,7 +110,8 @@ public static double getUncachedDouble() { public enum H2UnaryPostfixOperator implements Operator { - IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"), IS_TRUE("IS TRUE"), IS_NOT_TRUE("IS NOT TRUE"), + IS_FALSE("IS FALSE"), IS_NOT_FALSE("IS NOT FALSE"), IS_UNKNOWN("IS NOT UNKNOWN"); private String textRepr; diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 7195766ec..9c8e2d25f 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -25,7 +25,9 @@ public enum Action implements AbstractAction { INSERT(H2InsertGenerator::getQuery), // INDEX(H2IndexGenerator::getQuery), // - ANALYZE((g) -> new QueryAdapter("ANALYZE")), CREATE_VIEW(H2ViewGenerator::getQuery); + ANALYZE((g) -> new QueryAdapter("ANALYZE")), // + CREATE_VIEW(H2ViewGenerator::getQuery), // + UPDATE(H2UpdateGenerator::getQuery), DELETE(H2DeleteGenerator::getQuery); private final QueryProvider queryProvider; @@ -50,6 +52,9 @@ private static int mapActions(H2GlobalState globalState, Action a) { return r.getInteger(0, 5); case CREATE_VIEW: return r.getInteger(0, 2); + case UPDATE: + case DELETE: + return r.getInteger(0, 10); default: throw new AssertionError(a); } diff --git a/src/sqlancer/h2/H2Schema.java b/src/sqlancer/h2/H2Schema.java index 44a6953c1..3f88abcba 100644 --- a/src/sqlancer/h2/H2Schema.java +++ b/src/sqlancer/h2/H2Schema.java @@ -30,10 +30,16 @@ public static H2DataType getRandom() { public static class H2CompositeDataType { + static final int NO_PRECISION = -1; + private final H2DataType dataType; + private final int size; + private final int precision; - public H2CompositeDataType(H2DataType dataType) { + public H2CompositeDataType(H2DataType dataType, int size, int precision) { this.dataType = dataType; + this.size = size; + this.precision = precision; } public H2DataType getPrimitiveDataType() { @@ -41,13 +47,83 @@ public H2DataType getPrimitiveDataType() { } public static H2CompositeDataType getRandom() { - return new H2CompositeDataType( - Randomly.fromOptions(H2DataType.INT, H2DataType.BOOL, H2DataType.DOUBLE, H2DataType.BINARY)); + H2DataType primitiveType = Randomly.fromOptions(H2DataType.INT, H2DataType.BOOL, H2DataType.DOUBLE, + H2DataType.BINARY); + int size = -1; + int precision = NO_PRECISION; + switch (primitiveType) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case DOUBLE: + size = Randomly.fromOptions(4, 8); + if (Randomly.getBoolean()) { + if (size == 4) { + precision = (int) Randomly.getNotCachedInteger(1, 25); // TODO: documentation states 0 as lower + // bound + } else { + precision = (int) Randomly.getNotCachedInteger(25, 54); + } + } + break; + case VARCHAR: + case BINARY: + precision = (int) Randomly.getNotCachedInteger(0, Integer.MAX_VALUE); + break; + default: + break; + } + return new H2CompositeDataType(primitiveType, size, precision); } @Override public String toString() { - return dataType.toString(); + switch (dataType) { + case INT: + switch (size) { + case 1: + return "TINYINT"; + case 2: + return Randomly.fromOptions("SMALLINT", "INT2"); + case 4: + return Randomly.fromOptions("INT", "INTEGER", "MEDIUMINT", "INT4", "SIGNED"); + case 8: + return Randomly.fromOptions("BIGINT", "INT8"); + default: + throw new AssertionError(size); + } + case DOUBLE: + switch (size) { + case 4: + if (precision == NO_PRECISION) { + return Randomly.fromOptions("REAL", "FLOAT4"); + } else { + assert precision >= 0 && precision <= 24; + return String.format("FLOAT(%d)", precision); + } + case 8: + if (precision == NO_PRECISION) { + return Randomly.fromOptions("DOUBLE", "DOUBLE PRECISION", "FLOAT8", "FLOAT"); + } else { + assert precision >= 25 && precision <= 53; + return String.format("FLOAT(%d)", precision); + } + default: + throw new AssertionError(size); + } + case VARCHAR: + return /* String varCharType = */ Randomly.fromOptions("VARCHAR", "VARCHAR_IGNORECASE"); + // if (precision == NO_PRECISION) { + // return varCharType; + // } else { + // return String.format("%s(%d)", varCharType, precision); + // } + case BINARY: + return "BINARY"; + // return String.format("BINARY(%d)", precision); + default: + return dataType.toString(); + } } } @@ -119,7 +195,8 @@ private static List getTableColumns(Connection con, String tableName) String columnName = rs.getString("COLUMN_NAME"); String columnType = rs.getString("TYPE"); H2DataType primitiveType = getColumnType(columnType); - H2Column c = new H2Column(columnName, new H2CompositeDataType(primitiveType)); + H2Column c = new H2Column(columnName, + new H2CompositeDataType(primitiveType, -1, -1 /* TODO: read size and precision */)); columns.add(c); } } @@ -128,13 +205,15 @@ private static List getTableColumns(Connection con, String tableName) } private static H2DataType getColumnType(String columnType) { - if (columnType.startsWith("INTEGER")) { + if (columnType.startsWith("INTEGER") || columnType.startsWith("SMALLINT") || columnType.startsWith("TINYINT") + || columnType.startsWith("BIGINT")) { return H2DataType.INT; } else if (columnType.startsWith("BOOLEAN")) { return H2DataType.BOOL; } else if (columnType.startsWith("CHARACTER VARYING")) { return H2DataType.VARCHAR; - } else if (columnType.startsWith("DOUBLE")) { + } else if (columnType.startsWith("DOUBLE") || columnType.startsWith("DECFLOAT") || columnType.startsWith("REAL") + || columnType.startsWith("FLOAT")) { return H2DataType.DOUBLE; } else if (columnType.startsWith("NUMERIC")) { return H2DataType.INT; diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java index d76bdde1f..1c54c7f5b 100644 --- a/src/sqlancer/h2/H2TableGenerator.java +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; @@ -11,6 +12,7 @@ import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2CompositeDataType; +import sqlancer.h2.H2Schema.H2Table; public class H2TableGenerator { @@ -32,12 +34,16 @@ public Query getQuery(H2GlobalState globalState) { sb.append(" "); sb.append(H2CompositeDataType.getRandom()); boolean generated = Randomly.getBooleanWithRatherLowProbability(); - if (Randomly.getBooleanWithRatherLowProbability() && !generated) { - sb.append(" UNIQUE"); - } if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); } + if (Randomly.getBooleanWithRatherLowProbability() && !generated) { + sb.append(" DEFAULT "); + sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); + } + if (Randomly.getBooleanWithRatherLowProbability() && !generated) { + sb.append(" UNIQUE"); + } if (generated) { sb.append(" AS ("); List columns = columnNames.stream().filter(cName -> !cName.contentEquals(c)) @@ -48,14 +54,56 @@ public Query getQuery(H2GlobalState globalState) { errors.add("not found"); // generated column cycles sb.append(')'); } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" CHECK "); + sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState) + .setColumns(columnNames.stream().map(c2 -> new H2Column(c2, null)).collect(Collectors.toList())) + .generateExpression())); + H2Errors.addExpressionErrors(errors); + } } if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(", PRIMARY KEY("); sb.append(Randomly.nonEmptySubset(columnNames).stream().collect(Collectors.joining(", "))); sb.append(")"); } + if (Randomly.getBooleanWithRatherLowProbability()) { + List foreignKeyColumns = Randomly.nonEmptySubset(columnNames); + sb.append(", FOREIGN KEY("); + sb.append(foreignKeyColumns.stream().collect(Collectors.joining(", "))); + sb.append(')'); + List foreignTableCandidates = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> !t.isView()).collect(Collectors.toList()); + if (foreignTableCandidates.isEmpty()) { + throw new IgnoreMeException(); + } + H2Table foreignKeyTable = Randomly.fromList(foreignTableCandidates); + sb.append(" REFERENCES "); + sb.append(foreignKeyTable.getName()); + sb.append('('); + if (foreignKeyTable.getColumns().size() < foreignKeyColumns.size()) { + throw new IgnoreMeException(); + } + sb.append(foreignKeyTable.getRandomNonEmptyColumnSubset(foreignKeyColumns.size()).stream() + .map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(')'); + if (Randomly.getBoolean()) { + sb.append(" ON DELETE "); + addReferentialAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" ON UPDATE "); + addReferentialAction(sb); + } + errors.add("are not comparable"); + errors.add(" cannot be updatable by a referential constraint with"); // generated columns + } sb.append(")"); return new QueryAdapter(sb.toString(), errors, true); } + private void addReferentialAction(StringBuilder sb) { + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT", "NO ACTION", "SET DEFAULT", "SET NULL")); + } + } diff --git a/src/sqlancer/h2/H2ToStringVisitor.java b/src/sqlancer/h2/H2ToStringVisitor.java index adf043ed0..cb86ede30 100644 --- a/src/sqlancer/h2/H2ToStringVisitor.java +++ b/src/sqlancer/h2/H2ToStringVisitor.java @@ -13,6 +13,8 @@ public void visitSpecific(Node expr) { visit((H2Select) expr); } else if (expr instanceof H2Join) { visit((H2Join) expr); + } else if (expr instanceof H2CastNode) { + visit((H2CastNode) expr); } else { throw new AssertionError(expr.getClass()); } @@ -22,6 +24,14 @@ private void visit(H2Constant constant) { sb.append(constant.toString()); } + private void visit(H2CastNode cast) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + sb.append(cast.getType()); + sb.append(')'); + } + private void visit(H2Join join) { visit(join.getLeftTable()); sb.append(" "); diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java new file mode 100644 index 000000000..111704eb0 --- /dev/null +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -0,0 +1,44 @@ +package sqlancer.h2; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; + +public final class H2UpdateGenerator { + + private H2UpdateGenerator() { + } + + public static Query getQuery(H2GlobalState globalState) { + StringBuilder sb = new StringBuilder("UPDATE "); + ExpectedErrors errors = new ExpectedErrors(); + H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(" SET "); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append("="); + sb.append(H2ToStringVisitor.asString(gen.generateConstant())); + } + H2Errors.addInsertErrors(errors); + H2Errors.addDeleteErrors(errors); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(H2ToStringVisitor.asString(gen.generateExpression())); + } + H2Errors.addExpressionErrors(errors); + return new QueryAdapter(sb.toString(), errors); + } + +} From 3d8d28a39275072aecdf3857c25ba53dcb1d6860 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Aug 2020 10:05:12 +0200 Subject: [PATCH 0117/1284] [H2] Extend the H2 implementation Generate SET and MERGE INTO statements. Add 93 functions. Consider the SELECTIVITY keyword when creating table columns. This is phase 5 of the systematic study on H2. --- src/sqlancer/h2/H2Errors.java | 10 +- src/sqlancer/h2/H2ExpressionGenerator.java | 154 ++++++++++++++++++--- src/sqlancer/h2/H2InsertGenerator.java | 17 ++- src/sqlancer/h2/H2Provider.java | 8 +- src/sqlancer/h2/H2SetGenerator.java | 64 +++++++++ src/sqlancer/h2/H2TableGenerator.java | 10 +- 6 files changed, 239 insertions(+), 24 deletions(-) create mode 100644 src/sqlancer/h2/H2SetGenerator.java diff --git a/src/sqlancer/h2/H2Errors.java b/src/sqlancer/h2/H2Errors.java index 8f87b2661..2a8d7af2d 100644 --- a/src/sqlancer/h2/H2Errors.java +++ b/src/sqlancer/h2/H2Errors.java @@ -27,12 +27,18 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("must be in the GROUP BY list"); errors.add("must be in the result list in this case"); // ORDER BY errors.add("Division by zero"); - errors.add("for parameter \"numeric\";"); // Invalid value "CHARACTER VARYING(1)" for parameter "numeric"; SQL - // statement: // regexp errors.add("Unclosed group near index"); errors.add("Error in LIKE ESCAPE"); + + // functions + errors.add("Invalid value" /* ... for parameter */); + + errors.add("String format error"); // STRINGDECODE + errors.add(/* precision */ "must be between"); // TRUNCATE_VALUE + errors.add("Cannot parse \"TIMESTAMP\" constant"); // TRUNCATE + errors.add("Invalid parameter count for \"TRUNC\", expected count: \"1\""); // TRUNCATE } public static void addDeleteErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index 41aa06578..b60529ad7 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -1,13 +1,12 @@ package sqlancer.h2; -import java.util.Random; - import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.NewCaseOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.NewInOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; @@ -20,8 +19,6 @@ public class H2ExpressionGenerator extends UntypedExpressionGenerator, H2Column> { - private static final Random RANDOM = new Random(); - private final H2GlobalState globalState; public H2ExpressionGenerator(H2GlobalState globalState) { @@ -29,7 +26,8 @@ public H2ExpressionGenerator(H2GlobalState globalState) { } private enum Expression { - BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX, IN, BETWEEN, CASE, BINARY_ARITHMETIC, CAST; + BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX, IN, BETWEEN, CASE, BINARY_ARITHMETIC, CAST, + FUNCTION; } @Override @@ -69,11 +67,141 @@ protected Node generateExpression(int depth) { H2BinaryArithmeticOperator.getRandom()); case CAST: return new H2CastNode(generateExpression(depth + 1), H2CompositeDataType.getRandom()); + case FUNCTION: + H2Function func = H2Function.getRandom(); + return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); default: throw new AssertionError(); } } + public enum H2Function { + + // numeric functions + ABS(1), // + ACOS(1), // + ASIN(1), // + ATAN(1), // + COS(1), // + COSH(1), // + COT(1), // + SIN(1), // + SINH(1), // + TAN(1), // + TANH(1), // + ATAN2(2), // + BITAND(2), // + BITGET(2), // + BITNOT(1), // + BITOR(2), // + BITXOR(2), // + LSHIFT(2), // + RSHIFT(2), // + MOD(2), // + CEILING(1), // + DEGREES(1), // + EXP(1), // + FLOOR(1), // + LN(1), // + LOG(2), // + LOG10(1), // + ORA_HASH(1), // + RADIANS(1), // + SQRT(1), // + PI(0), // + POWER(2), // + ROUND(2), // + ROUNDMAGIC(1), // + SIGN(1), // + TRUNCATE(2), // + COMPRESS(1), // + ZERO(0), // + // string functions + ASCII(1), // + BIT_LENGTH(1), // + LENGTH(1), // + OCTET_LENGTH(1), // + CHAR(1), // + CONCAT(2, true), // + CONCAT_WS(3, true), // + DIFFERENCE(2), // + HEXTORAW(1), // + RAWTOHEX(1), // + INSTR(3), // + INSERT(4), // + LOWER(1), // + UPPER(1), // + LEFT(2), // + RIGHT(2), // + LOCATE(3), // + POSITION(2), // + LTRIM(1), // + RTRIM(1), // + TRIM(1), // + REGEXP_REPLACE(3), // + REGEXP_LIKE(2), // + REPLACE(3), // + SOUNDEX(1), // + STRINGDECODE(1), // + STRINGENCODE(1), // + STRINGTOUTF8(1), // + SUBSTRING(2), // + UTF8TOSTRING(1), // + QUOTE_IDENT(1), // + XMLATTR(2), // + XMLNODE(1), // + XMLCOMMENT(1), // + XMLCDATA(1), // + XMLSTARTDOC(0), // + XMLTEXT(1), // + TRANSLATE(3), // + // TODO: time and date function + // systems functions + // TODO: array functions + CASEWHEN(3), // + COALESCE(1, true), // + CURRENT_SCHEMA(0), // + CURRENT_CATALOG(0), // + DATABASE_PATH(0), // + DECODE(3, true), // + GREATEST(2, true), // + IFNULL(2), // + LEAST(2, true), // + LOCK_MODE(0), // + LOCK_TIMEOUT(0), // + NULLIF(2), // + NVL2(3), // + READONLY(0), // + SESSION_ID(0), // + TRUNCATE_VALUE(3), // + USER(0); + // TODO JSON functions + + private int nrArgs; + private boolean isVariadic; + + H2Function(int nrArgs) { + this(nrArgs, false); + } + + H2Function(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static H2Function getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + } + @Override protected Node generateColumn() { return new ColumnReferenceNode(Randomly.fromList(columns)); @@ -86,28 +214,20 @@ public Node generateConstant() { } switch (H2DataType.getRandom()) { case INT: - return H2Constant.createIntConstant(getUncachedInt()); + return H2Constant.createIntConstant(globalState.getRandomly().getInteger()); case BOOL: return H2Constant.createBoolConstant(Randomly.getBoolean()); case VARCHAR: - return H2Constant.createStringConstant(Character.toString((char) (RANDOM.nextInt('z' - 'a') + 'a'))); + return H2Constant.createStringConstant(globalState.getRandomly().getString()); case DOUBLE: - return H2Constant.createDoubleConstant(getUncachedDouble()); + return H2Constant.createDoubleConstant(globalState.getRandomly().getDouble()); case BINARY: - return H2Constant.createBinaryConstant(getUncachedInt()); + return H2Constant.createBinaryConstant(globalState.getRandomly().getInteger()); default: throw new AssertionError(); } } - public static int getUncachedInt() { - return RANDOM.nextInt(); - } - - public static double getUncachedDouble() { - return RANDOM.nextDouble(); - } - public enum H2UnaryPostfixOperator implements Operator { IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"), IS_TRUE("IS TRUE"), IS_NOT_TRUE("IS NOT TRUE"), diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index c61353751..cc36d3ac4 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -3,6 +3,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; @@ -27,13 +28,27 @@ public static Query getQuery(H2GlobalState globalState) { } private Query generate() { - sb.append("INSERT INTO "); + boolean mergeInto = false; // Randomly.getBooleanWithRatherLowProbability(); + if (mergeInto) { + sb.append("MERGE INTO "); + errors.add("Index \"PRIMARY_KEY_\" not found"); + errors.add("contains null values"); + errors.add("Valid MERGE INTO statement with at least one updatable column"); + } else { + sb.append("INSERT INTO "); + } H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); sb.append(table.getName()); sb.append("("); sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); sb.append(")"); + if (mergeInto && Randomly.getBoolean()) { + sb.append(" KEY("); + sb.append(table.getRandomNonEmptyColumnSubset().stream().map(c -> c.getName()) + .collect(Collectors.joining(", "))); + sb.append(")"); + } sb.append(" VALUES "); insertColumns(columns); H2Errors.addInsertErrors(errors); diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 9c8e2d25f..15ff389ac 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -27,7 +27,9 @@ public enum Action implements AbstractAction { INDEX(H2IndexGenerator::getQuery), // ANALYZE((g) -> new QueryAdapter("ANALYZE")), // CREATE_VIEW(H2ViewGenerator::getQuery), // - UPDATE(H2UpdateGenerator::getQuery), DELETE(H2DeleteGenerator::getQuery); + UPDATE(H2UpdateGenerator::getQuery), // + DELETE(H2DeleteGenerator::getQuery), // + SET(H2SetGenerator::getQuery); private final QueryProvider queryProvider; @@ -49,6 +51,7 @@ private static int mapActions(H2GlobalState globalState, Action a) { case ANALYZE: return r.getInteger(0, 5); case INDEX: + case SET: return r.getInteger(0, 5); case CREATE_VIEW: return r.getInteger(0, 2); @@ -71,6 +74,9 @@ protected void updateSchema() throws SQLException { @Override public void generateDatabase(H2GlobalState globalState) throws SQLException { + if (Randomly.getBoolean()) { + H2SetGenerator.getQuery(globalState).execute(globalState); + } boolean success = false; for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { do { diff --git a/src/sqlancer/h2/H2SetGenerator.java b/src/sqlancer/h2/H2SetGenerator.java new file mode 100644 index 000000000..34065d544 --- /dev/null +++ b/src/sqlancer/h2/H2SetGenerator.java @@ -0,0 +1,64 @@ +package sqlancer.h2; + +import java.util.function.Function; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; + +public final class H2SetGenerator { + + private H2SetGenerator() { + } + + private enum Option { + + ALLOW_LITERALS((r) -> Randomly.fromOptions("ALL")), // + CACHE_SIZE((r) -> Randomly.getNotCachedInteger(0, 1000000)), // + BINARY_COLLATION((r) -> Randomly.fromOptions("SIGNED", "UNSIGNED")), // + UUID_COLLATION((r) -> Randomly.fromOptions("SIGNED", "UNSIGNED")), // + BUILTIN_ALIAS_OVERRIDE((r) -> Randomly.fromOptions("TRUE", "FALSE")), // + COLLATION((r) -> Randomly.fromOptions("OFF", "ENGLISH", "ENGLISH STRENGTH PRIMARY", + "ENGLISH STRENGTH SECONDARY", "ENGLISH STRENGTH TERTIARY", "ENGLISH STRENGTH IDENTICAL")), // + COMPRESS_LOB((r) -> Randomly.fromOptions("NO", "LZF", "DEFLATE")), // + DEFAULT_NULL_ORDERING((r) -> Randomly.fromOptions("LOW", "HIGH", "FIRST", "LAST")), // + DEFAULT_TABLE_TYPE((r) -> Randomly.fromOptions("MEMORY", "CACHED")), // + IGNORECASE((r) -> Randomly.fromOptions("TRUE", "FALSE")), // + LAZY_QUERY_EXECUTION((r) -> Randomly.fromOptions(0, 1)), LOG((r) -> Randomly.fromOptions(0, 1, 2)), // + MAX_MEMORY_ROWS((r) -> Randomly.getNotCachedInteger(0, 100000)), // + MAX_MEMORY_UNDO((r) -> Randomly.getNotCachedInteger(0, 100000)), // + MAX_OPERATION_MEMORY((r) -> Randomly.getNotCachedInteger(0, 100000)), // + // MODE((r) -> Randomly.fromOptions("REGULAR", "DB2", "DERBY", "HSQLDB", "MSSQLSERVER", "ORACLE", + // "POSTGRESQL")); + OPTIMIZE_REUSE_RESULTS((r) -> Randomly.fromOptions(0, 1)), // + QUERY_STATISTICS((r) -> Randomly.fromOptions("TRUE", "FALSE")), // + QUERY_STATISTICS_MAX_ENTRIES((r) -> Randomly.getNotCachedInteger(0, 100000)), // + REFERENTIAL_INTEGRITY((r) -> Randomly.fromOptions("TRUE", "FALSE")), // + UNDO_LOG((r) -> Randomly.fromOptions("TRUE", "FALSE")); + + private Function prod; + + Option(Function prod) { + this.prod = prod; + } + + public static Option getRandom() { + return Randomly.fromOptions(Option.values()); + } + } + + public static Query getQuery(H2GlobalState globalState) { + StringBuilder sb = new StringBuilder(); + Option o = Option.getRandom(); + sb.append("SET "); + sb.append(o); + sb.append(" "); + sb.append(o.prod.apply(globalState.getRandomly())); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("Collation cannot be changed because there is a data table"); + return new QueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java index 1c54c7f5b..3ba631ca6 100644 --- a/src/sqlancer/h2/H2TableGenerator.java +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -41,9 +41,6 @@ public Query getQuery(H2GlobalState globalState) { sb.append(" DEFAULT "); sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); } - if (Randomly.getBooleanWithRatherLowProbability() && !generated) { - sb.append(" UNIQUE"); - } if (generated) { sb.append(" AS ("); List columns = columnNames.stream().filter(cName -> !cName.contentEquals(c)) @@ -54,6 +51,13 @@ public Query getQuery(H2GlobalState globalState) { errors.add("not found"); // generated column cycles sb.append(')'); } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" SELECTIVITY "); + sb.append(Randomly.getNotCachedInteger(0, 101)); + } + if (Randomly.getBooleanWithRatherLowProbability() && !generated) { + sb.append(" UNIQUE"); + } if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" CHECK "); sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState) From 5b16b7727cbc2dfd3d3ea1fff728a6182a614d34 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Aug 2020 14:44:28 +0200 Subject: [PATCH 0118/1284] [H2] Use the default SQLancer random data generation mechanism This is phase 5 of the systematic study on H2. --- src/sqlancer/h2/H2Constant.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/h2/H2Constant.java b/src/sqlancer/h2/H2Constant.java index d4bb19423..4390fed26 100644 --- a/src/sqlancer/h2/H2Constant.java +++ b/src/sqlancer/h2/H2Constant.java @@ -56,7 +56,7 @@ public H2StringConstant(String value) { @Override public String toString() { - return String.format("'%s'", value); + return String.format("'%s'", value.replace("'", "''")); } } @@ -71,7 +71,17 @@ public H2DoubleConstant(double value) { @Override public String toString() { - return String.valueOf(value); + if (value == Double.POSITIVE_INFINITY) { + return "POWER(0, -1)"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "(-POWER(0, -1))"; + } else if (Double.compare(value, -0.0) == 0) { + return "(-CAST(0 AS DOUBLE))"; + } else if (Double.isNaN(value)) { + return "SQRT(-1)"; + } else { + return String.valueOf(value); + } } } From 67b1845181204f9fd55ef0eba4200037c689d8cf Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 20 Aug 2020 14:41:57 +0300 Subject: [PATCH 0119/1284] Re-enable ClickHouse fix fix --- .travis.yml | 2 + src/sqlancer/clickhouse/ClickHouseErrors.java | 17 ++---- src/sqlancer/clickhouse/ClickHouseSchema.java | 1 - .../gen/ClickHouseColumnBuilder.java | 6 +- .../gen/ClickHouseInsertGenerator.java | 6 +- .../gen/ClickHouseTableGenerator.java | 4 +- .../tlp/ClickHouseTLPAggregateOracle.java | 56 +++++++------------ .../oracle/tlp/ClickHouseTLPBase.java | 5 +- .../oracle/tlp/ClickHouseTLPHavingOracle.java | 1 + .../oracle/tlp/ClickHouseTLPWhereOracle.java | 3 + test/sqlancer/dbms/TestClickHouse.java | 16 +++--- 11 files changed, 52 insertions(+), 65 deletions(-) diff --git a/.travis.yml b/.travis.yml index d04ade287..2980122d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,3 +103,5 @@ matrix: - docker pull yandex/clickhouse-server:latest - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest - sleep 5 + script: + - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse test diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index d6657dcf0..a340ac036 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -8,20 +8,12 @@ private ClickHouseErrors() { } public static void addExpectedExpressionErrors(ExpectedErrors errors) { - // errors.add("Illegal type (String) of argument of function not"); - // errors.add("Illegal type String of column for constant filter. Must be UInt8 or Nullable(UInt8)"); - // errors.add("Illegal type Int32 of column for constant filter. Must be UInt8 or Nullable(UInt8)"); - // errors.add("Illegal type UInt32 of column for constant filter. Must be UInt8 or Nullable(UInt8)"); - // errors.add("Illegal type Int32 of column for filter. Must be UInt8 or Nullable(UInt8) or Const variants of - // them."); - // errors.add("Illegal type String of column for filter. Must be UInt8 or Nullable(UInt8) or Const variants of - // them."); - // errors.add("Illegal type Int64 of column for constant filter. Must be UInt8 or Nullable(UInt8)"); errors.add("Illegal type"); errors.add("Argument at index 1 for function like must be constant"); errors.add("Argument at index 1 for function notLike must be constant"); - - // regex + errors.add( + "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); + errors.add("does not return a value of type UInt8"); errors.add("invalid escape sequence"); errors.add("invalid character class range"); errors.add("Memory limit"); @@ -35,6 +27,9 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("is not under aggregate function"); errors.add("Invalid type for filter in"); errors.add("argument of function"); + errors.add(" is not under aggregate function and not in GROUP BY"); + errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () + errors.add("OptimizedRegularExpression: cannot compile re2"); } public static void addExpressionHavingErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index efece12dd..44ef2288b 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -86,7 +86,6 @@ public static ClickHouseConstant getConstant(ResultSet randomRowValues, int colu Object value; ClickHouseConstant constant; if (randomRowValues.getString(columnIndex) == null) { - value = null; constant = ClickHouseConstant.createNullConstant(); } else { switch (valueType) { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java index cf78388cb..65eb65623 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.gen; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -19,7 +20,7 @@ public class ClickHouseColumnBuilder { private static boolean allowCodec = true; private enum Constraints { - DEFAULT, MATERIALIZED, ALIAS, CODEC // TTL + DEFAULT, MATERIALIZED, CODEC, ALIAS // TTL } public String createColumn(String columnName, ClickHouseProvider.ClickHouseGlobalState globalState, @@ -44,6 +45,7 @@ public String createColumn(String columnName, ClickHouseProvider.ClickHouseGloba constraints.remove(Constraints.DEFAULT); } else if (constraints.contains(Constraints.ALIAS)) { constraints.remove(Constraints.DEFAULT); + constraints.remove(Constraints.CODEC); } } @@ -51,6 +53,8 @@ public String createColumn(String columnName, ClickHouseProvider.ClickHouseGloba sb.append(dataType); } + Collections.sort(constraints); + for (Constraints c : constraints) { switch (c) { case MATERIALIZED: diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 5cdc1260f..fb4cfb475 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; @@ -22,11 +23,12 @@ public class ClickHouseInsertGenerator extends AbstractInsertGenerator firstResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQuery, errors, state); + + List secondResult = ComparatorHelper.getResultSetFirstColumnAsString(metamorphicText, errors, state); - QueryAdapter q2 = new QueryAdapter(metamorphicText); - try (SQLancerResultSet result = q2.executeAndGet(state)) { - if (result == null) { - throw new IgnoreMeException(); - } - secondResult = result.getString(1); - } catch (Exception e) { - // TODO - throw new IgnoreMeException(); - } state.getState().getLocalState() - .log("--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " + secondResult); - if ((firstResult == null && secondResult != null - || firstResult != null && !firstResult.contentEquals(secondResult)) - && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + .log("--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " + secondResult + + "\n--first size " + firstResult.size() + "\n--second size " + secondResult.size()); + if (firstResult.size() != secondResult.size()) { + throw new AssertionError(); + } else if (firstResult.isEmpty()) { + return; + } else if (firstResult.size() == 1) { + if (firstResult.get(0).equals(secondResult.get(0))) { + return; + } else if (!ComparatorHelper.isEqualDouble(firstResult.get(0), secondResult.get(0))) { + throw new AssertionError(); + } + } else { throw new AssertionError(); - } - } private ClickHouseSelect getSelect(ClickHouseAggregate aggregate, List from, diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index b46199b1a..045aadc51 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -1,7 +1,6 @@ package sqlancer.clickhouse.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -52,10 +51,8 @@ public void check() throws SQLException { } List generateFetchColumns() { - List columns = new ArrayList<>(); - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); - return columns; } @Override diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index f6268ab30..3ab57ca1a 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -23,6 +23,7 @@ public class ClickHouseTLPHavingOracle extends ClickHouseTLPBase { public ClickHouseTLPHavingOracle(ClickHouseProvider.ClickHouseGlobalState state) { super(state); + ClickHouseErrors.addExpectedExpressionErrors(errors); ClickHouseErrors.addGroupingErrors(errors); } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java index ef0d508b6..5023b803a 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java @@ -6,6 +6,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; +import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseVisitor; @@ -13,6 +14,8 @@ public class ClickHouseTLPWhereOracle extends ClickHouseTLPBase { public ClickHouseTLPWhereOracle(ClickHouseProvider.ClickHouseGlobalState state) { super(state); + ClickHouseErrors.addExpectedExpressionErrors(errors); + ClickHouseErrors.addExpressionHavingErrors(errors); } @Override diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 6ce0b5838..1421137c3 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -14,7 +14,7 @@ public void testClickHouseTLPWhereGroupBy() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - Assertions.assertEquals(0, + assertEquals(0, Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPWhere", "--oracle", "TLPGroupBy" })); @@ -25,7 +25,7 @@ public void testClickHouseTLPWhere() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - Assertions.assertEquals(0, + assertEquals(0, Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPWhere" })); @@ -60,8 +60,9 @@ public void testClickHouseTLPDistinct() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", "0", "--num-threads", "5", - "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPDistinct" })); + Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, + "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", + "TLPDistinct" })); } @Test @@ -69,10 +70,9 @@ public void testClickHouseTLPAggregate() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, - "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", - "TLPAggregate" })); + assertEquals(0, Main.executeMain(new String[] { "--timeout-seconds", "0", "--num-queries", "0", "--num-threads", + "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPAggregate" })); // Disabled + // https://github.com/ClickHouse/ClickHouse/issues/13894 } } From f2125a9d64c8289eff9b3342cfe9363d18900ad0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Aug 2020 20:21:31 +0200 Subject: [PATCH 0120/1284] Add a common row value class for PQS --- .../common/schema/AbstractRowValue.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/sqlancer/common/schema/AbstractRowValue.java diff --git a/src/sqlancer/common/schema/AbstractRowValue.java b/src/sqlancer/common/schema/AbstractRowValue.java new file mode 100644 index 000000000..29bfb0c86 --- /dev/null +++ b/src/sqlancer/common/schema/AbstractRowValue.java @@ -0,0 +1,76 @@ +package sqlancer.common.schema; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public abstract class AbstractRowValue, C extends AbstractTableColumn, O> { + + private final T tables; + private final Map values; + + protected AbstractRowValue(T tables, Map values) { + this.tables = tables; + this.values = values; + } + + public T getTable() { + return tables; + } + + public Map getValues() { + return values; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + int i = 0; + for (C c : tables.getColumns()) { + if (i++ != 0) { + sb.append(", "); + } + sb.append(values.get(c)); + } + return sb.toString(); + } + + public String getRowValuesAsString() { + List columnsToCheck = tables.getColumns(); + return getRowValuesAsString(columnsToCheck); + } + + public String getRowValuesAsString(List columnsToCheck) { + StringBuilder sb = new StringBuilder(); + Map expectedValues = getValues(); + for (int i = 0; i < columnsToCheck.size(); i++) { + if (i != 0) { + sb.append(", "); + } + O expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); + sb.append(expectedColumnValue); + } + return sb.toString(); + } + + public String asStringGroupedByTables() { + StringBuilder sb = new StringBuilder(); + List columnList = getValues().keySet().stream().collect(Collectors.toList()); + List> tableList = columnList.stream().map(c -> c.getTable()).distinct().sorted() + .collect(Collectors.toList()); + for (AbstractTable t : tableList) { + sb.append("-- " + t.getName() + "\n"); + List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) + .collect(Collectors.toList()); + for (C c : columnsForTable) { + sb.append("--\t"); + sb.append(c); + sb.append("="); + sb.append(getValues().get(c)); + sb.append("\n"); + } + } + return sb.toString(); + } + +} From 212d202dbeacf648273eb626370571419fdd3b3b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Aug 2020 20:22:39 +0200 Subject: [PATCH 0121/1284] [SQLite] Use the row value and schema abstractions --- .../SQLite3PivotedQuerySynthesisOracle.java | 2 +- .../sqlite3/schema/SQLite3Schema.java | 97 ++----------------- 2 files changed, 10 insertions(+), 89 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index b2f2d7ef6..4c44ec09d 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -115,7 +115,7 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio columns.add(t.getRowid()); } } - rw = randomFromTables.getRandomRowValue(database, (SQLite3StateToReproduce) globalState.getState()); + rw = randomFromTables.getRandomRowValue(database); List joinStatements = new ArrayList<>(); for (int i = 1; i < tables.size(); i++) { diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index f0c10a276..d57cbac36 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -10,22 +10,21 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.StateToReproduce.SQLite3StateToReproduce; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table.TableKind; @@ -152,40 +151,13 @@ public static SQLite3Constant getConstant(ResultSet randomRowValues, int columnI return constant; } - public static class SQLite3Tables { - private final List tables; - private final List columns; + public static class SQLite3Tables extends AbstractTables { public SQLite3Tables(List tables) { - this.tables = tables; - columns = new ArrayList<>(); - for (SQLite3Table t : tables) { - columns.addAll(t.getColumns()); - } - } - - public String tableNamesAsString() { - return tables.stream().map(t -> t.getName()).collect(Collectors.joining(", ")); - } - - public List getTables() { - return tables; - } - - public List getColumns() { - return columns; + super(tables); } - public String columnNamesAsString() { - return getColumns().stream().map(t -> t.getTable().getName() + "." + t.getName()) - .collect(Collectors.joining(", ")); - } - - public String columnNamesAsString(Function function) { - return getColumns().stream().map(function).collect(Collectors.joining(", ")); - } - - public SQLite3RowValue getRandomRowValue(Connection con, SQLite3StateToReproduce state) throws SQLException { + public SQLite3RowValue getRandomRowValue(Connection con) throws SQLException { String randomRow = String.format("SELECT %s, %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + c.getName() + ")"), @@ -199,7 +171,7 @@ public SQLite3RowValue getRandomRowValue(Connection con, SQLite3StateToReproduce throw new IgnoreMeException(); } if (!randomRowValues.next()) { - throw new AssertionError("could not find random row! " + randomRow + "\n" + state); + throw new AssertionError("could not find random row! " + randomRow); } for (int i = 0; i < getColumns().size(); i++) { SQLite3Column column = getColumns().get(i); @@ -211,7 +183,6 @@ public SQLite3RowValue getRandomRowValue(Connection con, SQLite3StateToReproduce values.put(column, constant); } assert !randomRowValues.next(); - state.randomRowValues = values; return new SQLite3RowValue(this, values); } @@ -281,54 +252,10 @@ public boolean isReadOnly() { } - public static class SQLite3RowValue { - private final SQLite3Tables tables; - private final Map values; + public static class SQLite3RowValue extends AbstractRowValue { SQLite3RowValue(SQLite3Tables tables, Map values) { - this.tables = tables; - this.values = values; - } - - public SQLite3Tables getTable() { - return tables; - } - - public Map getValues() { - return values; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - int i = 0; - for (SQLite3Column c : tables.getColumns()) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(values.get(c)); - } - return sb.toString(); - } - - public String getRowValuesAsString() { - List columnsToCheck = tables.getColumns(); - return getRowValuesAsString(columnsToCheck); - } - - public String getRowValuesAsString(List columnsToCheck) { - StringBuilder sb = new StringBuilder(); - Map expectedValues = getValues(); - for (int i = 0; i < columnsToCheck.size(); i++) { - if (i != 0) { - sb.append(", "); - } - SQLite3Constant expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); - SQLite3ToStringVisitor visitor = new SQLite3ToStringVisitor(); - visitor.visit(expectedColumnValue); - sb.append(visitor.get()); - } - return sb.toString(); + super(tables, values); } } @@ -404,13 +331,7 @@ public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throw boolean isDbStatsTable = sqlString.contains("using dbstat"); List databaseColumns = getTableColumns(con, tableName, sqlString, isView, isDbStatsTable); - int nrRows; - try { - // FIXME - nrRows = getNrRows(globalState, tableName); - } catch (IgnoreMeException e) { - nrRows = 0; - } + int nrRows = getNrRows(globalState, tableName); SQLite3Table t = new SQLite3Table(tableName, databaseColumns, tableType.contentEquals("temp_table") ? TableKind.TEMP : TableKind.MAIN, withoutRowid, nrRows, isView, isVirtual, isReadOnly); From cfef87874bb7be3672de4af389002e12654f91b5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Aug 2020 20:23:16 +0200 Subject: [PATCH 0122/1284] [ClickHouse] Use the row value and tables abstractions --- src/sqlancer/clickhouse/ClickHouseSchema.java | 120 +----------------- 1 file changed, 7 insertions(+), 113 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index 44ef2288b..c9e77f30a 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -6,21 +6,18 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; import ru.yandex.clickhouse.domain.ClickHouseDataType; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.StateToReproduce; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ast.ClickHouseConstant; +import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; public class ClickHouseSchema extends AbstractSchema { @@ -140,123 +137,20 @@ public static ClickHouseConstant getConstant(ResultSet randomRowValues, int colu return constant; } - public static class ClickHouseRowValue { - private final ClickHouseSchema.ClickHouseTables tables; - private final Map values; + public static class ClickHouseRowValue + extends AbstractRowValue { ClickHouseRowValue(ClickHouseSchema.ClickHouseTables tables, Map values) { - this.tables = tables; - this.values = values; - } - - public ClickHouseSchema.ClickHouseTables getTable() { - return tables; - } - - public Map getValues() { - return values; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - int i = 0; - for (ClickHouseSchema.ClickHouseColumn c : tables.getColumns()) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(values.get(c)); - } - return sb.toString(); - } - - public String getRowValuesAsString() { - List columnsToCheck = tables.getColumns(); - return getRowValuesAsString(columnsToCheck); - } - - public String getRowValuesAsString(List columnsToCheck) { - StringBuilder sb = new StringBuilder(); - Map expectedValues = getValues(); - for (int i = 0; i < columnsToCheck.size(); i++) { - if (i != 0) { - sb.append(", "); - } - ClickHouseConstant expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); - ClickHouseToStringVisitor visitor = new ClickHouseToStringVisitor(); - visitor.visit(expectedColumnValue); - sb.append(visitor.get()); - } - return sb.toString(); + super(tables, values); } } - public static class ClickHouseTables { - private final List tables; - private final List columns; + public static class ClickHouseTables extends AbstractTables { public ClickHouseTables(List tables) { - this.tables = tables; - columns = new ArrayList<>(); - for (ClickHouseSchema.ClickHouseTable t : tables) { - columns.addAll(t.getColumns()); - } - } - - public String tableNamesAsString() { - return tables.stream().map(t -> t.getName()).collect(Collectors.joining(", ")); - } - - public List getTables() { - return tables; - } - - public List getColumns() { - return columns; - } - - public String columnNamesAsString() { - return getColumns().stream().map(t -> t.getTable().getName() + "." + t.getName()) - .collect(Collectors.joining(", ")); - } - - public String columnNamesAsString(Function function) { - return getColumns().stream().map(function).collect(Collectors.joining(", ")); - } - - public ClickHouseRowValue getRandomRowValue(Connection con, StateToReproduce.ClickHouseStateToReproduce state) - throws SQLException { - String randomRow = String.format("SELECT %s, %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( - c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), - columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + c.getName() + ")"), - tableNamesAsString()); - Map values = new HashMap<>(); - try (Statement s = con.createStatement()) { - ResultSet randomRowValues; - try { - randomRowValues = s.executeQuery(randomRow); - } catch (SQLException e) { - throw new IgnoreMeException(); - } - if (!randomRowValues.next()) { - throw new AssertionError("could not find random row! " + randomRow + "\n" + state); - } - for (int i = 0; i < getColumns().size(); i++) { - ClickHouseSchema.ClickHouseColumn column = getColumns().get(i); - int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); - assert columnIndex == i + 1; - String typeString = randomRowValues.getString(columnIndex + getColumns().size()); - ClickHouseDataType valueType = getColumnType(typeString).getType(); - ClickHouseConstant constant = getConstant(randomRowValues, columnIndex, valueType); - values.put(column, constant); - } - assert !randomRowValues.next(); - state.randomRowValues = values; - return new ClickHouseSchema.ClickHouseRowValue(this, values); - } - + super(tables); } } From 6f496002e7c45f2e8a6a9bddd46c6f1121cb4e93 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Aug 2020 20:23:35 +0200 Subject: [PATCH 0123/1284] [Postgres] Use the row value abstraction --- src/sqlancer/postgres/PostgresSchema.java | 52 ++--------------------- 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 170dfe78b..34049c127 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -17,6 +17,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.StateToReproduce.PostgresStateToReproduce; +import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; @@ -140,55 +141,10 @@ public static PostgresDataType getColumnType(String typeString) { } } - public static class PostgresRowValue { + public static class PostgresRowValue extends AbstractRowValue { - private final PostgresTables tables; - private final Map values; - - PostgresRowValue(PostgresTables tables, Map values) { - this.tables = tables; - this.values = values; - } - - public PostgresTables getTable() { - return tables; - } - - public Map getValues() { - return values; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - int i = 0; - for (PostgresColumn c : tables.getColumns()) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(values.get(c)); - } - return sb.toString(); - } - - public String getRowValuesAsString() { - List columnsToCheck = tables.getColumns(); - return getRowValuesAsString(columnsToCheck); - } - - public String getRowValuesAsString(List columnsToCheck) { - StringBuilder sb = new StringBuilder(); - Map expectedValues = getValues(); - for (int i = 0; i < columnsToCheck.size(); i++) { - if (i != 0) { - sb.append(", "); - } - PostgresConstant expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); - PostgresToStringVisitor visitor = new PostgresToStringVisitor(); - visitor.visit(expectedColumnValue); - sb.append(visitor.get()); - } - return sb.toString(); + protected PostgresRowValue(PostgresTables tables, Map values) { + super(tables, values); } } From bdf44ea7d49a6afedb7ee3ed990739e4fe78ce7f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Aug 2020 20:24:03 +0200 Subject: [PATCH 0124/1284] [MySQL] Use the row value abstraction --- src/sqlancer/mysql/MySQLSchema.java | 50 ++--------------------------- 1 file changed, 3 insertions(+), 47 deletions(-) diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 715180003..8b7502b8d 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -13,6 +13,7 @@ import sqlancer.Randomly; import sqlancer.StateToReproduce.MySQLStateToReproduce; +import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; @@ -167,55 +168,10 @@ private static MySQLDataType getColumnType(String typeString) { } } - public static class MySQLRowValue { - - private final MySQLTables tables; - private final Map values; + public static class MySQLRowValue extends AbstractRowValue { MySQLRowValue(MySQLTables tables, Map values) { - this.tables = tables; - this.values = values; - } - - public MySQLTables getTable() { - return tables; - } - - public Map getValues() { - return values; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - int i = 0; - for (MySQLColumn c : tables.getColumns()) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(values.get(c)); - } - return sb.toString(); - } - - public String getRowValuesAsString() { - List columnsToCheck = tables.getColumns(); - return getRowValuesAsString(columnsToCheck); - } - - public String getRowValuesAsString(List columnsToCheck) { - StringBuilder sb = new StringBuilder(); - Map expectedValues = getValues(); - for (int i = 0; i < columnsToCheck.size(); i++) { - if (i != 0) { - sb.append(", "); - } - MySQLConstant expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); - MySQLToStringVisitor visitor = new MySQLToStringVisitor(); - visitor.visit(expectedColumnValue); - sb.append(visitor.get()); - } - return sb.toString(); + super(tables, values); } } From f5395a7803a98bc6beebb733cbc649847b23c9ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Aug 2020 20:25:01 +0200 Subject: [PATCH 0125/1284] Disable the ClickHouse gate again due to failures --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2980122d9..d04ade287 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,5 +103,3 @@ matrix: - docker pull yandex/clickhouse-server:latest - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest - sleep 5 - script: - - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse test From 542bb527880a4b765e1ae351f28da751d33a4524 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 21 Aug 2020 09:53:56 +0200 Subject: [PATCH 0126/1284] [SQLite] Do not use the PQS test generator for generating SELECTs for CREATE VIEW statements This functionality was initially added since the PQS test generator generated a larger subset of different SELECT statements. --- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 40 ++++--------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index f4cf79136..07b7190d1 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -2,18 +2,15 @@ import java.sql.SQLException; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; +import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.ast.SQLite3Select.SelectType; import sqlancer.sqlite3.gen.SQLite3Common; -import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; import sqlancer.sqlite3.oracle.SQLite3RandomQuerySynthesizer; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -42,38 +39,15 @@ public static Query generate(SQLite3GlobalState globalState) throws SQLException } sb.append(SQLite3Common.getFreeViewName(globalState.getSchema())); ExpectedErrors errors = new ExpectedErrors(); + SQLite3Errors.addExpectedExpressionErrors(errors); errors.add("is circularly defined"); errors.add("unsupported frame specification"); errors.add("The database file is locked"); - if (Randomly.getBoolean()) { - SQLite3PivotedQuerySynthesisOracle queryGen = new SQLite3PivotedQuerySynthesisOracle(globalState); - try { - SQLite3Select q = queryGen.getQuery(globalState); - // for (SQLite3Expression expr : q.getFetchColumns()) { - // if (expr.getAffinity() != null || expr.getImplicitCollateSequence() != null || - // expr.getExplicitCollateSequence() != null) { - // throw new IgnoreMeException(); - // } - // } - if (!globalState.getDmbsSpecificOptions().testDistinctInView) { - q.setSelectType(SelectType.ALL); - } - int size = q.getFetchColumns().size(); - columnNamesAs(sb, size); - sb.append(SQLite3Visitor.asString(q)); - SQLite3PivotedQuerySynthesisOracle.addExpectedErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); - } catch (AssertionError e) { - throw new IgnoreMeException(); - } - } else { - int size = 1 + Randomly.smallNumber(); - columnNamesAs(sb, size); - SQLite3Expression randomQuery = SQLite3RandomQuerySynthesizer.generate(globalState, size); - sb.append(SQLite3Visitor.asString(randomQuery)); - SQLite3PivotedQuerySynthesisOracle.addExpectedErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); - } + int size = 1 + Randomly.smallNumber(); + columnNamesAs(sb, size); + SQLite3Expression randomQuery = SQLite3RandomQuerySynthesizer.generate(globalState, size); + sb.append(SQLite3Visitor.asString(randomQuery)); + return new QueryAdapter(sb.toString(), errors, true); } From 64147691f09eac05fe18c87dfc190afffe697651 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 21 Aug 2020 09:57:50 +0200 Subject: [PATCH 0127/1284] [SQLite] Do not use the DBMS to determine CASTs results While this worked well, this was a non-systematic small experiment. --- src/sqlancer/sqlite3/ast/SQLite3Cast.java | 45 +---------------------- 1 file changed, 2 insertions(+), 43 deletions(-) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Cast.java b/src/sqlancer/sqlite3/ast/SQLite3Cast.java index c3ac74407..99db3c9cb 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Cast.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Cast.java @@ -2,7 +2,6 @@ import java.math.BigDecimal; import java.sql.Connection; -import java.sql.Statement; import java.util.Optional; import java.util.regex.Pattern; @@ -235,54 +234,14 @@ public static SQLite3Constant castToText(SQLite3Constant cons) { return SQLite3Constant.createTextConstant("Inf"); } else if (cons.asDouble() == Double.NEGATIVE_INFINITY) { return SQLite3Constant.createTextConstant("-Inf"); + } else { + return null; } - return castRealToText(cons); - // if (true) { - // throw new IgnoreMeException(); - // } - // NumberFormat fmt = NumberFormat.getInstance(); - // fmt.setGroupingUsed(false); - // fmt.setMaximumIntegerDigits(10); - // fmt.setMinimumFractionDigits(1); - // fmt.setRoundingMode(RoundingMode.UNNECESSARY); - // int digits; - // if (cons.asDouble() < 0) { - // digits = 15; - // } else { - // digits = 15; - // } - // fmt.setMaximumFractionDigits(digits); - // try { - // String s = fmt.format(cons.asDouble()); - // if (s.contentEquals("")) { - // throw new IgnoreMeException(); - // } - // return SQLite3Constant.createTextConstant(s); - // } catch (Exception e) { - // throw new IgnoreMeException(); - // } } if (cons.getDataType() == SQLite3DataType.INT) { return SQLite3Constant.createTextConstant(String.valueOf(cons.asInt())); } - // if (cons.getDataType() == SQLite3DataType.BINARY) { - // try { - // return SQLite3Constant.createTextConstant(new String(cons.asBinary(), "UTF-8").replaceAll("\\p{C}", "")); - // } catch (UnsupportedEncodingException e) { - // throw new AssertionError(e); - // } - // } return null; - // throw new AssertionError(); - } - - private static synchronized SQLite3Constant castRealToText(SQLite3Constant cons) throws AssertionError { - try (Statement s = castDatabase.createStatement()) { - String castResult = s.executeQuery("SELECT CAST(" + cons.asDouble() + " AS TEXT)").getString(1); - return SQLite3Constant.createTextConstant(castResult); - } catch (Exception e) { - throw new AssertionError(e); - } } public static SQLite3Constant asBoolean(SQLite3Constant val) { From b1266a81fce7e0861cb193c376d134d6588281b1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 21 Aug 2020 10:20:28 +0200 Subject: [PATCH 0128/1284] Use the existing ORDER BY generation functionality for the PQS implementations --- .../mysql/gen/MySQLExpressionGenerator.java | 17 +++++++++++++++++ .../MySQLPivotedQuerySynthesisOracle.java | 15 ++------------- .../SQLite3PivotedQuerySynthesisOracle.java | 19 ++----------------- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 8f2ec6d6e..6c41d5dd7 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -26,6 +26,8 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLOrderByTerm; +import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; @@ -198,4 +200,19 @@ public MySQLExpression isNull(MySQLExpression expr) { return new MySQLUnaryPostfixOperation(expr, MySQLUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); } + @Override + public List generateOrderBys() { + List expressions = super.generateOrderBys(); + List newOrderBys = new ArrayList<>(); + for (MySQLExpression expr : expressions) { + if (Randomly.getBoolean()) { + MySQLOrderByTerm newExpr = new MySQLOrderByTerm(expr, MySQLOrder.getRandomOrder()); + newOrderBys.add(newExpr); + } else { + newOrderBys.add(expr); + } + } + return newOrderBys; + } + } diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index e061de2f4..5e576cfb9 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -3,7 +3,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -21,8 +20,6 @@ import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLConstant; import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLOrderByTerm; -import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; @@ -119,7 +116,8 @@ public String getQueryThatContainsAtLeastOneRow() throws SQLException { // "HIGH_PRIORITY" // TODO: Incorrect usage/placement of 'SQL_BUFFER_RESULT' selectStatement.setModifiers(modifiers); - List orderBy = generateOrderBy(columns); + List orderBy = new MySQLExpressionGenerator(globalState).setColumns(columns) + .generateOrderBys(); selectStatement.setOrderByExpressions(orderBy); StringBuilder sb2 = new StringBuilder(); @@ -156,15 +154,6 @@ private List generateGroupByClause(List columns, M } } - public List generateOrderBy(List columns) { - List orderBys = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber(); i++) { - orderBys.add(new MySQLOrderByTerm(MySQLColumnReference.create(Randomly.fromList(columns), null), - MySQLOrder.getRandomOrder())); - } - return orderBys; - } - private MySQLConstant generateLimit() { if (Randomly.getBoolean()) { return MySQLConstant.createIntConstant(Integer.MAX_VALUE); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 4c44ec09d..81e42739c 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -29,8 +29,6 @@ import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; @@ -189,7 +187,8 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio SQLite3Expression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } - List orderBy = generateOrderBy(columns); + List orderBy = new SQLite3ExpressionGenerator(globalState).setColumns(columns) + .generateOrderBys(); selectStatement.setOrderByExpressions(orderBy); if (!groupByClause.isEmpty() && Randomly.getBoolean()) { SQLite3Expression randomExpression = SQLite3Common.getTrueExpression(columns, globalState); @@ -258,20 +257,6 @@ private void addExpectedValues(StringBuilder sb) { } } - public List generateOrderBy(List columns) { - List orderBys = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber(); i++) { - SQLite3Expression expr; - expr = new SQLite3ExpressionGenerator(globalState).setColumns(columns).generateExpression(); - Ordering order = Randomly.fromOptions(Ordering.ASC, Ordering.DESC); - orderBys.add(new SQLite3OrderingTerm(expr, order)); - // TODO RANDOM() - } - // TODO collate - errors.add("ORDER BY term out of range"); - return orderBys; - } - private SQLite3Expression generateLimit(long l) { if (Randomly.getBoolean()) { return SQLite3Constant.createIntConstant(r.getLong(l, Long.MAX_VALUE)); From 124b08119c28161b0e88fb6e654de2bf61a4b6b8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 21 Aug 2020 10:26:57 +0200 Subject: [PATCH 0129/1284] [SQLite] Use the random expression generator to generate JOINs --- .../sqlite3/ast/SQLite3Expression.java | 6 +++- .../SQLite3PivotedQuerySynthesisOracle.java | 30 +++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index df0dd25d4..f7aad3fd7 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -136,7 +136,7 @@ public enum JoinType { private final SQLite3Table table; private SQLite3Expression onClause; - private final JoinType type; + private JoinType type; public Join(SQLite3Table table, SQLite3Expression onClause, JoinType type) { this.table = table; @@ -174,6 +174,10 @@ public void setOnClause(SQLite3Expression onClause) { this.onClause = onClause; } + public void setType(JoinType type) { + this.type = type; + } + } public static class Subquery extends SQLite3Expression { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 81e42739c..c6bde6441 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -115,19 +115,8 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio } rw = randomFromTables.getRandomRowValue(database); - List joinStatements = new ArrayList<>(); - for (int i = 1; i < tables.size(); i++) { - SQLite3Expression joinClause = generateWhereClauseThatContainsRowValue(columns, rw); - SQLite3Table table = Randomly.fromList(tables); - tables.remove(table); - JoinType options; - options = Randomly.fromOptions(JoinType.INNER, JoinType.CROSS, JoinType.OUTER); - if (options == JoinType.OUTER && tables.size() > 2) { - errors.add("ON clause references tables to its right"); - } - Join j = new SQLite3Expression.Join(table, joinClause, options); - joinStatements.add(j); - } + List joinStatements = getJoinStatements(globalState, tables, columns); + selectStatement.setJoinClauses(joinStatements); selectStatement.setFromTables(SQLite3Common.getTableRefs(tables, s)); @@ -201,6 +190,21 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio return selectStatement; } + private List getJoinStatements(SQLite3GlobalState globalState, List tables, + List columns) { + List joinStatements = new SQLite3ExpressionGenerator(globalState).getRandomJoinClauses(tables); + for (Join j : joinStatements) { + if (j.getType() == JoinType.NATURAL) { + /* NATURAL joins have no on clause and cannot be rectified */ + j.setType(JoinType.INNER); + } + // ensure that the join does not exclude the pivot row + j.setOnClause(generateWhereClauseThatContainsRowValue(columns, rw)); + } + errors.add("ON clause references tables to its right"); + return joinStatements; + } + private SQLite3Expression generateOffset() { if (Randomly.getBoolean()) { // OFFSET 0 From 581c4ce1b94f0d5df40fc6b2944191623c52e1d9 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Fri, 21 Aug 2020 14:42:46 +0300 Subject: [PATCH 0130/1284] Add Citus to SQLancer & Citus README, to CI tests, add CitusBugs class as interface with GitHub issues --- .travis.yml | 37 ++++++++++ README.md | 3 + docs/Citus README.md | 67 +++++++++++++++++++ src/sqlancer/citus/CitusBugs.java | 36 ++++++++++ src/sqlancer/citus/CitusOptions.java | 4 +- src/sqlancer/citus/gen/CitusCommon.java | 57 +++++++++------- .../citus/oracle/tlp/CitusTLPBase.java | 6 ++ src/sqlancer/postgres/PostgresOptions.java | 8 +-- src/sqlancer/postgres/gen/PostgresCommon.java | 4 ++ src/sqlancer/tidb/TiDBBugs.java | 3 + test/sqlancer/dbms/TestCitus.java | 23 +++++++ 11 files changed, 217 insertions(+), 31 deletions(-) create mode 100644 docs/Citus README.md create mode 100644 src/sqlancer/citus/CitusBugs.java create mode 100644 test/sqlancer/dbms/TestCitus.java diff --git a/.travis.yml b/.travis.yml index d04ade287..1701898c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -95,6 +95,43 @@ matrix: - createdb test -U postgres script: - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test + - name: Citus (PostgreSQL extension) + jdk : oraclejdk11 + before_install: + - sudo apt-get update + - sudo apt-get install -y postgresql-server-dev-12 postgresql-12 locales + - git clone https://github.com/citusdata/citus.git + - cd citus + - ./configure + - make clean -sj9 + - sudo make install -sj9 + - cd .. + before_script: + - sudo chown -R $USER:$USER /var/run/postgresql + - export PATH=/usr/lib/postgresql/12/bin:$PATH + - mkdir -p citus/coordinator citus/worker1 citus/worker2 + - initdb -D citus/coordinator + - initdb -D citus/worker1 + - initdb -D citus/worker2 + - echo "shared_preload_libraries = 'citus'" >> citus/coordinator/postgresql.conf + - echo "shared_preload_libraries = 'citus'" >> citus/worker1/postgresql.conf + - echo "shared_preload_libraries = 'citus'" >> citus/worker2/postgresql.conf + - pg_ctl -D citus/coordinator -o "-p 9700" -l coordinator_logfile start + - pg_ctl -D citus/worker1 -o "-p 9701" -l worker1_logfile start + - pg_ctl -D citus/worker2 -o "-p 9702" -l worker2_logfile start + - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9700 -d postgres -U $USER + - createdb test -p 9700 -U $USER + - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9701 -d postgres -U $USER + - createdb test -p 9701 -U $USER + - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9702 -d postgres -U $USER + - createdb test -p 9702 -U $USER + - psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test + - psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test + - psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test + - psql -c "SELECT * from master_add_node('localhost', 9701);" -p 9700 -U $USER -d test + - psql -c "SELECT * from master_add_node('localhost', 9702);" -p 9700 -U $USER -d test + script: + - CITUS_AVAILABLE=true mvn -Dtest=TestCitus test - name: ClickHouse jdk : oraclejdk11 services: diff --git a/README.md b/README.md index 399123444..8454ca972 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,9 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | | MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | | PostgreSQL | Working | Typed | | +| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | + + | MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | | CockroachDB | Working | Typed | | | TiDB | Working | Untyped | | diff --git a/docs/Citus README.md b/docs/Citus README.md new file mode 100644 index 000000000..603d7bf36 --- /dev/null +++ b/docs/Citus README.md @@ -0,0 +1,67 @@ +# SQLancer for Citus (PostgreSQL extension) + +SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Management Systems (DBMS) in order to find logic bugs in their implementation. More information about the tool can be found in the [SQLancer README](https://github.com/sqlancer/sqlancer). + +The Citus implementation of SQLancer supports the Ternary Logic Query Partitioning (TLP) test oracle. + +# Setting up + +Instructions for setting up SQLancer are described in [SQLancer - Getting Started](https://github.com/sqlancer/sqlancer#getting-started). + +Requirements for Citus: +* PostgreSQL & Citus - The steps required to build Citus from source are described in [Contributing to Citus](https://github.com/citusdata/citus/blob/master/CONTRIBUTING.md). +Optional Tools for Citus: +* [pgenv](https://github.com/thanodnl/pgenv) (for easier management of PostgreSQL versions) +* [citus_dev](https://github.com/citusdata/tools/tree/develop/citus_dev) (for easier configuration of Citus environment) + +# Using SQLancer + +The following commands run the Citus implementation of SQLancer using Ternary Logic Query Partitioning (TLP): + +``` +cd target +java -jar SQLancer-0.0.1-SNAPSHOT.jar --num-threads 4 citus --oracle QUERY_PARTITIONING +``` + +How to configure the run and how to interpret the output is explained in [SQLancer - Using SQLancer](https://github.com/sqlancer/sqlancer#using-sqlancer). + +# Maintaining & Contributing + +The instructions for setting up a development environment for contributing to SQLancer are explained in [SQLancer - Development](https://github.com/sqlancer/sqlancer/blob/master/CONTRIBUTING.md). + +## Updating expected/ignored Citus errors + +The `CitusBugs.java` file in the `src/sqlancer/citus/` directory and the `CitusCommon.java` file in the `src/sqlancer/citus/gen/` directory should be continuously updated to reflect the currently unsupported functionalities and active bugs. + +Not all SQL commands generated by SQLancer are supported by the DBMS - they might raise `SQLException`s. For instance, a command that involves an invalid casting may raise a `cannnot cast type` error. These errors do not indicate any bugs in the DBMS, which is why it is desirable to quietly ignore them if raised. The `PostgresCommon` and `CitusCommon` classes in SQLancer collect these expected errors and ensure that SQLancer does not explicitly raise an error if an expected error is thrown. + +The `addCitusErrors()` method in `CitusCommon.java` adds Citus-specific errors to the pool of expected errors. It is important to note that it is enough for a string to be a substring of the error message for an error to be ignored. This method is populated with errors that are expected in Citus behavior either because the SQL command generated by SQLancer is currently not supported by Citus, or because a bug that has already been identified has not been fixed yet and is redundantly re-appearing. Both of these, especially the latter group, are dynamic and require updating. + +The `CitusBugs` class in `CitusBugs.java` is an interface between [issues](https://github.com/citusdata/citus/issues?q=is%3Aissue+label%3Asqlancer) opened in the Citus GitHub repository and the bugs listed in the `addCitusErrors()` method in `CitusCommon.java`. Each bug is assigned a corresponding boolean variable, which can be switched to `false` (uninitialized) when the error is fixed on the Citus master branch. + +### What to do: new bug found + +If the bug found is a panic error, i.e. NOT a logic bug (mismatch in result sets identified by the TLP Oracle), this error should be added to the `CitusBugs` class and the `addCitusErrors()` method. +1. Open an issue for the bug in the [Citus GitHub repository](https://github.com/citusdata/citus/issues). +2. Add a boolean variable associated with this issue to the `CitusBugs` class and set it to `true`. +3. Add the error message to the `addCitusErrors()` method wrapped inside an if-statement referring to the boolean created in the `CitusBugs` class. + +If the bug found is a logic bug, i.e. a mismatch in result sets identified by the TLP Oracle, perform step 1 only. + +### What to do: bug fixed + +If the bug fixed was a panic error, i.e. NOT a logic bug (mismatch in result sets identified by the TLP Oracle), the boolean in the `CitusBugs` class corresponding to the issue resolved should be set to `false` (uninitialized) once the fix is merged to the Citus master branch. + +If the bug found was a logic bug, i.e. a mismatch in result sets identified by the TLP Oracle, no actions are necessary. + +### What to do: change in Citus support for PostgreSQL commands + +An error that was previously raised by Citus due to unsupported PostgreSQL functionalities can be removed from the `addCitusErrors()` method if Citus begins supporting this functionality. + +## Modifying the database environment setup + +The `CitusProvider.java` file in the `src/sqlancer/citus/` directory includes the methods for connecting to an existing database and creating the distributed database environment, as well as for preparing the environment for testing (creation of local, distributed, and reference tables and modification of these tables). + +## Modifying JOINs in the SELECT statements generated for testing + +The `CitusTLPBase.java` file in the `src/sqlancer/citus/oracle/tlp/` directory includes the methods for generating JOIN clauses, which can be modified to alter the scope of the JOINs. \ No newline at end of file diff --git a/src/sqlancer/citus/CitusBugs.java b/src/sqlancer/citus/CitusBugs.java new file mode 100644 index 000000000..bc81b9db6 --- /dev/null +++ b/src/sqlancer/citus/CitusBugs.java @@ -0,0 +1,36 @@ +package sqlancer.citus; + +// do not make the fields final to avoid warnings +public final class CitusBugs { + + // https://github.com/citusdata/citus/issues/3987 + public static boolean bug3987; + + // https://github.com/citusdata/citus/issues/3980 + public static boolean bug3980; + + // https://github.com/citusdata/citus/issues/3957 + public static boolean bug3957; + + // https://github.com/citusdata/citus/issues/4019 + public static boolean bug4019 = true; + + // https://github.com/citusdata/citus/issues/4013 + public static boolean bug4013 = true; + + // https://github.com/citusdata/citus/issues/3982 + public static boolean bug3982 = true; + + // https://github.com/citusdata/citus/issues/3981 + public static boolean bug3981 = true; + + // https://github.com/citusdata/citus/issues/4014 + public static boolean bug4014 = true; + + // https://github.com/citusdata/citus/issues/4079 + public static boolean bug4079 = true; + + private CitusBugs() { + } + +} diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index 718e43b91..cdeadb672 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -20,10 +20,10 @@ public class CitusOptions extends PostgresOptions { - @Parameter(names = "--repartition") + @Parameter(names = "--repartition", description = "Specifies whether repartition joins should be allowed", arity = 1) public boolean repartition = true; - @Parameter(names = "--citusoracle") + @Parameter(names = "--citusoracle", description = "Specifies which test oracle should be used for Citus extension to PostgreSQL") public List citusOracle = Arrays.asList(CitusOracleFactory.QUERY_PARTITIONING); public enum CitusOracleFactory implements OracleFactory { diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index b79edc4f8..e97b7b439 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -1,5 +1,6 @@ package sqlancer.citus.gen; +import sqlancer.citus.CitusBugs; import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -11,6 +12,10 @@ private CitusCommon() { } public static void addCitusErrors(ExpectedErrors errors) { + // not supported by Citus + errors.add("failed to evaluate partition key in insert"); + errors.add("cannot perform an INSERT without a partition column value"); + errors.add("cannot perform an INSERT with NULL in the partition column"); errors.add("recursive CTEs are not supported in distributed queries"); errors.add("could not run distributed query with GROUPING SETS, CUBE, or ROLLUP"); errors.add("Subqueries in HAVING cannot refer to outer query"); @@ -30,39 +35,41 @@ public static void addCitusErrors(ExpectedErrors errors) { errors.add("Distributed relations must not use GENERATED ... AS IDENTITY"); errors.add("cannot drop multiple distributed objects in a single command"); errors.add("is not distributed"); - // ERROR: cannot create foreign key constraint - // Detail: SET NULL or SET DEFAULT is not supported in ON DELETE operation when distribution key is included in - // the foreign key constraint - errors.add("cannot create foreign key constraint"); + errors.add("cannot create foreign key constraint"); // SET NULL or SET DEFAULT is not supported in ON DELETE + // operation when distribution key is included in the + // foreign key constraint - // Citus restrictions on SELECT queries + // not supported by Citus (restrictions on SELECT queries) errors.add( "complex joins are only supported when all distributed tables are co-located and joined on their distribution columns"); errors.add( "complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator"); errors.add("cannot perform distributed planning on this query"); errors.add("cannot pushdown the subquery"); - // Check for whether repartition joins are enabled is made during query generation - // errors.add("the query contains a join that requires repartitioning"); - - // SQLancer errors - errors.add("non-integer constant in GROUP BY"); - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("GROUP BY position"); - errors.add("not a foreign key or check constraint"); - - // current Citus errors to be removed once fixed - errors.add("unrecognized node type: 127"); - errors.add("failed to find conversion function from unknown to text"); - errors.add("failed to evaluate partition key in insert"); - errors.add("cannot perform an INSERT without a partition column value"); - errors.add("cannot perform an INSERT with NULL in the partition column"); - errors.add("ERROR: LIMIT must not be negative"); - errors.add("value too long for type"); - // current errors to be removed once upgraded to PostgreSQL 13? - errors.add("unrecognized configuration parameter \"enable_hashagg_disk\""); - errors.add("unrecognized configuration parameter \"enable_groupingsets_hash_disk\""); + // current errors in Citus (to be removed once fixed) + if (CitusBugs.bug3957) { + errors.add("unrecognized node type: 127"); + } + if (CitusBugs.bug3980 || CitusBugs.bug3987 || CitusBugs.bug4019) { + errors.add("syntax error at or near"); + } + if (CitusBugs.bug3982) { + errors.add("failed to find conversion function from unknown to text"); + errors.add("invalid input syntax for"); + } + if (CitusBugs.bug4013) { + errors.add("ERROR: LIMIT must not be negative"); + } + if (CitusBugs.bug3981) { + errors.add("value too long for type"); + } + if (CitusBugs.bug4014) { + errors.add("is ambiguous"); + } + if (CitusBugs.bug4079) { + errors.add("aggregate function calls cannot be nested"); + } } public static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java index 01e2ad6fb..96d718d03 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -136,10 +136,12 @@ private PostgresJoin allColocatedJoins(CitusTable joinTable, CitusTable fromTabl PostgresExpressionGenerator citusJoinGen) { PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); PostgresExpression rightExpr = new PostgresColumnValue(joinTable.getDistributionColumn(), null); + // JOIN over equality between the distribution columns of the tables being joined PostgresExpression equiJoinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); PostgresExpression joinClause = null; if (Randomly.getBooleanWithSmallProbability()) { + // add randomly generated boolean statement to JOIN clause joinClause = new PostgresBinaryLogicalOperation(equiJoinClause, citusJoinGen.generateExpression(PostgresDataType.BOOLEAN), PostgresBinaryLogicalOperation.BinaryLogicalOperator.AND); @@ -160,6 +162,8 @@ private PostgresJoin repartitionJoins(CitusTable joinTable, CitusTable fromTable return null; } PostgresExpression rightExpr = new PostgresColumnValue(Randomly.fromList(candidateRightColumns), null); + // JOIN over equality between the distribution column of one table and a column that matches the data type from + // the other table being joined PostgresExpression joinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); PostgresJoinType options = PostgresJoinType.INNER; @@ -175,6 +179,7 @@ List getCitusJoinStatements(CitusGlobalState globalState, List joinStatements = new ArrayList<>(); PostgresExpressionGenerator citusJoinGen = new PostgresExpressionGenerator(globalState).setColumns(columns); joinTables.remove(fromTable); + // check if all tables being joined are colocated boolean allColocated = true; for (PostgresTable t : joinTables) { boolean colocated = distributedTables.get(fromTable).equals(distributedTables.get(t)); @@ -200,6 +205,7 @@ List getCitusJoinStatements(CitusGlobalState globalState, List joinStatements, PostgresTable fromTable) { + // JOIN with subquery for (int i = 0; i < Randomly.smallNumber(); i++) { PostgresTables subqueryTables = new PostgresTables(Randomly.nonEmptySubset(localTables)); List columns = subqueryTables.getColumns(); diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 1f74e8cc1..8fa2d190e 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -22,16 +22,16 @@ @Parameters public class PostgresOptions implements DBMSSpecificOptions { - @Parameter(names = "--bulk-insert") + @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) public boolean allowBulkInsert; - @Parameter(names = "--oracle") + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for PostgreSQL") public List oracle = Arrays.asList(PostgresOracleFactory.QUERY_PARTITIONING); - @Parameter(names = "--test-collations", arity = 1) + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) public boolean testCollations = true; - @Parameter(names = "--connection-url") + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the PostgreSQL server", arity = 1) public String connectionURL = "postgresql://localhost:5432/test"; public enum PostgresOracleFactory implements OracleFactory { diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index e92b9e0cd..111b60ae0 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -28,6 +28,10 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("missing FROM-clause entry for table"); errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in GROUP BY"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); } public static void addCommonTableErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index da647fd98..08d5d423d 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -42,6 +42,9 @@ public final class TiDBBugs { // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/50 public static boolean bug50 = true; + // https://github.com/citusdata/citus/issues/4079 + public static boolean bug4079 = true; + private TiDBBugs() { } diff --git a/test/sqlancer/dbms/TestCitus.java b/test/sqlancer/dbms/TestCitus.java new file mode 100644 index 000000000..3ef4a7e50 --- /dev/null +++ b/test/sqlancer/dbms/TestCitus.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCitus { + + @Test + public void testCitus() { + String citusAvailable = System.getenv("CITUS_AVAILABLE"); + boolean citusIsAvailable = citusAvailable != null && citusAvailable.equalsIgnoreCase("true"); + assumeTrue(citusIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "citus", "--connection-url", + "postgresql://localhost:9700/test", "--test-collations", "false" })); + } + +} From e499f99e4e63e434ed2d63d9f114b51f80e37cf2 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Fri, 21 Aug 2020 18:29:57 +0300 Subject: [PATCH 0131/1284] Fix to bug that alters subquery columns in Citus subquery joins --- src/sqlancer/citus/oracle/tlp/CitusTLPBase.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java index 96d718d03..8512ed184 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -208,11 +208,12 @@ private void addSubqueryJoinStatements(PostgresGlobalState globalState, List columns = subqueryTables.getColumns(); + List columns = new ArrayList<>(); + columns.addAll(subqueryTables.getColumns()); columns.addAll(fromTable.getColumns()); + PostgresExpression subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); PostgresExpressionGenerator subqueryJoinGen = new PostgresExpressionGenerator(globalState) .setColumns(columns); - PostgresExpression subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); PostgresExpression joinClause = subqueryJoinGen.generateExpression(PostgresDataType.BOOLEAN); PostgresJoinType options = PostgresJoinType.getRandom(); PostgresJoin j = new PostgresJoin(subquery, joinClause, options); From 3e2e69dd81edb47bc4e2465c00b6b2e5046eb891 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 21 Aug 2020 19:45:17 +0200 Subject: [PATCH 0132/1284] Fix the table with supported DBMS --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 8454ca972..fbd28f7d5 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,6 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | | PostgreSQL | Working | Typed | | | Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | - - | MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | | CockroachDB | Working | Typed | | | TiDB | Working | Untyped | | From cb546c69192396a3f29bd18bca01a2d67a779193 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 21 Aug 2020 22:09:24 +0200 Subject: [PATCH 0133/1284] Remove the obsolete printDatabaseSpecificState method --- src/sqlancer/DatabaseProvider.java | 10 ----- src/sqlancer/Main.java | 3 -- src/sqlancer/ProviderAdapter.java | 6 --- src/sqlancer/mysql/MySQLProvider.java | 37 ------------------ src/sqlancer/postgres/PostgresProvider.java | 42 --------------------- src/sqlancer/sqlite3/SQLite3Provider.java | 5 --- 6 files changed, 103 deletions(-) diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index d90c24ba6..81fdbb2e7 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -1,6 +1,5 @@ package sqlancer; -import java.io.FileWriter; import java.sql.Connection; import java.sql.SQLException; @@ -38,15 +37,6 @@ public interface DatabaseProvider, O extends DBMSSpe */ String getDBMSName(); - // TODO: remove this - /** - * Deprecated method to print the database-specific state, previously used for PQS. - * - * @param writer - * @param state - */ - void printDatabaseSpecificState(FileWriter writer, StateToReproduce state); - StateToReproduce getStateToReproduce(String databaseName); } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a7ee59600..9d9c001af 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -66,7 +66,6 @@ public static final class StateLogger { public FileWriter currentFileWriter; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; - private final DatabaseProvider provider; private static final class AlsoWriteToConsoleFileWriter extends FileWriter { @@ -88,7 +87,6 @@ public void write(String str) throws IOException { } public StateLogger(String databaseName, DatabaseProvider provider, MainOptions options) { - this.provider = provider; File dir = new File(LOG_DIRECTORY, provider.getDBMSName()); if (dir.exists() && !dir.isDirectory()) { throw new AssertionError(dir); @@ -243,7 +241,6 @@ private void printState(FileWriter writer, StateToReproduce state) { } catch (IOException e) { throw new AssertionError(e); } - provider.printDatabaseSpecificState(writer, state); } } diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 4d3101ea9..5584efc4e 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -1,6 +1,5 @@ package sqlancer; -import java.io.FileWriter; import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; @@ -20,11 +19,6 @@ public ProviderAdapter(Class globalClass, Class optionClass) { this.optionClass = optionClass; } - @Override - public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { - - } - @Override public StateToReproduce getStateToReproduce(String databaseName) { return new StateToReproduce(databaseName); diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index e92468b43..8e42b0733 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -1,14 +1,10 @@ package sqlancer.mysql; -import java.io.FileWriter; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.List; -import java.util.stream.Collectors; import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; @@ -20,7 +16,6 @@ import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.QueryProvider; -import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; import sqlancer.mysql.gen.MySQLDeleteGenerator; @@ -192,38 +187,6 @@ public String getDBMSName() { return "mysql"; } - @Override - public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { - StringBuilder sb = new StringBuilder(); - MySQLStateToReproduce specificState = (MySQLStateToReproduce) state; - if (specificState.getRandomRowValues() != null) { - List columnList = specificState.getRandomRowValues().keySet().stream() - .collect(Collectors.toList()); - List tableList = columnList.stream().map(c -> c.getTable()).distinct().sorted() - .collect(Collectors.toList()); - for (MySQLTable t : tableList) { - sb.append("-- " + t.getName() + "\n"); - List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) - .collect(Collectors.toList()); - for (MySQLColumn c : columnsForTable) { - sb.append("--\t"); - sb.append(c); - sb.append("="); - sb.append(specificState.getRandomRowValues().get(c)); - sb.append("\n"); - } - } - sb.append("expected values: \n"); - sb.append(MySQLVisitor.asExpectedValues(((MySQLStateToReproduce) state).getWhereClause())); - } - try { - writer.write(sb.toString()); - writer.flush(); - } catch (IOException e) { - throw new AssertionError(); - } - } - @Override public StateToReproduce getStateToReproduce(String databaseName) { return new MySQLStateToReproduce(databaseName); diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 112b9257e..a2085b17c 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -1,7 +1,5 @@ package sqlancer.postgres; -import java.io.FileWriter; -import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.sql.Connection; @@ -9,8 +7,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; @@ -23,9 +19,6 @@ import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.gen.PostgresAlterTableGenerator; import sqlancer.postgres.gen.PostgresAnalyzeGenerator; import sqlancer.postgres.gen.PostgresClusterGenerator; @@ -322,41 +315,6 @@ public String getDBMSName() { return "postgres"; } - @Override - public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { - StringBuilder sb = new StringBuilder(); - PostgresStateToReproduce specificState = (PostgresStateToReproduce) state; - if (specificState.getRandomRowValues() != null) { - List columnList = specificState.getRandomRowValues().keySet().stream() - .collect(Collectors.toList()); - List tableList = columnList.stream().map(c -> c.getTable()).distinct().sorted() - .collect(Collectors.toList()); - for (PostgresTable t : tableList) { - sb.append("-- " + t.getName() + "\n"); - List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) - .collect(Collectors.toList()); - for (PostgresColumn c : columnsForTable) { - sb.append("--\t"); - sb.append(c); - sb.append("="); - sb.append(specificState.getRandomRowValues().get(c)); - sb.append("\n"); - } - } - sb.append("expected values: \n"); - PostgresExpression whereClause = ((PostgresStateToReproduce) state).getWhereClause(); - if (whereClause != null) { - sb.append(PostgresVisitor.asExpectedValues(whereClause).replace("\n", "\n-- ")); - } - } - try { - writer.write(sb.toString()); - writer.flush(); - } catch (IOException e) { - throw new AssertionError(); - } - } - @Override public StateToReproduce getStateToReproduce(String databaseName) { return new PostgresStateToReproduce(databaseName); diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 29d319ed8..7efd40f3e 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -1,7 +1,6 @@ package sqlancer.sqlite3; import java.io.File; -import java.io.FileWriter; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -360,10 +359,6 @@ public String getDBMSName() { return "sqlite3"; } - @Override - public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { - } - @Override public StateToReproduce getStateToReproduce(String databaseName) { return new SQLite3StateToReproduce(databaseName); From 989d834d30a7f0f3dee7056c74de8dcbea861a8d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2020 20:57:00 +0000 Subject: [PATCH 0134/1284] Bump postgresql from 42.2.14 to 42.2.16 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.14 to 42.2.16. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.14...REL42.2.16) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6e013920d..7c77d81f4 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ org.postgresql postgresql - 42.2.14 + 42.2.16 org.xerial From 20f22003c966df9c23b68cb70299aa379a399990 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Aug 2020 00:17:03 +0200 Subject: [PATCH 0135/1284] Add a PQS base class --- .../oracle/PivotedQuerySynthesisBase.java | 18 +++++++++++++ .../MySQLPivotedQuerySynthesisOracle.java | 23 ++++++++--------- .../PostgresPivotedQuerySynthesisOracle.java | 25 +++++++++---------- .../SQLite3PivotedQuerySynthesisOracle.java | 22 ++++++++-------- 4 files changed, 51 insertions(+), 37 deletions(-) create mode 100644 src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java new file mode 100644 index 000000000..9520ab52a --- /dev/null +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -0,0 +1,18 @@ +package sqlancer.common.oracle; + +import sqlancer.GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractRowValue; + +public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E> + implements TestOracle { + + protected final ExpectedErrors errors = new ExpectedErrors(); + protected final S globalState; + protected R pivotRow; + + public PivotedQuerySynthesisBase(S globalState) { + this.globalState = globalState; + } + +} diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 5e576cfb9..b4dc5fb34 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -9,7 +9,7 @@ import sqlancer.Randomly; import sqlancer.StateToReproduce.MySQLStateToReproduce; -import sqlancer.common.oracle.TestOracle; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLColumn; @@ -28,17 +28,16 @@ import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; import sqlancer.mysql.gen.MySQLExpressionGenerator; -public class MySQLPivotedQuerySynthesisOracle implements TestOracle { +public class MySQLPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { private final MySQLStateToReproduce state; private final MySQLSchema s; - private MySQLRowValue rw; private List fetchColumns; private List columns; - private final MySQLGlobalState globalState; public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { - this.globalState = globalState; + super(globalState); this.s = globalState.getSchema(); this.state = (MySQLStateToReproduce) globalState.getState(); } @@ -75,7 +74,7 @@ public String getQueryThatContainsAtLeastOneRow() throws SQLException { // columns.add(t.getRowid()); // } // } - rw = randomFromTables.getRandomRowValue(globalState.getConnection(), state); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection(), state); // List joinStatements = new ArrayList<>(); // for (int i = 1; i < tables.size(); i++) { @@ -100,10 +99,10 @@ public String getQueryThatContainsAtLeastOneRow() throws SQLException { selectStatement.setFetchColumns(fetchColumns); state.queryTargetedColumnsString = columns.stream().map(c -> c.getFullQualifiedName()) .collect(Collectors.joining(", ")); - MySQLExpression whereClause = generateWhereClauseThatContainsRowValue(columns, rw); + MySQLExpression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); selectStatement.setWhereClause(whereClause); state.whereClause = selectStatement; - List groupByClause = generateGroupByClause(columns, rw); + List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); MySQLExpression limitClause = generateLimit(); selectStatement.setLimitClause(limitClause); @@ -130,11 +129,11 @@ public String getQueryThatContainsAtLeastOneRow() throws SQLException { sb2.append(" AND "); } sb2.append(c.getFullQualifiedName()); - if (rw.getValues().get(c).isNull()) { + if (pivotRow.getValues().get(c).isNull()) { sb2.append(" IS NULL"); } else { sb2.append(" = "); - sb2.append(rw.getValues().get(c).getTextRepresentation()); + sb2.append(pivotRow.getValues().get(c).getTextRepresentation()); } } sb2.append(") as result;"); @@ -199,11 +198,11 @@ private boolean isContainedIn(String queryString) throws SQLException { } sb.append("result."); sb.append(c.getTable().getName() + c.getName()); - if (rw.getValues().get(c).isNull()) { + if (pivotRow.getValues().get(c).isNull()) { sb.append(" IS NULL"); } else { sb.append(" = "); - sb.append(rw.getValues().get(c).getTextRepresentation()); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); } } diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 8e400c4f1..4bef2b323 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -14,7 +14,7 @@ import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.StateToReproduce.PostgresStateToReproduce; -import sqlancer.common.oracle.TestOracle; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -28,19 +28,18 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class PostgresPivotedQuerySynthesisOracle implements TestOracle { +public class PostgresPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { private PostgresStateToReproduce state; - private PostgresRowValue rw; private final Connection database; private List fetchColumns; private final PostgresSchema s; private final MainOptions options; private final StateLogger logger; - private final PostgresGlobalState globalState; public PostgresPivotedQuerySynthesisOracle(PostgresGlobalState globalState) throws SQLException { - this.globalState = globalState; + super(globalState); this.database = globalState.getConnection(); this.s = globalState.getSchema(); options = globalState.getOptions(); @@ -72,19 +71,19 @@ public String getQueryThatContainsAtLeastOneRow(PostgresStateToReproduce state) PostgresSelect selectStatement = new PostgresSelect(); selectStatement.setSelectType(Randomly.fromOptions(PostgresSelect.SelectType.values())); List columns = randomFromTables.getColumns(); - rw = randomFromTables.getRandomRowValue(database, state); + pivotRow = randomFromTables.getRandomRowValue(database, state); fetchColumns = columns; selectStatement.setFromList(randomFromTables.getTables().stream().map(t -> new PostgresFromTable(t, false)) .collect(Collectors.toList())); selectStatement.setFetchColumns(fetchColumns.stream() - .map(c -> new PostgresColumnValue(c, rw.getValues().get(c))).collect(Collectors.toList())); + .map(c -> new PostgresColumnValue(c, pivotRow.getValues().get(c))).collect(Collectors.toList())); state.queryTargetedColumnsString = fetchColumns.stream().map(c -> c.getFullQualifiedName()) .collect(Collectors.joining(", ")); - PostgresExpression whereClause = generateWhereClauseThatContainsRowValue(columns, rw); + PostgresExpression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); selectStatement.setWhereClause(whereClause); state.whereClause = selectStatement; - List groupByClause = generateGroupByClause(columns, rw); + List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); PostgresExpression limitClause = generateLimit(); selectStatement.setLimitClause(limitClause); @@ -106,11 +105,11 @@ public String getQueryThatContainsAtLeastOneRow(PostgresStateToReproduce state) sb2.append(" AND "); } sb2.append(c.getFullQualifiedName()); - if (rw.getValues().get(c).isNull()) { + if (pivotRow.getValues().get(c).isNull()) { sb2.append(" IS NULL"); } else { sb2.append(" = "); - sb2.append(rw.getValues().get(c).getTextRepresentation()); + sb2.append(pivotRow.getValues().get(c).getTextRepresentation()); } } sb2.append(") as result;"); @@ -168,11 +167,11 @@ private boolean isContainedIn(String queryString, MainOptions options, StateLogg sb.append("result."); sb.append(c.getTable().getName()); sb.append(c.getName()); - if (rw.getValues().get(c).isNull()) { + if (pivotRow.getValues().get(c).isNull()) { sb.append(" IS NULL"); } else { sb.append(" = "); - sb.append(rw.getValues().get(c).getTextRepresentation()); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); } } String resultingQueryString = sb.toString(); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index c6bde6441..1b0418f10 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -13,7 +13,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.StateToReproduce.SQLite3StateToReproduce; -import sqlancer.common.oracle.TestOracle; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; @@ -44,22 +44,20 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3PivotedQuerySynthesisOracle implements TestOracle { +public class SQLite3PivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { private final Connection database; private final SQLite3Schema s; private final Randomly r; private SQLite3StateToReproduce state; - private SQLite3RowValue rw; private List fetchColumns; - private final ExpectedErrors errors = new ExpectedErrors(); private List colExpressions; - private final SQLite3GlobalState globalState; public SQLite3PivotedQuerySynthesisOracle(SQLite3GlobalState globalState) throws SQLException { + super(globalState); this.database = globalState.getConnection(); this.r = globalState.getRandomly(); - this.globalState = globalState; s = SQLite3Schema.fromConnection(globalState); } @@ -113,7 +111,7 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio columns.add(t.getRowid()); } } - rw = randomFromTables.getRandomRowValue(database); + pivotRow = randomFromTables.getRandomRowValue(database); List joinStatements = getJoinStatements(globalState, tables, columns); @@ -132,7 +130,7 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio allTables.addAll(joinStatements.stream().map(join -> join.getTable()).collect(Collectors.toList())); boolean allTablesContainOneRow = allTables.stream().allMatch(t -> t.getNrRows() == 1); for (SQLite3Column c : fetchColumns) { - SQLite3Expression colName = new SQLite3ColumnName(c, rw.getValues().get(c)); + SQLite3Expression colName = new SQLite3ColumnName(c, pivotRow.getValues().get(c)); if (allTablesContainOneRow && Randomly.getBoolean()) { boolean generateDistinct = Randomly.getBoolean(); if (generateDistinct) { @@ -164,10 +162,10 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio selectStatement.setFetchColumns(colExpressions); globalState.getState().queryTargetedColumnsString = fetchColumns.stream().map(c -> c.getFullQualifiedName()) .collect(Collectors.joining(", ")); - SQLite3Expression whereClause = generateWhereClauseThatContainsRowValue(columns, rw); + SQLite3Expression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); selectStatement.setWhereClause(whereClause); ((SQLite3StateToReproduce) globalState.getState()).whereClause = selectStatement; - List groupByClause = generateGroupByClause(columns, rw, allTablesContainOneRow); + List groupByClause = generateGroupByClause(columns, pivotRow, allTablesContainOneRow); selectStatement.setGroupByClause(groupByClause); SQLite3Expression limitClause = generateLimit((long) (Math.pow(globalState.getOptions().getMaxNumberInserts(), joinStatements.size() + randomFromTables.getTables().size()))); @@ -199,7 +197,7 @@ private List getJoinStatements(SQLite3GlobalState globalState, List columns, SQ private void appendFilter(List columns, StringBuilder sb) { sb.append(" FILTER (WHERE "); - sb.append(SQLite3Visitor.asString(generateWhereClauseThatContainsRowValue(columns, rw))); + sb.append(SQLite3Visitor.asString(generateWhereClauseThatContainsRowValue(columns, pivotRow))); sb.append(")"); } From 20e490006b0893426272677cf987646e9d0856f5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Aug 2020 00:26:18 +0200 Subject: [PATCH 0136/1284] Remove not-strictly necessary fields from the PQS implementations --- .../MySQLPivotedQuerySynthesisOracle.java | 5 +---- .../PostgresPivotedQuerySynthesisOracle.java | 12 +++-------- .../SQLite3PivotedQuerySynthesisOracle.java | 20 ++++++------------- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index b4dc5fb34..383e144ef 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -11,7 +11,6 @@ import sqlancer.StateToReproduce.MySQLStateToReproduce; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -32,13 +31,11 @@ public class MySQLPivotedQuerySynthesisOracle extends PivotedQuerySynthesisBase { private final MySQLStateToReproduce state; - private final MySQLSchema s; private List fetchColumns; private List columns; public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { super(globalState); - this.s = globalState.getSchema(); this.state = (MySQLStateToReproduce) globalState.getState(); } @@ -60,7 +57,7 @@ public void check() throws SQLException { } public String getQueryThatContainsAtLeastOneRow() throws SQLException { - MySQLTables randomFromTables = s.getRandomTableNonEmptyTables(); + MySQLTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); List tables = randomFromTables.getTables(); state.queryTargetedTablesString = randomFromTables.tableNamesAsString(); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 4bef2b323..026750ae8 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -1,6 +1,5 @@ package sqlancer.postgres.oracle; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -16,7 +15,6 @@ import sqlancer.StateToReproduce.PostgresStateToReproduce; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresRowValue; import sqlancer.postgres.PostgresSchema.PostgresTables; @@ -32,16 +30,12 @@ public class PostgresPivotedQuerySynthesisOracle extends PivotedQuerySynthesisBase { private PostgresStateToReproduce state; - private final Connection database; private List fetchColumns; - private final PostgresSchema s; private final MainOptions options; private final StateLogger logger; public PostgresPivotedQuerySynthesisOracle(PostgresGlobalState globalState) throws SQLException { super(globalState); - this.database = globalState.getConnection(); - this.s = globalState.getSchema(); options = globalState.getOptions(); logger = globalState.getLogger(); } @@ -64,14 +58,14 @@ public void check() throws SQLException { public String getQueryThatContainsAtLeastOneRow(PostgresStateToReproduce state) throws SQLException { this.state = state; - PostgresTables randomFromTables = s.getRandomTableNonEmptyTables(); + PostgresTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); state.queryTargetedTablesString = randomFromTables.tableNamesAsString(); PostgresSelect selectStatement = new PostgresSelect(); selectStatement.setSelectType(Randomly.fromOptions(PostgresSelect.SelectType.values())); List columns = randomFromTables.getColumns(); - pivotRow = randomFromTables.getRandomRowValue(database, state); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection(), state); fetchColumns = columns; selectStatement.setFromList(randomFromTables.getTables().stream().map(t -> new PostgresFromTable(t, false)) @@ -153,7 +147,7 @@ private PostgresExpression generateWhereClauseThatContainsRowValue(List { - private final Connection database; - private final SQLite3Schema s; - private final Randomly r; private SQLite3StateToReproduce state; private List fetchColumns; private List colExpressions; public SQLite3PivotedQuerySynthesisOracle(SQLite3GlobalState globalState) throws SQLException { super(globalState); - this.database = globalState.getConnection(); - this.r = globalState.getRandomly(); - s = SQLite3Schema.fromConnection(globalState); } @Override @@ -96,10 +88,10 @@ public static void addExpectedErrors(ExpectedErrors errors) { public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLException { this.state = (SQLite3StateToReproduce) globalState.getState(); - if (s.getDatabaseTables().isEmpty()) { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } - SQLite3Tables randomFromTables = s.getRandomTableNonEmptyTables(); + SQLite3Tables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); List tables = randomFromTables.getTables(); globalState.getState().queryTargetedTablesString = randomFromTables.tableNamesAsString(); @@ -111,12 +103,12 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio columns.add(t.getRowid()); } } - pivotRow = randomFromTables.getRandomRowValue(database); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); List joinStatements = getJoinStatements(globalState, tables, columns); selectStatement.setJoinClauses(joinStatements); - selectStatement.setFromTables(SQLite3Common.getTableRefs(tables, s)); + selectStatement.setFromTables(SQLite3Common.getTableRefs(tables, globalState.getSchema())); // TODO: also implement a wild-card check (*) // filter out row ids from the select because the hinder the reduction process @@ -222,7 +214,7 @@ public static boolean shouldIgnoreException(SQLException e) { private boolean isContainedIn(Query query) throws SQLException { Statement createStatement; - createStatement = database.createStatement(); + createStatement = globalState.getConnection().createStatement(); StringBuilder sb = new StringBuilder(); sb.append("SELECT "); @@ -261,7 +253,7 @@ private void addExpectedValues(StringBuilder sb) { private SQLite3Expression generateLimit(long l) { if (Randomly.getBoolean()) { - return SQLite3Constant.createIntConstant(r.getLong(l, Long.MAX_VALUE)); + return SQLite3Constant.createIntConstant(globalState.getRandomly().getLong(l, Long.MAX_VALUE)); } else { return null; } From f059fa7fc3ffcd4c864f4968b88100b38d10ee23 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Aug 2020 00:45:59 +0200 Subject: [PATCH 0137/1284] Remove the StateToReproduce subclasses --- src/sqlancer/StateToReproduce.java | 98 ------------------- src/sqlancer/mysql/MySQLProvider.java | 7 -- src/sqlancer/mysql/MySQLSchema.java | 6 +- .../MySQLPivotedQuerySynthesisOracle.java | 13 +-- src/sqlancer/postgres/PostgresProvider.java | 7 -- src/sqlancer/postgres/PostgresSchema.java | 6 +- .../PostgresPivotedQuerySynthesisOracle.java | 18 +--- src/sqlancer/sqlite3/SQLite3Provider.java | 7 -- .../SQLite3PivotedQuerySynthesisOracle.java | 6 -- 9 files changed, 9 insertions(+), 159 deletions(-) diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 30ebaff2d..a74c2d628 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -4,22 +4,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; -import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ast.ClickHouseConstant; -import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; -import sqlancer.mysql.MySQLSchema.MySQLColumn; -import sqlancer.mysql.ast.MySQLConstant; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.ast.PostgresConstant; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.sqlite3.ast.SQLite3Constant; -import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; public class StateToReproduce { @@ -101,91 +88,6 @@ public OracleRunReproductionState getLocalState() { return localState; } - public static class MySQLStateToReproduce extends StateToReproduce { - - public Map randomRowValues; - - public MySQLExpression whereClause; - - public String queryThatSelectsRow; - - public MySQLStateToReproduce(String databaseName) { - super(databaseName); - } - - public Map getRandomRowValues() { - return randomRowValues; - } - - public MySQLExpression getWhereClause() { - return whereClause; - } - - } - - public static class SQLite3StateToReproduce extends StateToReproduce { - public Map randomRowValues; - - public SQLite3Expression whereClause; - - public SQLite3StateToReproduce(String databaseName) { - super(databaseName); - } - - public Map getRandomRowValues() { - return randomRowValues; - } - - public SQLite3Expression getWhereClause() { - return whereClause; - } - - } - - public static class PostgresStateToReproduce extends StateToReproduce { - - public Map randomRowValues; - - public PostgresExpression whereClause; - - public String queryThatSelectsRow; - - public PostgresStateToReproduce(String databaseName) { - super(databaseName); - } - - public Map getRandomRowValues() { - return randomRowValues; - } - - public PostgresExpression getWhereClause() { - return whereClause; - } - - } - - public static class ClickHouseStateToReproduce extends StateToReproduce { - - public Map randomRowValues; - - public ClickHouseExpression whereClause; - - public String queryThatSelectsRow; - - public ClickHouseStateToReproduce(String databaseName) { - super(databaseName); - } - - public Map getRandomRowValues() { - return randomRowValues; - } - - public ClickHouseExpression getWhereClause() { - return whereClause; - } - - } - /** * State information that is logged if the test oracle finds a bug or if an exception is thrown. */ diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 8e42b0733..4069b0aab 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -10,8 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Randomly; -import sqlancer.StateToReproduce; -import sqlancer.StateToReproduce.MySQLStateToReproduce; import sqlancer.StatementExecutor; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; @@ -187,9 +185,4 @@ public String getDBMSName() { return "mysql"; } - @Override - public StateToReproduce getStateToReproduce(String databaseName) { - return new MySQLStateToReproduce(databaseName); - } - } diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 8b7502b8d..0173dcd2c 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -12,7 +12,6 @@ import java.util.stream.Stream; import sqlancer.Randomly; -import sqlancer.StateToReproduce.MySQLStateToReproduce; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; @@ -85,7 +84,7 @@ public MySQLTables(List tables) { super(tables); } - public MySQLRowValue getRandomRowValue(Connection con, MySQLStateToReproduce state) throws SQLException { + public MySQLRowValue getRandomRowValue(Connection con) throws SQLException { String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", columnNamesAsString( c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), // columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + @@ -95,7 +94,7 @@ public MySQLRowValue getRandomRowValue(Connection con, MySQLStateToReproduce sta try (Statement s = con.createStatement()) { ResultSet randomRowValues = s.executeQuery(randomRow); if (!randomRowValues.next()) { - throw new AssertionError("could not find random row! " + randomRow + "\n" + state); + throw new AssertionError("could not find random row! " + randomRow + "\n"); } for (int i = 0; i < getColumns().size(); i++) { MySQLColumn column = getColumns().get(i); @@ -135,7 +134,6 @@ public MySQLRowValue getRandomRowValue(Connection con, MySQLStateToReproduce sta values.put(column, constant); } assert !randomRowValues.next(); - state.randomRowValues = values; return new MySQLRowValue(this, values); } diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 383e144ef..f6e9df9fd 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -8,7 +8,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.StateToReproduce.MySQLStateToReproduce; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; @@ -30,13 +29,11 @@ public class MySQLPivotedQuerySynthesisOracle extends PivotedQuerySynthesisBase { - private final MySQLStateToReproduce state; private List fetchColumns; private List columns; public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { super(globalState); - this.state = (MySQLStateToReproduce) globalState.getState(); } @Override @@ -60,18 +57,15 @@ public String getQueryThatContainsAtLeastOneRow() throws SQLException { MySQLTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); List tables = randomFromTables.getTables(); - state.queryTargetedTablesString = randomFromTables.tableNamesAsString(); - MySQLSelect selectStatement = new MySQLSelect(); selectStatement.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); - state.whereClause = selectStatement; columns = randomFromTables.getColumns(); // for (MySQLTable t : tables) { // if (t.getRowid() != null) { // columns.add(t.getRowid()); // } // } - pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection(), state); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); // List joinStatements = new ArrayList<>(); // for (int i = 1; i < tables.size(); i++) { @@ -94,11 +88,8 @@ public String getQueryThatContainsAtLeastOneRow() throws SQLException { fetchColumns = columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); selectStatement.setFetchColumns(fetchColumns); - state.queryTargetedColumnsString = columns.stream().map(c -> c.getFullQualifiedName()) - .collect(Collectors.joining(", ")); MySQLExpression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); selectStatement.setWhereClause(whereClause); - state.whereClause = selectStatement; List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); MySQLExpression limitClause = generateLimit(); @@ -134,7 +125,6 @@ public String getQueryThatContainsAtLeastOneRow() throws SQLException { } } sb2.append(") as result;"); - state.queryThatSelectsRow = sb2.toString(); MySQLToStringVisitor visitor = new MySQLToStringVisitor(); visitor.visit(selectStatement); @@ -204,7 +194,6 @@ private boolean isContainedIn(String queryString) throws SQLException { } String resultingQueryString = sb.toString(); - state.getLocalState().log(resultingQueryString); if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(resultingQueryString); } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index a2085b17c..2e23beea7 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -12,8 +12,6 @@ import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Randomly; -import sqlancer.StateToReproduce; -import sqlancer.StateToReproduce.PostgresStateToReproduce; import sqlancer.StatementExecutor; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; @@ -315,9 +313,4 @@ public String getDBMSName() { return "postgres"; } - @Override - public StateToReproduce getStateToReproduce(String databaseName) { - return new PostgresStateToReproduce(databaseName); - } - } diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 34049c127..add3fde39 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -16,7 +16,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.StateToReproduce.PostgresStateToReproduce; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; @@ -63,7 +62,7 @@ public PostgresTables(List tables) { super(tables); } - public PostgresRowValue getRandomRowValue(Connection con, PostgresStateToReproduce state) throws SQLException { + public PostgresRowValue getRandomRowValue(Connection con) throws SQLException { String randomRow = String.format("SELECT %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), // columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + @@ -73,7 +72,7 @@ public PostgresRowValue getRandomRowValue(Connection con, PostgresStateToReprodu try (Statement s = con.createStatement()) { ResultSet randomRowValues = s.executeQuery(randomRow); if (!randomRowValues.next()) { - throw new AssertionError("could not find random row! " + randomRow + "\n" + state); + throw new AssertionError("could not find random row! " + randomRow + "\n"); } for (int i = 0; i < getColumns().size(); i++) { PostgresColumn column = getColumns().get(i); @@ -100,7 +99,6 @@ public PostgresRowValue getRandomRowValue(Connection con, PostgresStateToReprodu values.put(column, constant); } assert !randomRowValues.next(); - state.randomRowValues = values; return new PostgresRowValue(this, values); } diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 026750ae8..c1c4202c2 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -12,7 +12,6 @@ import sqlancer.Main.StateLogger; import sqlancer.MainOptions; import sqlancer.Randomly; -import sqlancer.StateToReproduce.PostgresStateToReproduce; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -29,7 +28,6 @@ public class PostgresPivotedQuerySynthesisOracle extends PivotedQuerySynthesisBase { - private PostgresStateToReproduce state; private List fetchColumns; private final MainOptions options; private final StateLogger logger; @@ -42,8 +40,8 @@ public PostgresPivotedQuerySynthesisOracle(PostgresGlobalState globalState) thro @Override public void check() throws SQLException { - String queryString = getQueryThatContainsAtLeastOneRow(state); - state.getLocalState().log(queryString); + String queryString = getQueryThatContainsAtLeastOneRow(); + globalState.getState().getLocalState().log(queryString); if (options.logEachSelect()) { logger.writeCurrent(queryString); } @@ -56,27 +54,21 @@ public void check() throws SQLException { } - public String getQueryThatContainsAtLeastOneRow(PostgresStateToReproduce state) throws SQLException { - this.state = state; + public String getQueryThatContainsAtLeastOneRow() throws SQLException { PostgresTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); - state.queryTargetedTablesString = randomFromTables.tableNamesAsString(); - PostgresSelect selectStatement = new PostgresSelect(); selectStatement.setSelectType(Randomly.fromOptions(PostgresSelect.SelectType.values())); List columns = randomFromTables.getColumns(); - pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection(), state); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); fetchColumns = columns; selectStatement.setFromList(randomFromTables.getTables().stream().map(t -> new PostgresFromTable(t, false)) .collect(Collectors.toList())); selectStatement.setFetchColumns(fetchColumns.stream() .map(c -> new PostgresColumnValue(c, pivotRow.getValues().get(c))).collect(Collectors.toList())); - state.queryTargetedColumnsString = fetchColumns.stream().map(c -> c.getFullQualifiedName()) - .collect(Collectors.joining(", ")); PostgresExpression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); selectStatement.setWhereClause(whereClause); - state.whereClause = selectStatement; List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); PostgresExpression limitClause = generateLimit(); @@ -107,7 +99,6 @@ public String getQueryThatContainsAtLeastOneRow(PostgresStateToReproduce state) } } sb2.append(") as result;"); - state.queryThatSelectsRow = sb2.toString(); PostgresToStringVisitor visitor = new PostgresToStringVisitor(); visitor.visit(selectStatement); @@ -170,7 +161,6 @@ private boolean isContainedIn(String queryString, MainOptions options, StateLogg } String resultingQueryString = sb.toString(); // log both SELECT queries at the bottom of the error log file - state.getLocalState().log(String.format("-- %s;\n-- %s;", queryString, resultingQueryString)); if (options.logEachSelect()) { logger.writeCurrent(resultingQueryString); } diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 7efd40f3e..8a10dba7d 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -15,8 +15,6 @@ import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Randomly; -import sqlancer.StateToReproduce; -import sqlancer.StateToReproduce.SQLite3StateToReproduce; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; @@ -359,9 +357,4 @@ public String getDBMSName() { return "sqlite3"; } - @Override - public StateToReproduce getStateToReproduce(String databaseName) { - return new SQLite3StateToReproduce(databaseName); - } - } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index dbe9acb2b..41c2c4c2a 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -11,7 +11,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.StateToReproduce.SQLite3StateToReproduce; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; @@ -45,7 +44,6 @@ public class SQLite3PivotedQuerySynthesisOracle extends PivotedQuerySynthesisBase { - private SQLite3StateToReproduce state; private List fetchColumns; private List colExpressions; @@ -87,7 +85,6 @@ public static void addExpectedErrors(ExpectedErrors errors) { } public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLException { - this.state = (SQLite3StateToReproduce) globalState.getState(); if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } @@ -156,7 +153,6 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio .collect(Collectors.joining(", ")); SQLite3Expression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); selectStatement.setWhereClause(whereClause); - ((SQLite3StateToReproduce) globalState.getState()).whereClause = selectStatement; List groupByClause = generateGroupByClause(columns, pivotRow, allTablesContainOneRow); selectStatement.setGroupByClause(groupByClause); SQLite3Expression limitClause = generateLimit((long) (Math.pow(globalState.getOptions().getMaxNumberInserts(), @@ -221,12 +217,10 @@ private boolean isContainedIn(Query query) throws SQLException { addExpectedValues(sb); StringBuilder sb2 = new StringBuilder(); addExpectedValues(sb2); - state.values = sb2.toString(); sb.append(" INTERSECT SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions sb.append(query.getQueryString()); sb.append(")"); String resultingQueryString = sb.toString(); - state.getLocalState().log(resultingQueryString); Query finalQuery = new QueryAdapter(resultingQueryString, query.getExpectedErrors()); try (ResultSet result = createStatement.executeQuery(finalQuery.getQueryString())) { boolean isContainedIn = !result.isClosed(); From fe64ae60520ad3c33b57eed6b10fc0f4589323a3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Aug 2020 10:48:44 +0200 Subject: [PATCH 0138/1284] Remove the obsolete SQLite methods for adding errors in the PQS class --- .../SQLite3PivotedQuerySynthesisOracle.java | 24 ++----------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 41c2c4c2a..03b0fe10d 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -12,9 +12,9 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.oracle.PivotedQuerySynthesisBase; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; +import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; import sqlancer.sqlite3.SQLite3Visitor; @@ -68,22 +68,10 @@ public Query getQueryThatContainsAtLeastOneRow(SQLite3GlobalState state) throws SQLite3ToStringVisitor visitor = new SQLite3ToStringVisitor(); visitor.visit(selectStatement); String queryString = visitor.get(); - addExpectedErrors(errors); + SQLite3Errors.addExpectedExpressionErrors(errors); return new QueryAdapter(queryString, errors); } - public static void addExpectedErrors(ExpectedErrors errors) { - errors.add("no such index"); - errors.add("no query solution"); - errors.add( - "[SQLITE_ERROR] SQL error or missing database (second argument to likelihood() must be a constant between 0.0 and 1.0)"); - errors.add("[SQLITE_ERROR] SQL error or missing database (integer overflow)"); - errors.add("[SQLITE_ERROR] SQL error or missing database (parser stack overflow)"); - errors.add("second argument to nth_value must be a positive integer"); - errors.add("misuse of aggregate"); - errors.add("GROUP BY term out of range"); - } - public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLException { if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); @@ -200,14 +188,6 @@ private SQLite3Expression generateOffset() { } } - public static boolean shouldIgnoreException(SQLException e) { - return e.getMessage().contentEquals("[SQLITE_ERROR] SQL error or missing database (integer overflow)") - || e.getMessage().startsWith("[SQLITE_ERROR] SQL error or missing database (parser stack overflow)") - || e.getMessage().startsWith( - "[SQLITE_ERROR] SQL error or missing database (second argument to likelihood() must be a constant between 0.0 and 1.0)") - || e.getMessage().contains("second argument to nth_value must be a positive integer"); - } - private boolean isContainedIn(Query query) throws SQLException { Statement createStatement; createStatement = globalState.getConnection().createStatement(); From 2bf37fb9c2da02c9d7395fb0edd35a5b12ecf7ab Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Sat, 22 Aug 2020 11:22:11 +0300 Subject: [PATCH 0139/1284] Citus does not support some contraints that don't include partition column --- src/sqlancer/citus/gen/CitusCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index e97b7b439..a5a4ca200 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -35,6 +35,7 @@ public static void addCitusErrors(ExpectedErrors errors) { errors.add("Distributed relations must not use GENERATED ... AS IDENTITY"); errors.add("cannot drop multiple distributed objects in a single command"); errors.add("is not distributed"); + errors.add("cannot create constraint on"); errors.add("cannot create foreign key constraint"); // SET NULL or SET DEFAULT is not supported in ON DELETE // operation when distribution key is included in the // foreign key constraint From e99efafb650469b55233415d7db220a93d71ec59 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Aug 2020 23:13:22 +0200 Subject: [PATCH 0140/1284] Provide a basic method skeleton in the PQS base class --- .../oracle/PivotedQuerySynthesisBase.java | 38 +++++++++++++++++++ .../MySQLPivotedQuerySynthesisOracle.java | 34 +++++++---------- .../PostgresPivotedQuerySynthesisOracle.java | 32 +++++++--------- .../SQLite3PivotedQuerySynthesisOracle.java | 23 +++++------ 4 files changed, 74 insertions(+), 53 deletions(-) diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index 9520ab52a..0c39b75a1 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -1,13 +1,19 @@ package sqlancer.common.oracle; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + import sqlancer.GlobalState; import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; import sqlancer.common.schema.AbstractRowValue; public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E> implements TestOracle { protected final ExpectedErrors errors = new ExpectedErrors(); + protected final List rectifiedPredicates = new ArrayList<>(); protected final S globalState; protected R pivotRow; @@ -15,4 +21,36 @@ public PivotedQuerySynthesisBase(S globalState) { this.globalState = globalState; } + @Override + public final void check() throws SQLException { + rectifiedPredicates.clear(); + Query pivotRowQuery = getQueryThatContainsAtLeastOneRow(); + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(pivotRowQuery.getQueryString()); + } + boolean isContainedIn = isContainedIn(pivotRowQuery); + if (!isContainedIn) { + reportMissingPivotRow(pivotRowQuery); + } + } + + protected void reportMissingPivotRow(Query query) { + String expectedPivotRowString = pivotRow.asStringGroupedByTables(); + globalState.getState().getLocalState().log(expectedPivotRowString); + + StringBuilder sb = new StringBuilder("-- rectified predicates:\n"); + for (E rectifiedPredicate : rectifiedPredicates) { + sb.append("--"); + sb.append(asString(rectifiedPredicate).replace("\n", "\n-- ")); + } + globalState.getState().getLocalState().log(sb.toString()); + throw new AssertionError(query); + } + + protected abstract boolean isContainedIn(Query pivotRowQuery) throws SQLException; + + protected abstract Query getQueryThatContainsAtLeastOneRow() throws SQLException; + + protected abstract String asString(E expr); + } diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index f6e9df9fd..0c0c0c1f9 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -9,12 +9,16 @@ import sqlancer.Randomly; import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLSchema.MySQLTables; import sqlancer.mysql.MySQLToStringVisitor; +import sqlancer.mysql.MySQLVisitor; import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLConstant; import sqlancer.mysql.ast.MySQLExpression; @@ -37,23 +41,7 @@ public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQL } @Override - public void check() throws SQLException { - String queryString = getQueryThatContainsAtLeastOneRow(); - - try { - boolean isContainedIn = isContainedIn(queryString); - if (!isContainedIn) { - throw new AssertionError(queryString); - } - } catch (SQLException e) { - if (!e.getMessage().contains("BIGINT value is out of range")) { - throw e; - } - } - - } - - public String getQueryThatContainsAtLeastOneRow() throws SQLException { + public Query getQueryThatContainsAtLeastOneRow() throws SQLException { MySQLTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); List tables = randomFromTables.getTables(); @@ -128,7 +116,7 @@ public String getQueryThatContainsAtLeastOneRow() throws SQLException { MySQLToStringVisitor visitor = new MySQLToStringVisitor(); visitor.visit(selectStatement); - return visitor.get(); + return new QueryAdapter(visitor.get(), ExpectedErrors.from("BIGINT value is out of range")); } private List generateGroupByClause(List columns, MySQLRowValue rw) { @@ -170,13 +158,14 @@ private MySQLExpression generateWhereClauseThatContainsRowValue(List generateGroupByClause(List columns, PostgresRowValue rw) { @@ -136,13 +124,14 @@ private PostgresExpression generateWhereClauseThatContainsRowValue(List Date: Sat, 22 Aug 2020 23:55:52 +0200 Subject: [PATCH 0141/1284] [SQLite] Add a method to the expression generator that returns an expression with known result --- .../gen/SQLite3ExpressionGenerator.java | 8 ++++++ .../SQLite3PivotedQuerySynthesisOracle.java | 25 ++++++++----------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index b22caeba6..230f94ce5 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -654,4 +654,12 @@ public SQLite3Expression isNull(SQLite3Expression expr) { return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, expr); } + public SQLite3Expression generateResultKnownExpression() { + SQLite3Expression expr; + do { + expr = generateExpression(); + } while (expr.getExpectedValue() == null); + return expr; + } + } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 95e623bc6..47d26c30e 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -110,11 +110,8 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio errors.add("second argument to nth_value must be a positive integer"); } if (Randomly.getBoolean()) { - SQLite3Expression randomExpression; - do { - randomExpression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) - .generateExpression(); - } while (randomExpression.getExpectedValue() == null); + SQLite3Expression randomExpression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) + .generateResultKnownExpression(); colExpressions.add(randomExpression); } else { colExpressions.add(colName); @@ -259,16 +256,14 @@ private SQLite3Expression generateWhereClauseThatContainsRowValue(List columns, SQLite3RowValue rw) { do { SQLite3Expression expr = new SQLite3ExpressionGenerator(globalState).setRowValue(rw).setColumns(columns) - .generateExpression(); - if (expr.getExpectedValue() != null) { - if (expr.getExpectedValue().isNull()) { - return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, expr); - } - if (SQLite3Cast.isTrue(expr.getExpectedValue()).get()) { - return expr; - } else { - return new SQLite3UnaryOperation(UnaryOperator.NOT, expr); - } + .generateResultKnownExpression(); + if (expr.getExpectedValue().isNull()) { + return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, expr); + } + if (SQLite3Cast.isTrue(expr.getExpectedValue()).get()) { + return expr; + } else { + return new SQLite3UnaryOperation(UnaryOperator.NOT, expr); } } while (true); } From 1a322bac487c16cb5178f712ff7d42afa3887e21 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Aug 2020 23:58:44 +0200 Subject: [PATCH 0142/1284] [SQLite] Improve and document the PQS method for rectifying expressions --- .../SQLite3PivotedQuerySynthesisOracle.java | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 47d26c30e..8dd7dfb3b 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -125,7 +125,7 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio selectStatement.setFetchColumns(colExpressions); globalState.getState().queryTargetedColumnsString = fetchColumns.stream().map(c -> c.getFullQualifiedName()) .collect(Collectors.joining(", ")); - SQLite3Expression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); + SQLite3Expression whereClause = generateRectifiedExpression(columns, pivotRow); selectStatement.setWhereClause(whereClause); List groupByClause = generateGroupByClause(columns, pivotRow, allTablesContainOneRow); selectStatement.setGroupByClause(groupByClause); @@ -159,7 +159,7 @@ private List getJoinStatements(SQLite3GlobalState globalState, List generateGroupByClause(List column } } - private SQLite3Expression generateWhereClauseThatContainsRowValue(List columns, SQLite3RowValue rw) { - - return generateNewExpression(columns, rw); - - } - - private SQLite3Expression generateNewExpression(List columns, SQLite3RowValue rw) { - do { - SQLite3Expression expr = new SQLite3ExpressionGenerator(globalState).setRowValue(rw).setColumns(columns) - .generateResultKnownExpression(); - if (expr.getExpectedValue().isNull()) { - return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, expr); - } - if (SQLite3Cast.isTrue(expr.getExpectedValue()).get()) { - return expr; - } else { - return new SQLite3UnaryOperation(UnaryOperator.NOT, expr); - } - } while (true); + /** + * Generates a predicate that is guaranteed to evaluate to true for the given pivot row. PQS uses this + * method to generate predicates used in WHERE and JOIN clauses. See step 4 of the PQS paper. + * + * @param columns + * @param pivotRow + * + * @return an expression that evaluates to true. + */ + private SQLite3Expression generateRectifiedExpression(List columns, SQLite3RowValue pivotRow) { + SQLite3Expression expr = new SQLite3ExpressionGenerator(globalState).setRowValue(pivotRow).setColumns(columns) + .generateResultKnownExpression(); + SQLite3Expression rectifiedPredicate; + if (expr.getExpectedValue().isNull()) { + // the expr evaluates to NULL => rectify to "expr IS NULL" + rectifiedPredicate = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, expr); + } else if (SQLite3Cast.isTrue(expr.getExpectedValue()).get()) { + // the expr evaluates to TRUE => we can directly return it + rectifiedPredicate = expr; + } else { + // the expr evaluates to FALSE 0> rectify to "NOT expr" + rectifiedPredicate = new SQLite3UnaryOperation(UnaryOperator.NOT, expr); + } + rectifiedPredicates.add(rectifiedPredicate); + return rectifiedPredicate; } // @@ -316,7 +322,7 @@ private SQLite3Expression generateWindowFunction(List columns, SQ private void appendFilter(List columns, StringBuilder sb) { sb.append(" FILTER (WHERE "); - sb.append(SQLite3Visitor.asString(generateWhereClauseThatContainsRowValue(columns, pivotRow))); + sb.append(SQLite3Visitor.asString(generateRectifiedExpression(columns, pivotRow))); sb.append(")"); } From af93a55176f216ac4de108ece359f00741f1adcd Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 00:01:07 +0200 Subject: [PATCH 0143/1284] [SQLite] Ensure that the PQS subquery string does not end with a semicolon --- .../sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 8dd7dfb3b..6e7c4a0ef 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -185,7 +185,11 @@ protected boolean isContainedIn(Query query) throws SQLException { StringBuilder sb2 = new StringBuilder(); addExpectedValues(sb2); sb.append(" INTERSECT SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions - sb.append(query.getQueryString()); + if (query.getQueryString().endsWith(";")) { + sb.append(query.getQueryString().substring(0, query.getQueryString().length() - 1)); + } else { + sb.append(query.getQueryString()); + } sb.append(")"); String resultingQueryString = sb.toString(); Query finalQuery = new QueryAdapter(resultingQueryString, query.getExpectedErrors()); From 4ba8cd757593af56a93e5e1dd05fac57d12b72a4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 00:17:32 +0200 Subject: [PATCH 0144/1284] [SQLite] Reliably exclude rowid columns in the PQS implementation --- .../sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java | 5 +++-- src/sqlancer/sqlite3/schema/SQLite3Schema.java | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 6e7c4a0ef..720851619 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -36,6 +36,7 @@ import sqlancer.sqlite3.ast.SQLite3WindowFunction; import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3RowValue; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; @@ -87,8 +88,8 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio // TODO: also implement a wild-card check (*) // filter out row ids from the select because the hinder the reduction process // once a bug is found - List columnsWithoutRowid = columns.stream().filter(c -> !c.getName().matches("rowid")) - .collect(Collectors.toList()); + List columnsWithoutRowid = columns.stream() + .filter(c -> !SQLite3Schema.ROWID_STRINGS.contains(c.getName())).collect(Collectors.toList()); fetchColumns = Randomly.nonEmptySubset(columnsWithoutRowid); colExpressions = new ArrayList<>(); List allTables = new ArrayList<>(); diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index d57cbac36..d595916a2 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -31,6 +31,11 @@ public class SQLite3Schema { + /** + * All possible aliases for the rowid column. + */ + public static final List ROWID_STRINGS = Collections + .unmodifiableList(Arrays.asList("rowid", "_rowid_", "oid")); private final List databaseTables; private final List indexNames; @@ -336,7 +341,7 @@ public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throw tableType.contentEquals("temp_table") ? TableKind.TEMP : TableKind.MAIN, withoutRowid, nrRows, isView, isVirtual, isReadOnly); if (isRowIdTable(withoutRowid, isView, isVirtual)) { - String rowId = Randomly.fromOptions("rowid", "_rowid_", "oid"); + String rowId = Randomly.fromList(ROWID_STRINGS); SQLite3Column rowid = new SQLite3Column(rowId, SQLite3DataType.INT, true, null, true); t.addRowid(rowid); rowid.setTable(t); From 0254bf169125d156a1a24f879e73a4411fd3b4f8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 11:45:00 +0200 Subject: [PATCH 0145/1284] [SQLite] Apply various minor PQS refactorings --- .../SQLite3PivotedQuerySynthesisOracle.java | 122 +++++++++++------- 1 file changed, 74 insertions(+), 48 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 720851619..e596ed6cc 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -11,6 +11,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.StateToReproduce.OracleRunReproductionState; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; @@ -46,15 +47,16 @@ public class SQLite3PivotedQuerySynthesisOracle extends PivotedQuerySynthesisBase { private List fetchColumns; - private List colExpressions; + private List pivotRowExpression; + private OracleRunReproductionState localState; - public SQLite3PivotedQuerySynthesisOracle(SQLite3GlobalState globalState) throws SQLException { + public SQLite3PivotedQuerySynthesisOracle(SQLite3GlobalState globalState) { super(globalState); } @Override public Query getQueryThatContainsAtLeastOneRow() throws SQLException { - SQLite3Select selectStatement = getQuery(globalState); + SQLite3Select selectStatement = getQuery(); SQLite3ToStringVisitor visitor = new SQLite3ToStringVisitor(); visitor.visit(selectStatement); String queryString = visitor.get(); @@ -62,13 +64,15 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { return new QueryAdapter(queryString, errors); } - public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLException { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } + public SQLite3Select getQuery() throws SQLException { + assert !globalState.getSchema().getDatabaseTables().isEmpty(); + localState = globalState.getState().getLocalState(); + assert localState != null; SQLite3Tables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); List tables = randomFromTables.getTables(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + globalState.getState().queryTargetedTablesString = randomFromTables.tableNamesAsString(); SQLite3Select selectStatement = new SQLite3Select(); selectStatement.setSelectType(Randomly.fromOptions(SQLite3Select.SelectType.values())); @@ -78,10 +82,8 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio columns.add(t.getRowid()); } } - pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); List joinStatements = getJoinStatements(globalState, tables, columns); - selectStatement.setJoinClauses(joinStatements); selectStatement.setFromTables(SQLite3Common.getTableRefs(tables, globalState.getSchema())); @@ -91,41 +93,14 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio List columnsWithoutRowid = columns.stream() .filter(c -> !SQLite3Schema.ROWID_STRINGS.contains(c.getName())).collect(Collectors.toList()); fetchColumns = Randomly.nonEmptySubset(columnsWithoutRowid); - colExpressions = new ArrayList<>(); List allTables = new ArrayList<>(); allTables.addAll(tables); allTables.addAll(joinStatements.stream().map(join -> join.getTable()).collect(Collectors.toList())); boolean allTablesContainOneRow = allTables.stream().allMatch(t -> t.getNrRows() == 1); - for (SQLite3Column c : fetchColumns) { - SQLite3Expression colName = new SQLite3ColumnName(c, pivotRow.getValues().get(c)); - if (allTablesContainOneRow && Randomly.getBoolean()) { - boolean generateDistinct = Randomly.getBoolean(); - if (generateDistinct) { - colName = new SQLite3Distinct(colName); - } - SQLite3AggregateFunction aggFunc = SQLite3AggregateFunction.getRandom(c.getType()); - colName = new SQLite3Aggregate(Arrays.asList(colName), aggFunc); - if (Randomly.getBoolean() && !generateDistinct) { - colName = generateWindowFunction(columns, colName, true); - } - errors.add("second argument to nth_value must be a positive integer"); - } - if (Randomly.getBoolean()) { - SQLite3Expression randomExpression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) - .generateResultKnownExpression(); - colExpressions.add(randomExpression); - } else { - colExpressions.add(colName); - } - } - if (Randomly.getBoolean() && allTablesContainOneRow) { - SQLite3WindowFunction windowFunction = SQLite3WindowFunction.getRandom(columnsWithoutRowid, globalState); - SQLite3Expression windowExpr = generateWindowFunction(columnsWithoutRowid, windowFunction, false); - colExpressions.add(windowExpr); - } - selectStatement.setFetchColumns(colExpressions); - globalState.getState().queryTargetedColumnsString = fetchColumns.stream().map(c -> c.getFullQualifiedName()) - .collect(Collectors.joining(", ")); + pivotRowExpression = getColExpressions(allTablesContainOneRow, columns, columnsWithoutRowid); + selectStatement.setFetchColumns(pivotRowExpression); + localState.log("queryTargetedColumnsString: " + + fetchColumns.stream().map(c -> c.getFullQualifiedName()).collect(Collectors.joining(", "))); SQLite3Expression whereClause = generateRectifiedExpression(columns, pivotRow); selectStatement.setWhereClause(whereClause); List groupByClause = generateGroupByClause(columns, pivotRow, allTablesContainOneRow); @@ -137,8 +112,8 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio SQLite3Expression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } - List orderBy = new SQLite3ExpressionGenerator(globalState).setColumns(columns) - .generateOrderBys(); + /* PQS does not check for ordering, so we can generate any ORDER BY clause */ + List orderBy = new SQLite3ExpressionGenerator(globalState).generateOrderBys(); selectStatement.setOrderByExpressions(orderBy); if (!groupByClause.isEmpty() && Randomly.getBoolean()) { SQLite3Expression randomExpression = SQLite3Common.getTrueExpression(columns, globalState); @@ -166,6 +141,52 @@ private List getJoinStatements(SQLite3GlobalState globalState, List getColExpressions(boolean allTablesContainOneRow, List columns, + List columnsWithoutRowid) { + List colExpressions = new ArrayList<>(); + + for (SQLite3Column c : fetchColumns) { + SQLite3Expression colName = new SQLite3ColumnName(c, pivotRow.getValues().get(c)); + if (allTablesContainOneRow && Randomly.getBoolean()) { + + /* + * PQS cannot detect omitted or incorrectly-fetched duplicate rows, so we can generate DISTINCT + * statements + */ + boolean generateDistinct = Randomly.getBooleanWithRatherLowProbability(); + if (generateDistinct) { + colName = new SQLite3Distinct(colName); + } + + SQLite3AggregateFunction aggFunc = SQLite3AggregateFunction.getRandom(c.getType()); + colName = new SQLite3Aggregate(Arrays.asList(colName), aggFunc); + if (Randomly.getBoolean() && !generateDistinct) { + colName = generateWindowFunction(columns, colName, true); + } + errors.add("second argument to nth_value must be a positive integer"); + } + if (Randomly.getBoolean()) { + SQLite3Expression randomExpression; + randomExpression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) + .generateResultKnownExpression(); + colExpressions.add(randomExpression); + } else { + colExpressions.add(colName); + } + } + if (allTablesContainOneRow) { + SQLite3WindowFunction windowFunction = SQLite3WindowFunction.getRandom(columnsWithoutRowid, globalState); + SQLite3Expression windowExpr = generateWindowFunction(columnsWithoutRowid, windowFunction, false); + colExpressions.add(windowExpr); + } + for (SQLite3Expression expr : colExpressions) { + if (expr.getExpectedValue() == null) { + throw new IgnoreMeException(); // TODO: aggregates + } + } + return colExpressions; + } + private SQLite3Expression generateOffset() { if (Randomly.getBoolean()) { // OFFSET 0 @@ -182,9 +203,11 @@ protected boolean isContainedIn(Query query) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("SELECT "); - addExpectedValues(sb); - StringBuilder sb2 = new StringBuilder(); - addExpectedValues(sb2); + String checkForContainmentValues = getGeneralizedPivotRowValues(); + sb.append(checkForContainmentValues); + globalState.getState().getLocalState() + .log("-- we expect the following expression to be contained in the result set: " + + checkForContainmentValues); sb.append(" INTERSECT SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions if (query.getQueryString().endsWith(";")) { sb.append(query.getQueryString().substring(0, query.getQueryString().length() - 1)); @@ -193,6 +216,7 @@ protected boolean isContainedIn(Query query) throws SQLException { } sb.append(")"); String resultingQueryString = sb.toString(); + globalState.getState().getLocalState().log(resultingQueryString); Query finalQuery = new QueryAdapter(resultingQueryString, query.getExpectedErrors()); try (ResultSet result = createStatement.executeQuery(finalQuery.getQueryString())) { boolean isContainedIn = !result.isClosed(); @@ -207,14 +231,16 @@ protected boolean isContainedIn(Query query) throws SQLException { } } - private void addExpectedValues(StringBuilder sb) { - for (int i = 0; i < colExpressions.size(); i++) { + private String getGeneralizedPivotRowValues() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < pivotRowExpression.size(); i++) { if (i != 0) { sb.append(", "); } - SQLite3Constant expectedValue = colExpressions.get(i).getExpectedValue(); + SQLite3Constant expectedValue = pivotRowExpression.get(i).getExpectedValue(); sb.append(SQLite3Visitor.asString(expectedValue)); } + return sb.toString(); } private SQLite3Expression generateLimit(long l) { From a9abf279a1f8efd09a889e048d10ba0c8b71f399 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 11:56:11 +0200 Subject: [PATCH 0146/1284] [SQLite] Add a PQS option to test aggregate functions --- src/sqlancer/MainOptions.java | 7 +++++++ .../oracle/SQLite3PivotedQuerySynthesisOracle.java | 9 +++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index e76595d5d..f5f9340a4 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -65,6 +65,9 @@ public class MainOptions { @Parameter(names = "--print-succeeding-statements", description = "Print statements that are successfully processed by the DBMS to stdout (not yet implemented for all oracles)", arity = 1) private boolean printSucceedingStatements; // NOPMD + @Parameter(names = "--pqs-test-aggregates", description = "Partially test aggregate functions when all tables contain only a single row.") + private boolean testAggregateFunctions; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -146,4 +149,8 @@ public long getRandomSeed() { return randomSeed; } + public boolean testAggregateFunctionsPQS() { + return testAggregateFunctions; + } + } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index e596ed6cc..6b2bd5049 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -97,7 +97,8 @@ public SQLite3Select getQuery() throws SQLException { allTables.addAll(tables); allTables.addAll(joinStatements.stream().map(join -> join.getTable()).collect(Collectors.toList())); boolean allTablesContainOneRow = allTables.stream().allMatch(t -> t.getNrRows() == 1); - pivotRowExpression = getColExpressions(allTablesContainOneRow, columns, columnsWithoutRowid); + boolean testAggregateFunctions = allTablesContainOneRow && globalState.getOptions().testAggregateFunctionsPQS(); + pivotRowExpression = getColExpressions(testAggregateFunctions, columns, columnsWithoutRowid); selectStatement.setFetchColumns(pivotRowExpression); localState.log("queryTargetedColumnsString: " + fetchColumns.stream().map(c -> c.getFullQualifiedName()).collect(Collectors.joining(", "))); @@ -141,13 +142,13 @@ private List getJoinStatements(SQLite3GlobalState globalState, List getColExpressions(boolean allTablesContainOneRow, List columns, + private List getColExpressions(boolean testAggregateFunctions, List columns, List columnsWithoutRowid) { List colExpressions = new ArrayList<>(); for (SQLite3Column c : fetchColumns) { SQLite3Expression colName = new SQLite3ColumnName(c, pivotRow.getValues().get(c)); - if (allTablesContainOneRow && Randomly.getBoolean()) { + if (testAggregateFunctions && Randomly.getBoolean()) { /* * PQS cannot detect omitted or incorrectly-fetched duplicate rows, so we can generate DISTINCT @@ -174,7 +175,7 @@ private List getColExpressions(boolean allTablesContainOneRow colExpressions.add(colName); } } - if (allTablesContainOneRow) { + if (testAggregateFunctions) { SQLite3WindowFunction windowFunction = SQLite3WindowFunction.getRandom(columnsWithoutRowid, globalState); SQLite3Expression windowExpr = generateWindowFunction(columnsWithoutRowid, windowFunction, false); colExpressions.add(windowExpr); From 84faa7e67404737216dd976fbcc563018b8431f8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 12:41:39 +0200 Subject: [PATCH 0147/1284] [Postgres] Use the abstract schema class --- .../common/schema/AbstractSchema.java | 14 ++++++ src/sqlancer/postgres/PostgresSchema.java | 48 ++----------------- 2 files changed, 19 insertions(+), 43 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java index f0905df03..842932833 100644 --- a/src/sqlancer/common/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -2,9 +2,11 @@ import java.util.Collections; import java.util.List; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; public class AbstractSchema> { @@ -33,6 +35,14 @@ public A getRandomTable(Predicate predicate) { return Randomly.fromList(getDatabaseTables().stream().filter(predicate).collect(Collectors.toList())); } + public A getRandomTableOrBailout(Function f) { + List relevantTables = databaseTables.stream().filter(t -> f.apply(t)).collect(Collectors.toList()); + if (relevantTables.isEmpty()) { + throw new IgnoreMeException(); + } + return Randomly.fromList(relevantTables); + } + public List getDatabaseTables() { return databaseTables; } @@ -41,6 +51,10 @@ public List getDatabaseTablesRandomSubsetNotEmpty() { return Randomly.nonEmptySubset(databaseTables); } + public A getDatabaseTable(String name) { + return databaseTables.stream().filter(t -> t.getName().equals(name)).findAny().orElse(null); + } + public String getFreeIndexName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index add3fde39..3a028dace 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -7,26 +7,23 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; +import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; import sqlancer.postgres.ast.PostgresConstant; -public class PostgresSchema { +public class PostgresSchema extends AbstractSchema { - private final List databaseTables; private final String databaseName; public enum PostgresDataType { @@ -306,51 +303,16 @@ protected static List getTableColumns(Connection con, String tab } public PostgresSchema(List databaseTables, String databaseName) { - this.databaseTables = Collections.unmodifiableList(databaseTables); + super(databaseTables); this.databaseName = databaseName; } - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - for (PostgresTable t : getDatabaseTables()) { - sb.append(t); - sb.append("\n"); - } - return sb.toString(); - } - - public PostgresTable getRandomTable() { - return Randomly.fromList(getDatabaseTables()); - } - public PostgresTables getRandomTableNonEmptyTables() { - return new PostgresTables(Randomly.nonEmptySubset(databaseTables)); - } - - public List getDatabaseTables() { - return databaseTables; - } - - public PostgresTable getDatabaseTable(String name) { - return databaseTables.stream().filter(t -> t.getName().equals(name)).findAny().orElse(null); - } - - public List getDatabaseTablesRandomSubsetNotEmpty() { - return Randomly.nonEmptySubset(databaseTables); + return new PostgresTables(Randomly.nonEmptySubset(getDatabaseTables())); } public String getDatabaseName() { return databaseName; } - public PostgresTable getRandomTable(Function f) { - List relevantTables = databaseTables.stream().filter(t -> f.apply(t)) - .collect(Collectors.toList()); - if (relevantTables.isEmpty()) { - throw new IgnoreMeException(); - } - return Randomly.fromList(relevantTables); - } - } From 29d2052aa43a0f8c6962c3602d4ce2b65343da39 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 12:50:16 +0200 Subject: [PATCH 0148/1284] [SQLite] Use the abstract schema class --- .../common/schema/AbstractSchema.java | 36 ++++++++ .../gen/dml/SQLite3UpdateGenerator.java | 2 +- .../sqlite3/schema/SQLite3Schema.java | 82 +++---------------- 3 files changed, 48 insertions(+), 72 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java index 842932833..d2ef2d51d 100644 --- a/src/sqlancer/common/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -31,6 +31,14 @@ public A getRandomTable() { return Randomly.fromList(getDatabaseTables()); } + public A getRandomTableOrBailout() { + if (databaseTables.isEmpty()) { + throw new IgnoreMeException(); + } else { + return Randomly.fromList(getDatabaseTables()); + } + } + public A getRandomTable(Predicate predicate) { return Randomly.fromList(getDatabaseTables().stream().filter(predicate).collect(Collectors.toList())); } @@ -47,6 +55,10 @@ public List getDatabaseTables() { return databaseTables; } + public List getTables(Predicate predicate) { + return databaseTables.stream().filter(predicate).collect(Collectors.toList()); + } + public List getDatabaseTablesRandomSubsetNotEmpty() { return Randomly.nonEmptySubset(databaseTables); } @@ -55,6 +67,30 @@ public A getDatabaseTable(String name) { return databaseTables.stream().filter(t -> t.getName().equals(name)).findAny().orElse(null); } + public List getViews() { + return databaseTables.stream().filter(t -> t.isView()).collect(Collectors.toList()); + } + + public List getDatabaseTablesWithoutViews() { + return databaseTables.stream().filter(t -> !t.isView()).collect(Collectors.toList()); + } + + public A getRandomViewOrBailout() { + if (getViews().isEmpty()) { + throw new IgnoreMeException(); + } else { + return Randomly.fromList(getViews()); + } + } + + public A getRandomTableNoViewOrBailout() { + List databaseTablesWithoutViews = getDatabaseTablesWithoutViews(); + if (databaseTablesWithoutViews.isEmpty()) { + throw new IgnoreMeException(); + } + return Randomly.fromList(databaseTablesWithoutViews); + } + public String getFreeIndexName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index bde23fb1d..5c7fa9bf6 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -29,7 +29,7 @@ public SQLite3UpdateGenerator(SQLite3GlobalState globalState, Randomly r) { public static Query updateRow(SQLite3GlobalState globalState) { SQLite3Table randomTableNoViewOrBailout = globalState.getSchema() - .getRandomTable(t -> !t.isView() && !t.isReadOnly()); + .getRandomTableOrBailout(t -> !t.isView() && !t.isReadOnly()); return updateRow(globalState, randomTableNoViewOrBailout); } diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index d595916a2..6b5197019 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -10,7 +10,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Predicate; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; @@ -19,6 +18,7 @@ import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; @@ -27,16 +27,16 @@ import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table.TableKind; -public class SQLite3Schema { +public class SQLite3Schema extends AbstractSchema { /** * All possible aliases for the rowid column. */ public static final List ROWID_STRINGS = Collections .unmodifiableList(Arrays.asList("rowid", "_rowid_", "oid")); - private final List databaseTables; private final List indexNames; public List getIndexNames() { @@ -51,14 +51,6 @@ public String getRandomIndexOrBailout() { } } - public SQLite3Table getRandomTableOrBailout() { - if (databaseTables.isEmpty()) { - throw new IgnoreMeException(); - } else { - return Randomly.fromList(getDatabaseTables()); - } - } - public static class SQLite3Column extends AbstractTableColumn { private final boolean isInteger; // "INTEGER" type, not "INT" @@ -176,7 +168,8 @@ public SQLite3RowValue getRandomRowValue(Connection con) throws SQLException { throw new IgnoreMeException(); } if (!randomRowValues.next()) { - throw new AssertionError("could not find random row! " + randomRow); + throw new IgnoreMeException(); + // throw new AssertionError("could not find random row! " + randomRow); } for (int i = 0; i < getColumns().size(); i++) { SQLite3Column column = getColumns().get(i); @@ -243,7 +236,7 @@ public boolean isSystemTable() { return getName().startsWith("sqlit"); } - public int getNrRows() { + public long getNrRows() { return nrRows; } @@ -266,8 +259,8 @@ public static class SQLite3RowValue extends AbstractRowValue databaseTables, List indexNames) { + super(databaseTables); this.indexNames = indexNames; - this.databaseTables = Collections.unmodifiableList(databaseTables); } @Override @@ -471,80 +464,27 @@ public static SQLite3DataType getColumnType(String columnTypeString) { return columnType; } - public SQLite3Table getRandomTable() { - return Randomly.fromList(getDatabaseTables()); - } - - public SQLite3Table getRandomTable(Predicate predicate) { - List collect = databaseTables.stream().filter(predicate).collect(Collectors.toList()); - if (collect.isEmpty()) { - throw new IgnoreMeException(); - } - return Randomly.fromList(collect); - } - - public List getTables(Predicate predicate) { - return databaseTables.stream().filter(predicate).collect(Collectors.toList()); - } - - public SQLite3Table getRandomTableOrBailout(Predicate predicate) { - List tables = databaseTables.stream().filter(predicate).collect(Collectors.toList()); - if (tables.isEmpty()) { - throw new IgnoreMeException(); - } else { - return Randomly.fromList(tables); - } - } - public SQLite3Table getRandomVirtualTable() { return getRandomTable(p -> p.isVirtual); } - public List getDatabaseTables() { - return databaseTables; - } - public SQLite3Tables getTables() { - return new SQLite3Tables(databaseTables); + return new SQLite3Tables(getDatabaseTables()); } public SQLite3Tables getRandomTableNonEmptyTables() { - if (databaseTables.isEmpty()) { + if (getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } - return new SQLite3Tables(Randomly.nonEmptySubset(databaseTables)); - } - - public SQLite3Table getRandomTableNoViewOrBailout() { - List databaseTablesWithoutViews = getDatabaseTablesWithoutViews(); - if (databaseTablesWithoutViews.isEmpty()) { - throw new IgnoreMeException(); - } - return Randomly.fromList(databaseTablesWithoutViews); + return new SQLite3Tables(Randomly.nonEmptySubset(getDatabaseTables())); } public SQLite3Table getRandomTableNoViewNoVirtualTable() { return Randomly.fromList(getDatabaseTablesWithoutViewsWithoutVirtualTables()); } - public List getDatabaseTablesWithoutViews() { - return databaseTables.stream().filter(t -> !t.isView()).collect(Collectors.toList()); - } - - public List getViews() { - return databaseTables.stream().filter(t -> t.isView()).collect(Collectors.toList()); - } - - public SQLite3Table getRandomViewOrBailout() { - if (getViews().isEmpty()) { - throw new IgnoreMeException(); - } else { - return Randomly.fromList(getViews()); - } - } - public List getDatabaseTablesWithoutViewsWithoutVirtualTables() { - return databaseTables.stream().filter(t -> !t.isView() && !t.isVirtual).collect(Collectors.toList()); + return getDatabaseTables().stream().filter(t -> !t.isView() && !t.isVirtual).collect(Collectors.toList()); } } From 7f53b9ef3c6222822b6afe6682c6d614fdc0cfb8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 13:03:28 +0200 Subject: [PATCH 0149/1284] [MariaDB] Use the abstract schema classes --- src/sqlancer/mariadb/MariaDBSchema.java | 177 ++---------------- .../mariadb/gen/MariaDBInsertGenerator.java | 4 +- 2 files changed, 19 insertions(+), 162 deletions(-) diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index a0f9b715e..dfefdcd6f 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -6,30 +6,30 @@ import java.sql.SQLIntegrityConstraintViolationException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import sqlancer.Randomly; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.MariaDBSchema.MariaDBTable.MariaDBEngine; -public class MariaDBSchema { +public class MariaDBSchema extends AbstractSchema { private static final int NR_SCHEMA_READ_TRIES = 10; - private final List databaseTables; public enum MariaDBDataType { INT, VARCHAR, REAL, BOOLEAN; } - public static class MariaDBColumn implements Comparable { + public static class MariaDBColumn extends AbstractTableColumn { - private final String name; - private final MariaDBDataType columnType; private final boolean isPrimaryKey; - private MariaDBTable table; private final int precision; public enum CollateSequence { @@ -41,44 +41,11 @@ public static CollateSequence random() { } public MariaDBColumn(String name, MariaDBDataType columnType, boolean isPrimaryKey, int precision) { - this.name = name; - this.columnType = columnType; + super(name, null, columnType); this.isPrimaryKey = isPrimaryKey; this.precision = precision; } - @Override - public String toString() { - return String.format("%s.%s: %s", table.getName(), name, columnType); - } - - @Override - public int hashCode() { - return name.hashCode() + 11 * columnType.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof MariaDBColumn)) { - return false; - } else { - MariaDBColumn c = (MariaDBColumn) obj; - return table.getName().contentEquals(getName()) && name.equals(c.name); - } - } - - public String getName() { - return name; - } - - public String getFullQualifiedName() { - return table.getName() + "." + getName(); - } - - public MariaDBDataType getColumnType() { - return columnType; - } - public int getPrecision() { return precision; } @@ -87,23 +54,6 @@ public boolean isPrimaryKey() { return isPrimaryKey; } - public void setTable(MariaDBTable t) { - this.table = t; - } - - public MariaDBTable getTable() { - return table; - } - - @Override - public int compareTo(MariaDBColumn o) { - if (o.getTable().equals(this.getTable())) { - return name.compareTo(o.getName()); - } else { - return o.getTable().compareTo(table); - } - } - } public static class MariaDBTables { @@ -163,7 +113,7 @@ private static MariaDBDataType getColumnType(String typeString) { } } - public static class MariaDBTable implements Comparable { + public static class MariaDBTable extends AbstractTable { public enum MariaDBEngine { @@ -189,99 +139,32 @@ public static MariaDBEngine getRandomEngine() { } - private final String tableName; - private final List columns; - private final List indexes; private final MariaDBEngine engine; public MariaDBTable(String tableName, List columns, List indexes, MariaDBEngine engine) { - this.tableName = tableName; - this.indexes = indexes; + super(tableName, columns, indexes, false); this.engine = engine; - this.columns = Collections.unmodifiableList(columns); - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - sb.append(tableName); - sb.append("\n"); - for (MariaDBColumn c : columns) { - sb.append("\t"); - sb.append(c); - sb.append("\n"); - } - return sb.toString(); - } - - public List getIndexes() { - return indexes; - } - - public String getName() { - return tableName; - } - - public List getColumns() { - return columns; - } - - public String getColumnsAsString() { - return columns.stream().map(c -> c.getName()).collect(Collectors.joining(", ")); - } - - public String getColumnsAsString(Function function) { - return columns.stream().map(function).collect(Collectors.joining(", ")); - } - - public MariaDBColumn getRandomColumn() { - return Randomly.fromList(columns); - } - - public boolean hasIndexes() { - return !indexes.isEmpty(); - } - - public MariaDBIndex getRandomIndex() { - return Randomly.fromList(indexes); - } - - @Override - public int compareTo(MariaDBTable o) { - return o.getName().compareTo(tableName); - } - - public List getRandomNonEmptyColumnSubset() { - return Randomly.nonEmptySubset(getColumns()); } public MariaDBEngine getEngine() { return engine; } - public boolean hasPrimaryKey() { - return columns.stream().anyMatch(c -> c.isPrimaryKey()); - } } - public static final class MariaDBIndex { - - private final String indexName; + public static final class MariaDBIndex extends TableIndex { private MariaDBIndex(String indexName) { - this.indexName = indexName; - } - - public static MariaDBIndex create(String indexName) { - return new MariaDBIndex(indexName); + super(indexName); } + @Override public String getIndexName() { - if (indexName.contentEquals("PRIMARY")) { + if (super.getIndexName().contentEquals("PRIMARY")) { return "`PRIMARY`"; } else { - return indexName; + return super.getIndexName(); } } @@ -328,7 +211,7 @@ private static List getIndexes(Connection con, String tableName, S databaseName, tableName))) { while (rs.next()) { String indexName = rs.getString("INDEX_NAME"); - indexes.add(MariaDBIndex.create(indexName)); + indexes.add(new MariaDBIndex(indexName)); } } } @@ -355,33 +238,7 @@ private static List getTableColumns(Connection con, String tableN } public MariaDBSchema(List databaseTables) { - this.databaseTables = Collections.unmodifiableList(databaseTables); - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - for (MariaDBTable t : getDatabaseTables()) { - sb.append(t); - sb.append("\n"); - } - return sb.toString(); - } - - public MariaDBTable getRandomTable() { - return Randomly.fromList(getDatabaseTables()); - } - - public MariaDBTables getRandomTableNonEmptyTables() { - return new MariaDBTables(Randomly.nonEmptySubset(databaseTables)); - } - - public List getDatabaseTables() { - return databaseTables; - } - - public List getDatabaseTablesRandomSubsetNotEmpty() { - return Randomly.nonEmptySubset(databaseTables); + super(databaseTables); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java index 903089d3a..b45b43afc 100644 --- a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java @@ -27,8 +27,8 @@ public static Query insert(MariaDBSchema s, Randomly r) { if (Randomly.getBooleanWithSmallProbability()) { sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); } else { - sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r, - randomTable.getColumns().get(i).getColumnType()))); + sb.append(MariaDBVisitor.asString( + MariaDBExpressionGenerator.getRandomConstant(r, randomTable.getColumns().get(i).getType()))); } } sb.append(")"); From d0b4218f941005143a9caa0ff34271fee8c919dd Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 14:13:45 +0200 Subject: [PATCH 0150/1284] Add a method to allow the oracle factory to indicate that each table must contains at least one row --- src/sqlancer/OracleFactory.java | 9 +++++++++ src/sqlancer/common/oracle/TestOracle.java | 4 ---- src/sqlancer/mysql/MySQLOptions.java | 14 ++++++++++++++ src/sqlancer/postgres/PostgresOptions.java | 5 +++++ src/sqlancer/sqlite3/SQLite3Options.java | 6 ++++++ 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/OracleFactory.java b/src/sqlancer/OracleFactory.java index 424e271be..5b64cc4c2 100644 --- a/src/sqlancer/OracleFactory.java +++ b/src/sqlancer/OracleFactory.java @@ -8,4 +8,13 @@ public interface OracleFactory> { TestOracle create(G globalState) throws SQLException; + /** + * Indicates whether the test oracle requires that all tables (including views) contain at least one row. + * + * @return whether the test oracle requires at least one row per table + */ + default boolean requiresAllTablesToContainRows() { + return false; + } + } diff --git a/src/sqlancer/common/oracle/TestOracle.java b/src/sqlancer/common/oracle/TestOracle.java index 4640b9185..04101b950 100644 --- a/src/sqlancer/common/oracle/TestOracle.java +++ b/src/sqlancer/common/oracle/TestOracle.java @@ -6,8 +6,4 @@ public interface TestOracle { void check() throws SQLException; - default boolean onlyWorksForNonEmptyTables() { - return false; - } - } diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 4e0f1fabf..3e4f755be 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -11,6 +11,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; +import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @Parameters @@ -28,6 +29,19 @@ public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLTLPWhereOracle(globalState); } + }, + PQS { + + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + } } diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 8fa2d190e..0e23fb710 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -46,6 +46,11 @@ public TestOracle create(PostgresGlobalState globalState) throws SQLException { public TestOracle create(PostgresGlobalState globalState) throws SQLException { return new PostgresPivotedQuerySynthesisOracle(globalState); } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } }, HAVING { diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 3e5d36ebf..e5be902a6 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -94,6 +94,12 @@ public enum SQLite3OracleFactory implements OracleFactory { public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3PivotedQuerySynthesisOracle(globalState); } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, NoREC { @Override From ed89ce224849939611139ead3fde0643469cce6c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 14:14:32 +0200 Subject: [PATCH 0151/1284] Add an user option to indicate that only databases with >0 rows per table should be tested --- src/sqlancer/MainOptions.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index f5f9340a4..a44f9d56f 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -65,6 +65,9 @@ public class MainOptions { @Parameter(names = "--print-succeeding-statements", description = "Print statements that are successfully processed by the DBMS to stdout (not yet implemented for all oracles)", arity = 1) private boolean printSucceedingStatements; // NOPMD + @Parameter(names = "--test-only-nonempty-tables", description = "Test only databases each of whose tables contain at least a single row", arity = 1) + private boolean testOnlyWithMoreThanZeroRows; // NOPMD + @Parameter(names = "--pqs-test-aggregates", description = "Partially test aggregate functions when all tables contain only a single row.") private boolean testAggregateFunctions; // NOPMD @@ -153,4 +156,8 @@ public boolean testAggregateFunctionsPQS() { return testAggregateFunctions; } + public boolean testOnlyWithMoreThanZeroRows() { + return testOnlyWithMoreThanZeroRows; + } + } From f329cb9e1f8711bc81da21bee2b79d98ef5daf54 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 14:16:07 +0200 Subject: [PATCH 0152/1284] Specify that the schema in the GlobalState class should extend the AbstractSchema class --- src/sqlancer/GlobalState.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 89c36f269..7255facb6 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -7,6 +7,7 @@ import sqlancer.Main.StateLogger; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractSchema; /** * Represents a global state that is valid for a testing session on a given database. @@ -16,7 +17,7 @@ * @param * the schema parameter */ -public abstract class GlobalState, S> { +public abstract class GlobalState, S extends AbstractSchema> { private Connection con; private Randomly r; From 1fdbaa014e6aad46b80e75ed7bedb9093d159ef8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 14:17:06 +0200 Subject: [PATCH 0153/1284] Add a general method in AbstractTable to obtain a table's row count --- src/sqlancer/common/schema/AbstractTable.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 594bde47f..dc7f2574f 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -1,19 +1,26 @@ package sqlancer.common.schema; +import java.sql.SQLException; import java.util.Collections; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; +import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; public class AbstractTable, I extends TableIndex> implements Comparable> { + private static final int NO_ROW_COUNT_AVAILABLE = -1; private final String name; private final List columns; private final List indexes; private final boolean isView; + private long rowCount = NO_ROW_COUNT_AVAILABLE; public AbstractTable(String name, List columns, List indexes, boolean isView) { this.name = name; @@ -96,4 +103,22 @@ public String getFreeColumnName() { } + public long getNrRows(GlobalState globalState) { + if (rowCount == NO_ROW_COUNT_AVAILABLE) { + QueryAdapter q = new QueryAdapter("SELECT COUNT(*) FROM " + name); + try (SQLancerResultSet query = q.executeAndGet(globalState)) { + if (query == null) { + throw new IgnoreMeException(); + } + query.next(); + rowCount = query.getLong(1); + return rowCount; + } catch (SQLException e) { + // an exception might be expected, for example, when invalid view is created + throw new IgnoreMeException(); + } + } else { + return rowCount; + } + } } From 381944524cfc69390f8a8a76fa11372467602565 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 14:21:46 +0200 Subject: [PATCH 0154/1284] Test only non-empty databases when the test oracle or user requires --- src/sqlancer/ProviderAdapter.java | 7 +++++++ src/sqlancer/common/schema/AbstractSchema.java | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 5584efc4e..6435e8c74 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -62,6 +62,13 @@ public void generateAndTestDatabase(G globalState) throws SQLException { protected TestOracle getTestOracle(G globalState) throws SQLException { List> testOracleFactory = globalState.getDmbsSpecificOptions() .getTestOracleFactory(); + boolean testOracleRequiresMoreThanZeroRows = testOracleFactory.stream() + .anyMatch(p -> p.requiresAllTablesToContainRows()); + boolean userRequiresMoreThanZeroRows = globalState.getOptions().testOnlyWithMoreThanZeroRows(); + boolean checkZeroRows = testOracleRequiresMoreThanZeroRows || userRequiresMoreThanZeroRows; + if (checkZeroRows && globalState.getSchema().containsTableWithZeroRows(globalState)) { + throw new IgnoreMeException(); + } if (testOracleFactory.size() == 1) { return testOracleFactory.get(0).create(globalState); } else { diff --git a/src/sqlancer/common/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java index d2ef2d51d..d3f671ca0 100644 --- a/src/sqlancer/common/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -6,6 +6,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -136,7 +137,10 @@ public String getFreeViewName() { return tableName; } } while (true); + } + public boolean containsTableWithZeroRows(GlobalState globalState) { + return databaseTables.stream().anyMatch(t -> t.getNrRows(globalState) == 0); } } From 87b06146b24c54133680df48f56da05af4eeea9a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 14:23:00 +0200 Subject: [PATCH 0155/1284] [SQLite3] Remove the obsolete method for obtaining the row count --- src/sqlancer/sqlite3/SQLite3Provider.java | 2 +- .../SQLite3PivotedQuerySynthesisOracle.java | 2 +- .../sqlite3/schema/SQLite3Schema.java | 34 ++----------------- 3 files changed, 4 insertions(+), 34 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 8a10dba7d..c7157164a 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -102,7 +102,7 @@ public boolean canBeRetried() { CREATE_TRIGGER(SQLite3CreateTriggerGenerator::create), // MANIPULATE_STAT_TABLE((g) -> { List columns = new ArrayList<>(); - SQLite3Table t = new SQLite3Table("sqlite_stat1", columns, TableKind.MAIN, false, 1, false, false, false); + SQLite3Table t = new SQLite3Table("sqlite_stat1", columns, TableKind.MAIN, false, false, false, false); if (Randomly.getBoolean()) { return SQLite3DeleteGenerator.deleteContent(g, t); } else { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 6b2bd5049..c086100e3 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -96,7 +96,7 @@ public SQLite3Select getQuery() throws SQLException { List allTables = new ArrayList<>(); allTables.addAll(tables); allTables.addAll(joinStatements.stream().map(join -> join.getTable()).collect(Collectors.toList())); - boolean allTablesContainOneRow = allTables.stream().allMatch(t -> t.getNrRows() == 1); + boolean allTablesContainOneRow = allTables.stream().allMatch(t -> t.getNrRows(globalState) == 1); boolean testAggregateFunctions = allTablesContainOneRow && globalState.getOptions().testAggregateFunctionsPQS(); pivotRowExpression = getColExpressions(testAggregateFunctions, columns, columnsWithoutRowid); selectStatement.setFetchColumns(pivotRowExpression); diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 6b5197019..ad8f0d62a 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -14,16 +14,12 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.QueryAdapter; -import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; -import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; @@ -198,18 +194,16 @@ public enum TableKind { private final TableKind tableType; private SQLite3Column rowid; private final boolean withoutRowid; - private final int nrRows; private final boolean isVirtual; private final boolean isReadOnly; public SQLite3Table(String tableName, List columns, TableKind tableType, boolean withoutRowid, - int nrRows, boolean isView, boolean isVirtual, boolean isReadOnly) { + boolean isView, boolean isVirtual, boolean isReadOnly) { super(tableName, columns, Collections.emptyList(), isView); this.tableType = tableType; this.withoutRowid = withoutRowid; this.isVirtual = isVirtual; this.isReadOnly = isReadOnly; - this.nrRows = nrRows; } public boolean hasWithoutRowid() { @@ -236,10 +230,6 @@ public boolean isSystemTable() { return getName().startsWith("sqlit"); } - public long getNrRows() { - return nrRows; - } - public boolean isTemp() { return tableType == TableKind.TEMP; } @@ -273,25 +263,6 @@ public String toString() { return sb.toString(); } - public static int getNrRows(SQLite3GlobalState globalState, String table) throws SQLException { - String string = "SELECT COUNT(*) FROM " + table; - ExpectedErrors errors = new ExpectedErrors(); - errors.add("ORDER BY term out of range"); - errors.addAll(Arrays.asList("second argument to nth_value must be a positive integer", - "ON clause references tables to its right", "no such table", "no query solution", "no such index", - "GROUP BY term", "is circularly defined", "misuse of aggregate", "no such column", - "misuse of window function", "table does not support scanning")); - SQLite3Errors.addExpectedExpressionErrors(errors); - QueryAdapter q = new QueryAdapter(string, errors); - try (SQLancerResultSet query = q.executeAndGet(globalState)) { - if (query == null) { - throw new IgnoreMeException(); - } - query.next(); - return query.getInt(1); - } - } - public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throws SQLException { List databaseTables = new ArrayList<>(); List indexNames = new ArrayList<>(); @@ -329,10 +300,9 @@ public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throw boolean isDbStatsTable = sqlString.contains("using dbstat"); List databaseColumns = getTableColumns(con, tableName, sqlString, isView, isDbStatsTable); - int nrRows = getNrRows(globalState, tableName); SQLite3Table t = new SQLite3Table(tableName, databaseColumns, tableType.contentEquals("temp_table") ? TableKind.TEMP : TableKind.MAIN, withoutRowid, - nrRows, isView, isVirtual, isReadOnly); + isView, isVirtual, isReadOnly); if (isRowIdTable(withoutRowid, isView, isVirtual)) { String rowId = Randomly.fromList(ROWID_STRINGS); SQLite3Column rowid = new SQLite3Column(rowId, SQLite3DataType.INT, true, null, true); From cc915472e0cf701610b9fccba12a94132fd20147 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 23 Aug 2020 16:45:29 +0200 Subject: [PATCH 0156/1284] Provide multiple random string generators --- src/sqlancer/Main.java | 1 + src/sqlancer/MainOptions.java | 9 ++ src/sqlancer/Randomly.java | 169 +++++++++++++++++++++++----------- 3 files changed, 124 insertions(+), 55 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 9d9c001af..4380d4a01 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -421,6 +421,7 @@ public static int executeMain(String... args) throws AssertionError { return options.getErrorExitCode(); } + Randomly.initialize(options); if (options.printProgressInformation()) { startProgressMonitor(); if (options.printProgressSummary()) { diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index a44f9d56f..7843c8ac4 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -3,6 +3,8 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import sqlancer.Randomly.StringGenerationStrategy; + @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { @@ -71,6 +73,9 @@ public class MainOptions { @Parameter(names = "--pqs-test-aggregates", description = "Partially test aggregate functions when all tables contain only a single row.") private boolean testAggregateFunctions; // NOPMD + @Parameter(names = "--random-string-generation", description = "Select the random-string eneration approach") + private StringGenerationStrategy randomStringGenerationStrategy = StringGenerationStrategy.SOPHISTICATED; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -160,4 +165,8 @@ public boolean testOnlyWithMoreThanZeroRows() { return testOnlyWithMoreThanZeroRows; } + public StringGenerationStrategy getRandomStringGenerationStrategy() { + return randomStringGenerationStrategy; + } + } diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index ada420a3b..e1e45a859 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -12,11 +12,11 @@ public final class Randomly { private static final boolean USE_CACHING = true; private static final int CACHE_SIZE = 100; + static StringGenerationStrategy stringGenerationStrategy = StringGenerationStrategy.SOPHISTICATED; private final List cachedLongs = new ArrayList<>(); private final List cachedStrings = new ArrayList<>(); private final List cachedDoubles = new ArrayList<>(); private final List cachedBytes = new ArrayList<>(); - private static final String ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzöß!#<>/.,~-+'*()[]{} ^*?%_\t\n\r|&\\"; private Supplier provider; private static final ThreadLocal THREAD_RANDOM = new ThreadLocal<>(); @@ -76,19 +76,7 @@ private String getFromStringCache() { if (Randomly.getBoolean()) { return randomString; } else { - if (Randomly.getBoolean()) { - return randomString.toLowerCase(); - } else if (Randomly.getBoolean()) { - return randomString.toUpperCase(); - } else { - char[] chars = randomString.toCharArray(); - if (chars.length != 0) { - for (int i = 0; i < Randomly.smallNumber(); i++) { - chars[getInteger(0, chars.length)] = ALPHABET.charAt(getInteger(0, ALPHABET.length())); - } - } - return new String(chars); - } + return stringGenerationStrategy.transformCachedString(this, randomString); } } else { return null; @@ -200,59 +188,126 @@ public long getInteger() { } } - public String getString() { - if (smallBiasProbability()) { - return Randomly.fromOptions("TRUE", "FALSE", "0.0", "-0.0", "1e500", "-1e500"); - } - if (cacheProbability()) { - String s = getFromStringCache(); - if (s != null) { - return s; + public enum StringGenerationStrategy { + + NUMERIC { + @Override + public String getString(Randomly r) { + return getStringOfAlphabet(r, "0123456789"); } - } - int n = ALPHABET.length(); + }, + ALPHANUMERIC { - StringBuilder sb = new StringBuilder(); + @Override + public String getString(Randomly r) { + return getStringOfAlphabet(r, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); - int chars; - if (Randomly.getBoolean()) { - chars = Randomly.smallNumber(); - } else { - chars = getInteger(0, 30); - } - for (int i = 0; i < chars; i++) { - if (Randomly.getBooleanWithRatherLowProbability()) { - char val = (char) getInteger(); - if (val != 0) { - sb.append(val); + } + + }, + SOPHISTICATED { + + private static final String ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzöß!#<>/.,~-+'*()[]{} ^*?%_\t\n\r|&\\"; + + @Override + public String getString(Randomly r) { + if (smallBiasProbability()) { + return Randomly.fromOptions("TRUE", "FALSE", "0.0", "-0.0", "1e500", "-1e500"); } - } else { - sb.append(ALPHABET.charAt(getNextInt(0, n))); + if (cacheProbability()) { + String s = r.getFromStringCache(); + if (s != null) { + return s; + } + } + + int n = ALPHABET.length(); + + StringBuilder sb = new StringBuilder(); + + int chars = getStringLength(r); + for (int i = 0; i < chars; i++) { + if (Randomly.getBooleanWithRatherLowProbability()) { + char val = (char) r.getInteger(); + if (val != 0) { + sb.append(val); + } + } else { + sb.append(ALPHABET.charAt(getNextInt(0, n))); + } + } + while (Randomly.getBooleanWithSmallProbability()) { + String[][] pairs = { { "{", "}" }, { "[", "]" }, { "(", ")" } }; + int idx = (int) Randomly.getNotCachedInteger(0, pairs.length); + int left = (int) Randomly.getNotCachedInteger(0, sb.length() + 1); + sb.insert(left, pairs[idx][0]); + int right = (int) Randomly.getNotCachedInteger(left + 1, sb.length() + 1); + sb.insert(right, pairs[idx][1]); + } + if (r.provider != null) { + while (Randomly.getBooleanWithSmallProbability()) { + if (sb.length() == 0) { + sb.append(r.provider.get()); + } else { + sb.insert((int) Randomly.getNotCachedInteger(0, sb.length()), r.provider.get()); + } + } + } + + String s = sb.toString(); + + r.addToCache(s); + return s; } - } - while (Randomly.getBooleanWithSmallProbability()) { - String[][] pairs = { { "{", "}" }, { "[", "]" }, { "(", ")" } }; - int idx = (int) Randomly.getNotCachedInteger(0, pairs.length); - int left = (int) Randomly.getNotCachedInteger(0, sb.length() + 1); - sb.insert(left, pairs[idx][0]); - int right = (int) Randomly.getNotCachedInteger(left + 1, sb.length() + 1); - sb.insert(right, pairs[idx][1]); - } - if (provider != null) { - while (Randomly.getBooleanWithSmallProbability()) { - if (sb.length() == 0) { - sb.append(provider.get()); + + public String transformCachedString(Randomly r, String randomString) { + if (Randomly.getBoolean()) { + return randomString.toLowerCase(); + } else if (Randomly.getBoolean()) { + return randomString.toUpperCase(); } else { - sb.insert((int) Randomly.getNotCachedInteger(0, sb.length()), provider.get()); + char[] chars = randomString.toCharArray(); + if (chars.length != 0) { + for (int i = 0; i < Randomly.smallNumber(); i++) { + chars[r.getInteger(0, chars.length)] = ALPHABET.charAt(r.getInteger(0, ALPHABET.length())); + } + } + return new String(chars); } } + + }; + + private static int getStringLength(Randomly r) { + int chars; + if (Randomly.getBoolean()) { + chars = Randomly.smallNumber(); + } else { + chars = r.getInteger(0, 30); + } + return chars; } - String s = sb.toString(); + private static String getStringOfAlphabet(Randomly r, String alphabet) { + int chars = getStringLength(r); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < chars; i++) { + sb.append(alphabet.charAt(getNextInt(0, alphabet.length()))); + } + return sb.toString(); + } - addToCache(s); - return s; + public abstract String getString(Randomly r); + + public String transformCachedString(Randomly r, String s) { + return s; + } + + } + + public String getString() { + return stringGenerationStrategy.getString(this); } public byte[] getBytes() { @@ -435,4 +490,8 @@ public long getSeed() { return seed; } + public static void initialize(MainOptions options) { + stringGenerationStrategy = options.getRandomStringGenerationStrategy(); + } + } From e5cf23fc41d5e05868c2c7a0a64620825fad3fd4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 11:14:31 +0200 Subject: [PATCH 0157/1284] Provide an option to the maximum length of string constants and set it to a lower value --- src/sqlancer/MainOptions.java | 7 +++++++ src/sqlancer/Randomly.java | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 7843c8ac4..472540991 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -76,6 +76,9 @@ public class MainOptions { @Parameter(names = "--random-string-generation", description = "Select the random-string eneration approach") private StringGenerationStrategy randomStringGenerationStrategy = StringGenerationStrategy.SOPHISTICATED; // NOPMD + @Parameter(names = "--string-constant-max-length", description = "Specify the maximum-length of generated string constants") + private int maxStringConstantLength = 10; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -169,4 +172,8 @@ public StringGenerationStrategy getRandomStringGenerationStrategy() { return randomStringGenerationStrategy; } + public int getMaxStringConstantLength() { + return maxStringConstantLength; + } + } diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index e1e45a859..6df6848ab 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -12,7 +12,9 @@ public final class Randomly { private static final boolean USE_CACHING = true; private static final int CACHE_SIZE = 100; - static StringGenerationStrategy stringGenerationStrategy = StringGenerationStrategy.SOPHISTICATED; + private static StringGenerationStrategy stringGenerationStrategy = StringGenerationStrategy.SOPHISTICATED; + private static int maxStringLength = 10; + private final List cachedLongs = new ArrayList<>(); private final List cachedStrings = new ArrayList<>(); private final List cachedDoubles = new ArrayList<>(); @@ -284,7 +286,7 @@ private static int getStringLength(Randomly r) { if (Randomly.getBoolean()) { chars = Randomly.smallNumber(); } else { - chars = r.getInteger(0, 30); + chars = r.getInteger(0, maxStringLength); } return chars; } @@ -492,6 +494,7 @@ public long getSeed() { public static void initialize(MainOptions options) { stringGenerationStrategy = options.getRandomStringGenerationStrategy(); + maxStringLength = options.getMaxStringConstantLength(); } } From e3286bb25fa4cabc41e4e74a5abfc3855dfbd8e6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 13:16:00 +0200 Subject: [PATCH 0158/1284] Temporarily use OpenJDK8 due to Travis issues --- .travis.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1701898c2..9132ee046 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ branches: matrix: include: - name: MariaDB - jdk : oraclejdk11 + jdk : openjdk8 before_install: - sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 - sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main' @@ -27,14 +27,14 @@ matrix: - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test - name : MySQL - jdk : oraclejdk11 + jdk : openjdk8 script: - sudo apt-get update && sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 - sudo apt-get remove mysql-* && wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && yes | sudo dpkg -i *.deb - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - MYSQL_AVAILABLE=true mvn -Dtest=TestMySQL test - name: CockroachDB - jdk : oraclejdk11 + jdk : openjdk8 before_install: - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.2.linux-amd64.tgz | tar xvz - cd cockroach-v20.1.2.linux-amd64/ && ./cockroach start-single-node --insecure & @@ -43,7 +43,7 @@ matrix: script: - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - name: TiDB - jdk : oraclejdk11 + jdk : openjdk8 services: - docker before_install: @@ -54,11 +54,11 @@ matrix: script: - TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test - name: SQLite3 - jdk : oraclejdk11 + jdk : openjdk8 script: - mvn -Dtest=TestSQLite3 test - name: DuckDB - jdk : oraclejdk11 + jdk : openjdk8 script: - mvn -Dtest=TestDuckDB test - name: DuckDB (Java 8) @@ -66,15 +66,15 @@ matrix: script: - mvn -Dtest=TestDuckDB test - name: DuckDB (java 13) - jdk : openjdk13 + jdk : openjdk8 script: - mvn -Dtest=TestDuckDB test - name: Misc - jdk : oraclejdk11 + jdk : openjdk8 script: - mvn '-Dtest=!sqlancer.dbms.**' test - name: PostgreSQL - jdk : oraclejdk11 + jdk : openjdk8 before_install: - sudo apt-get update - sudo apt-get --yes remove postgresql\* @@ -96,7 +96,7 @@ matrix: script: - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test - name: Citus (PostgreSQL extension) - jdk : oraclejdk11 + jdk : openjdk8 before_install: - sudo apt-get update - sudo apt-get install -y postgresql-server-dev-12 postgresql-12 locales @@ -133,7 +133,7 @@ matrix: script: - CITUS_AVAILABLE=true mvn -Dtest=TestCitus test - name: ClickHouse - jdk : oraclejdk11 + jdk : openjdk8 services: - docker before_install: From eaf44aba61e7f2c87030fd7bfc74c291999343db Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 13:42:10 +0200 Subject: [PATCH 0159/1284] [CockroachDB] Work around newly-appearing error --- src/sqlancer/cockroachdb/CockroachDBProvider.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 587c77d23..d171ebfbd 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -259,6 +259,12 @@ public Connection createDatabase(CockroachDBGlobalState globalState) throws SQLE globalState.getState().logStatement("USE " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); + } catch (SQLException e) { + if (e.getMessage().contains("ERROR: invalid interleave backreference")) { + throw new IgnoreMeException(); // TODO: investigate + } else { + throw e; + } } try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); From 17a18225012a95f04bad27e7ab64ac525da50eb4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 13:42:31 +0200 Subject: [PATCH 0160/1284] [MySQL] Appear newly-appearing error --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 8a6ee5482..9f4414180 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -94,6 +94,7 @@ private void addCommonErrors(ExpectedErrors list) { list.add("not allowed type for this type of partitioning"); list.add("doesn't support BLOB/TEXT columns"); list.add("A BLOB field is not allowed in partition function"); + list.add("Too many keys specified; max 1 keys allowed"); } private enum PartitionOptions { From 6de02b3db9b594380e5ea9aa45dcff5b62bce1d1 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 24 Aug 2020 10:15:31 +0300 Subject: [PATCH 0161/1284] Make Citus error logs reproduceable source files --- src/sqlancer/StateToReproduce.java | 9 +++++++++ src/sqlancer/citus/CitusProvider.java | 16 ++++------------ src/sqlancer/citus/CitusSchema.java | 3 --- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index a74c2d628..3ae4d0228 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -74,6 +74,15 @@ public List getStatements() { return Collections.unmodifiableList(statements); } + public void commentStatements() { + for (int i = 0; i < statements.size(); i ++) { + Query statement = statements.get(i); + String queryString = statement.getQueryString(); + String newQueryString = "-- " + queryString; + statements.set(i, new QueryAdapter(newQueryString)); + } + } + public long getSeedValue() { return seedValue; } diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 6925dedf3..553f1ca97 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -208,12 +208,6 @@ private static void distributeTable(List columns, String tableNa QueryAdapter query = new QueryAdapter(queryString, getCitusErrors()); globalState.executeStatement(query, "SELECT create_distributed_table(?, ?);", tableName, columnToDistribute.getName()); - // distribution column cannot take NULL value - // TODO: find a way to protect from SQL injection without '' around string input - query = new QueryAdapter( - "ALTER TABLE " + tableName + " ALTER COLUMN " + columnToDistribute.getName() + " SET NOT NULL;", - getCitusErrors()); - globalState.executeStatement(query); } } @@ -272,12 +266,12 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl } } for (PostgresColumn c : columnConstraints.keySet()) { - // TODO: check if table and column constraint sets are equal? but then it's O(N) instead of O(1) + // check if all table contraints are included in column constraints, i.e. column eligible to distribute if (tableConstraints.size() == columnConstraints.get(c).size()) { columns.add(c); } } - // TODO: figure out how to use EXCLUDE + // TODO: diffferent behavior for EXCLUDE constraint? } distributeTable(columns, tableName, globalState); } @@ -352,10 +346,8 @@ private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes) throws SQLException { for (CitusWorkerNode w : citusWorkerNodes) { - // TODO: protect from sql injection - is it necessary though since these are read from the system? + // TODO: protect from sql injection String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; globalState.getState().logStatement(addWorkers); try (Statement s = con.createStatement()) { @@ -417,7 +409,6 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce prepareCitusWorkerNodes(globalState, citusWorkerNodes, databaseIndex, entryDatabaseName); // reconnect to coordinator node, test database - // TODO: better way of logging this globalState.getState().logStatement("\\q"); globalState.getState().logStatement(testURL); con = DriverManager.getConnection("jdbc:" + testURL, username, password); @@ -429,6 +420,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce con = DriverManager.getConnection("jdbc:" + testURL, username, password); ((CitusGlobalState) globalState) .setRepartition(((CitusOptions) globalState.getDmbsSpecificOptions()).repartition); + globalState.getState().commentStatements(); return con; } } diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index a902cb0b6..03d7669a6 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -14,7 +14,6 @@ public class CitusSchema extends PostgresSchema { public CitusSchema(List databaseTables, String databaseName) { - // FIXME: Will casting to PostgresTable lose CitusTable features? super(new ArrayList<>(databaseTables), databaseName); } @@ -71,8 +70,6 @@ public static CitusSchema fromConnection(Connection con, String databaseName) th if (rs.wasNull()) { colocationId = null; } - // FIXME: Are the CitusTable-specific features I'm adding going to persist after the function - // call? PostgresTable t = schema.getDatabaseTable(tableName); PostgresColumn distributionColumn = null; if (t == null) { From 449e3930dcc8bfb42b907849aee5b6edb9d99076 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 24 Aug 2020 11:55:03 +0300 Subject: [PATCH 0162/1284] Fix Postgres & Citus bug in not using any Postgres functions --- src/sqlancer/StateToReproduce.java | 2 +- src/sqlancer/citus/CitusProvider.java | 4 +--- src/sqlancer/postgres/PostgresProvider.java | 1 + 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 3ae4d0228..ea3523624 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -75,7 +75,7 @@ public List getStatements() { } public void commentStatements() { - for (int i = 0; i < statements.size(); i ++) { + for (int i = 0; i < statements.size(); i++) { Query statement = statements.get(i); String queryString = statement.getQueryString(); String newQueryString = "-- " + queryString; diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 553f1ca97..907a37e2f 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -271,14 +271,13 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl columns.add(c); } } - // TODO: diffferent behavior for EXCLUDE constraint? } distributeTable(columns, tableName, globalState); } @Override public void generateDatabase(PostgresGlobalState globalState) throws SQLException { - // TODO: function reading? add to Postgres implementation? + readFunctions(globalState); createTables(globalState, Randomly.fromOptions(4, 5, 6)); for (PostgresTable table : globalState.getSchema().getDatabaseTables()) { if (!(table.getTableType() == TableType.TEMPORARY || Randomly.getBooleanWithRatherLowProbability())) { @@ -378,7 +377,6 @@ private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes) throws SQLException { for (CitusWorkerNode w : citusWorkerNodes) { - // TODO: protect from sql injection String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; globalState.getState().logStatement(addWorkers); try (Statement s = con.createStatement()) { diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 2e23beea7..7100bdd5a 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -188,6 +188,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { @Override public void generateDatabase(PostgresGlobalState globalState) throws SQLException { + readFunctions(globalState); createTables(globalState, Randomly.fromOptions(4, 5, 6)); prepareTables(globalState); } From 21af9ba5cd4a8de4569a14781f7494dee6eb56f2 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 24 Aug 2020 12:45:21 +0300 Subject: [PATCH 0163/1284] Update Citus README with error log interpretation instructions --- docs/Citus README.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/Citus README.md b/docs/Citus README.md index 603d7bf36..bde865684 100644 --- a/docs/Citus README.md +++ b/docs/Citus README.md @@ -23,7 +23,27 @@ cd target java -jar SQLancer-0.0.1-SNAPSHOT.jar --num-threads 4 citus --oracle QUERY_PARTITIONING ``` -How to configure the run and how to interpret the output is explained in [SQLancer - Using SQLancer](https://github.com/sqlancer/sqlancer#using-sqlancer). +How to configure the run and how to find the output logs is explained in [SQLancer - Using SQLancer](https://github.com/sqlancer/sqlancer#using-sqlancer). + +## Interpreting output logs + +### Current logs + +If the `--log-each-select` option is enabled, each database being tested has a corresponding `-cur.log` file that is populated with all SQL statements sent to the database. + +### Error logs + +When a bug is found in a database being tested, a corresponding `.log` file is created and is populated with all SQL statements necessary to reproduce the bug. + +1. At the top of the file is the (commented-out) error message, which provides information about the panic error/logic bug detected. +2. Below that are (commented-out) lines that give more information about the specific thread being run, including the seed value (which can be passed in as a command line flag in a later run to reproduce the same thread run). +3. Then, the steps to create the Citus database cluster are provided as commented-out lines. (Following these steps are equivalent to running `citus_dev make XXX` or following the [Citus Docs instructions](https://docs.citusdata.com/en/v9.3/installation/single_machine_debian.html) for setting up a single-machine cluster.) +4. The rest of the file (not commented-out) contains the SQL statements that prepare the testing database. +5. If the bug detected is a logic bug (the error was raised by the TLP Oracle), then the pair of buggy SELECT statements whose result sets mismatch are also appended to the end of the file as commented-out lines. + +It is important to note that these `.log` files are valid sources of SQL commands that can be passed in with the `-f` flag to the `psql` command. As long as the empty database that the file is being passed into is created with Citus support and the proper worker nodes as described in step 3, this will reproduce the state that the testing database was in when the error was detected. Then, the SQL statement(s) that caused the error can be executed to reproduce the error itself. + +Once a bug is identified, it is also possible to check whether the bug is particular to Citus or was inherited from PostgreSQL, since Citus is a PostgreSQL extension. For this, a copy of the `.log` file can be made where all Citus-specific statements (distributing a table, creating a reference table etc.) are removed. Executing this file on an empty database would produce the “vanilla” state that the database would be in without any Citus functionalities. Then, the SQL statement(s) that caused the error can be executed here to check whether the error is reproduced in “vanilla” PostgreSQL as well. # Maintaining & Contributing From a9446d98aed2b8404890e555dd98aa883f0eb258 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Mon, 24 Aug 2020 14:51:50 +0300 Subject: [PATCH 0164/1284] Add error corresponding to duplicate constraints --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 590973505..3dfd8c707 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -253,6 +253,7 @@ public Query generate() { sb.append("ADD "); sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); PostgresCommon.addTableConstraint(sb, randomTable, globalState, errors); + errors.add("already exists"); errors.add("multiple primary keys for table"); errors.add("could not create unique index"); errors.add("contains null values"); @@ -283,6 +284,7 @@ public Query generate() { sb.append("ADD "); sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); + errors.add("already exists"); errors.add("not valid"); sb.append(" USING INDEX "); sb.append(randomTable.getRandomIndex().getIndexName()); From 89efb6b315b274bd142764e25e9020ca43f32789 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 15:05:05 +0200 Subject: [PATCH 0165/1284] [SQLite] Do not set PRAGMA case_sensitive_like=ON when using PQS --- src/sqlancer/sqlite3/SQLite3Provider.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index c7157164a..4586c9b94 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -20,6 +20,7 @@ import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.QueryProvider; +import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; import sqlancer.sqlite3.gen.SQLite3Common; @@ -326,7 +327,8 @@ private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws SQ if (!Randomly.getBooleanWithSmallProbability()) { pragmasToExecute.addAll(DEFAULT_PRAGMAS); } - if (Randomly.getBoolean() && !mustKnowResult) { + if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { + // the PQS implementation currently assumes the default behavior of LIKE pragmasToExecute.add("PRAGMA case_sensitive_like=ON;"); } if (Randomly.getBoolean()) { From 3541db38a6e083ea0dda0f9daf95ca346bc306db Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 15:06:42 +0200 Subject: [PATCH 0166/1284] Print the pivot row values as part of a comment --- src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index 0c39b75a1..279e31ae1 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -35,10 +35,11 @@ public final void check() throws SQLException { } protected void reportMissingPivotRow(Query query) { + globalState.getState().getLocalState().log("-- " + "pivot row values:\n"); String expectedPivotRowString = pivotRow.asStringGroupedByTables(); globalState.getState().getLocalState().log(expectedPivotRowString); - StringBuilder sb = new StringBuilder("-- rectified predicates:\n"); + StringBuilder sb = new StringBuilder("-- rectified predicates and their expected values:\n"); for (E rectifiedPredicate : rectifiedPredicates) { sb.append("--"); sb.append(asString(rectifiedPredicate).replace("\n", "\n-- ")); From 1517e0bc59fe86b1d6c97c2571fcf6627ecc8196 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 15:08:06 +0200 Subject: [PATCH 0167/1284] Ignore Throwables when trying to get the row count --- src/sqlancer/common/schema/AbstractTable.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index dc7f2574f..74f3ee7da 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -1,6 +1,5 @@ package sqlancer.common.schema; -import java.sql.SQLException; import java.util.Collections; import java.util.List; import java.util.function.Function; @@ -113,7 +112,7 @@ public long getNrRows(GlobalState globalState) { query.next(); rowCount = query.getLong(1); return rowCount; - } catch (SQLException e) { + } catch (Throwable t) { // an exception might be expected, for example, when invalid view is created throw new IgnoreMeException(); } From 73214e98d195e1c92db194bf88091cbf922f1f8e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 15:08:52 +0200 Subject: [PATCH 0168/1284] [SQLite] Fix generating HAVING clauses --- src/sqlancer/sqlite3/gen/SQLite3Common.java | 25 ---------------- .../SQLite3PivotedQuerySynthesisOracle.java | 29 +++++++++---------- 2 files changed, 14 insertions(+), 40 deletions(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3Common.java b/src/sqlancer/sqlite3/gen/SQLite3Common.java index 46de2c35f..97cb45787 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3Common.java +++ b/src/sqlancer/sqlite3/gen/SQLite3Common.java @@ -2,17 +2,11 @@ import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; -import sqlancer.sqlite3.ast.SQLite3Cast; -import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; import sqlancer.sqlite3.schema.SQLite3DataType; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -46,25 +40,6 @@ public static String getCheckConstraint(SQLite3GlobalState globalState, List columns, SQLite3GlobalState globalState) { - SQLite3Expression randomExpression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) - .generateExpression(); - SQLite3Constant expectedValue = randomExpression.getExpectedValue(); - if (expectedValue == null) { - throw new IgnoreMeException(); - } - Optional val = SQLite3Cast.isTrue(expectedValue); - if (!val.isPresent()) { - return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, randomExpression); - } - if (val.get()) { - return randomExpression; - } else { - return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_FALSE, randomExpression); - } - - } - // TODO: refactor others to use this method // https://www.sqlite.org/syntax/ordering-term.html public static String getOrderingTerm(List columns, SQLite3GlobalState globalState) { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index c086100e3..274becd3f 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -100,9 +100,7 @@ public SQLite3Select getQuery() throws SQLException { boolean testAggregateFunctions = allTablesContainOneRow && globalState.getOptions().testAggregateFunctionsPQS(); pivotRowExpression = getColExpressions(testAggregateFunctions, columns, columnsWithoutRowid); selectStatement.setFetchColumns(pivotRowExpression); - localState.log("queryTargetedColumnsString: " - + fetchColumns.stream().map(c -> c.getFullQualifiedName()).collect(Collectors.joining(", "))); - SQLite3Expression whereClause = generateRectifiedExpression(columns, pivotRow); + SQLite3Expression whereClause = generateRectifiedExpression(columns, pivotRow, false); selectStatement.setWhereClause(whereClause); List groupByClause = generateGroupByClause(columns, pivotRow, allTablesContainOneRow); selectStatement.setGroupByClause(groupByClause); @@ -117,12 +115,7 @@ public SQLite3Select getQuery() throws SQLException { List orderBy = new SQLite3ExpressionGenerator(globalState).generateOrderBys(); selectStatement.setOrderByExpressions(orderBy); if (!groupByClause.isEmpty() && Randomly.getBoolean()) { - SQLite3Expression randomExpression = SQLite3Common.getTrueExpression(columns, globalState); - if (Randomly.getBoolean()) { - SQLite3AggregateFunction aggFunc = SQLite3AggregateFunction.getRandom(); - randomExpression = new SQLite3Aggregate(Arrays.asList(randomExpression), aggFunc); - } - selectStatement.setHavingClause(randomExpression); + selectStatement.setHavingClause(generateRectifiedExpression(columns, pivotRow, true)); } return selectStatement; } @@ -136,7 +129,7 @@ private List getJoinStatements(SQLite3GlobalState globalState, List generateGroupByClause(List column * * @param columns * @param pivotRow + * @param allowAggregates * * @return an expression that evaluates to true. */ - private SQLite3Expression generateRectifiedExpression(List columns, SQLite3RowValue pivotRow) { - SQLite3Expression expr = new SQLite3ExpressionGenerator(globalState).setRowValue(pivotRow).setColumns(columns) - .generateResultKnownExpression(); + private SQLite3Expression generateRectifiedExpression(List columns, SQLite3RowValue pivotRow, + boolean allowAggregates) { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState).setRowValue(pivotRow) + .setColumns(columns); + if (allowAggregates) { + gen = gen.allowAggregateFunctions(); + } + SQLite3Expression expr = gen.generateResultKnownExpression(); SQLite3Expression rectifiedPredicate; if (expr.getExpectedValue().isNull()) { // the expr evaluates to NULL => rectify to "expr IS NULL" @@ -354,7 +353,7 @@ private SQLite3Expression generateWindowFunction(List columns, SQ private void appendFilter(List columns, StringBuilder sb) { sb.append(" FILTER (WHERE "); - sb.append(SQLite3Visitor.asString(generateRectifiedExpression(columns, pivotRow))); + sb.append(SQLite3Visitor.asString(generateRectifiedExpression(columns, pivotRow, false))); sb.append(")"); } @@ -379,7 +378,7 @@ private enum FrameSpec { @Override protected String asString(SQLite3Expression expr) { - return SQLite3Visitor.asString(expr); + return SQLite3Visitor.asExpectedValues(expr); } } From 93462d9646a846bae91142fb1977bbd5d6ed967e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 15:10:08 +0200 Subject: [PATCH 0169/1284] [SQLite] Ignore cases where we cannot get an expected result for a cast --- src/sqlancer/sqlite3/ast/SQLite3Expression.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index f7aad3fd7..b256ddf87 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1630,17 +1630,25 @@ public static ConstantTuple applyAffinities(TypeAffinity leftAffinity, TypeAffin if (leftAffinity.isNumeric() && (rightAffinity == TypeAffinity.TEXT || rightAffinity == TypeAffinity.BLOB || rightAffinity == TypeAffinity.NONE)) { right = right.applyNumericAffinity(); + assert right != null; } else if (rightAffinity.isNumeric() && (leftAffinity == TypeAffinity.TEXT || leftAffinity == TypeAffinity.BLOB || leftAffinity == TypeAffinity.NONE)) { left = left.applyNumericAffinity(); + assert left != null; } // If one operand has TEXT affinity and the other has no affinity, then TEXT // affinity is applied to the other operand. if (leftAffinity == TypeAffinity.TEXT && rightAffinity == TypeAffinity.NONE) { right = right.applyTextAffinity(); + if (right == null) { + throw new IgnoreMeException(); + } } else if (rightAffinity == TypeAffinity.TEXT && leftAffinity == TypeAffinity.NONE) { left = left.applyTextAffinity(); + if (left == null) { + throw new IgnoreMeException(); + } } return new ConstantTuple(left, right); } From e17f2167716bad223730b00f4b3cc6fa8787f4d8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 16:16:46 +0200 Subject: [PATCH 0170/1284] Add a random generation strategy that generates alphanumeric and special characters --- src/sqlancer/Randomly.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 6df6848ab..809d4403c 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -195,7 +195,7 @@ public enum StringGenerationStrategy { NUMERIC { @Override public String getString(Randomly r) { - return getStringOfAlphabet(r, "0123456789"); + return getStringOfAlphabet(r, NUMERIC_ALPHABET); } }, @@ -203,14 +203,23 @@ public String getString(Randomly r) { @Override public String getString(Randomly r) { - return getStringOfAlphabet(r, "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); + return getStringOfAlphabet(r, ALPHANUMERIC_ALPHABET); + + } + + }, + ALPHANUMERIC_SPECIALCHAR { + + @Override + public String getString(Randomly r) { + return getStringOfAlphabet(r, ALPHANUMERIC_SPECIALCHAR_ALPHABET); } }, SOPHISTICATED { - private static final String ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzöß!#<>/.,~-+'*()[]{} ^*?%_\t\n\r|&\\"; + private static final String ALPHABET = ALPHANUMERIC_SPECIALCHAR_ALPHABET; @Override public String getString(Randomly r) { @@ -281,6 +290,10 @@ public String transformCachedString(Randomly r, String randomString) { }; + private static final String ALPHANUMERIC_SPECIALCHAR_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzöß!#<>/.,~-+'*()[]{} ^*?%_\t\n\r|&\\"; + private static final String ALPHANUMERIC_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + private static final String NUMERIC_ALPHABET = "0123456789"; + private static int getStringLength(Randomly r) { int chars; if (Randomly.getBoolean()) { From 1028b502dfed4150eb12c325b14303a5594aead9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 16:26:14 +0200 Subject: [PATCH 0171/1284] Add options to control constant caching --- src/sqlancer/MainOptions.java | 14 ++++++++++++++ src/sqlancer/Randomly.java | 21 +++++++++++---------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 472540991..91a0d76c1 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -79,6 +79,12 @@ public class MainOptions { @Parameter(names = "--string-constant-max-length", description = "Specify the maximum-length of generated string constants") private int maxStringConstantLength = 10; // NOPMD + @Parameter(names = "--use-constant-caching", description = "Specifies whether constants should be cached and re-used with a certain probability") + private boolean useConstantCaching = true; // NOPMD + + @Parameter(names = "--constant-cache-size", description = "Specifies the size of the constant cache. This option only takes effect when constant caching is enabled") + private int constantCacheSize = 100; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -176,4 +182,12 @@ public int getMaxStringConstantLength() { return maxStringConstantLength; } + public boolean useConstantCaching() { + return useConstantCaching; + } + + public int getConstantCacheSize() { + return constantCacheSize; + } + } diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 809d4403c..14b747c14 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -9,11 +9,10 @@ public final class Randomly { - private static final boolean USE_CACHING = true; - private static final int CACHE_SIZE = 100; - private static StringGenerationStrategy stringGenerationStrategy = StringGenerationStrategy.SOPHISTICATED; private static int maxStringLength = 10; + private static boolean useCaching = true; + private static int cacheSize = 100; private final List cachedLongs = new ArrayList<>(); private final List cachedStrings = new ArrayList<>(); @@ -25,25 +24,25 @@ public final class Randomly { private long seed; private void addToCache(long val) { - if (USE_CACHING && cachedLongs.size() < CACHE_SIZE && !cachedLongs.contains(val)) { + if (useCaching && cachedLongs.size() < cacheSize && !cachedLongs.contains(val)) { cachedLongs.add(val); } } private void addToCache(double val) { - if (USE_CACHING && cachedDoubles.size() < CACHE_SIZE && !cachedDoubles.contains(val)) { + if (useCaching && cachedDoubles.size() < cacheSize && !cachedDoubles.contains(val)) { cachedDoubles.add(val); } } private void addToCache(String val) { - if (USE_CACHING && cachedStrings.size() < CACHE_SIZE && !cachedStrings.contains(val)) { + if (useCaching && cachedStrings.size() < cacheSize && !cachedStrings.contains(val)) { cachedStrings.add(val); } } private Long getFromLongCache() { - if (!USE_CACHING || cachedLongs.isEmpty()) { + if (!useCaching || cachedLongs.isEmpty()) { return null; } else { return Randomly.fromList(cachedLongs); @@ -51,7 +50,7 @@ private Long getFromLongCache() { } private Double getFromDoubleCache() { - if (!USE_CACHING) { + if (!useCaching) { return null; } if (Randomly.getBoolean() && !cachedLongs.isEmpty()) { @@ -64,7 +63,7 @@ private Double getFromDoubleCache() { } private String getFromStringCache() { - if (!USE_CACHING) { + if (!useCaching) { return null; } if (Randomly.getBoolean() && !cachedLongs.isEmpty()) { @@ -86,7 +85,7 @@ private String getFromStringCache() { } private static boolean cacheProbability() { - return USE_CACHING && getNextLong(0, 3) == 1; + return useCaching && getNextLong(0, 3) == 1; } // CACHING END @@ -508,6 +507,8 @@ public long getSeed() { public static void initialize(MainOptions options) { stringGenerationStrategy = options.getRandomStringGenerationStrategy(); maxStringLength = options.getMaxStringConstantLength(); + useCaching = options.useConstantCaching(); + cacheSize = options.getConstantCacheSize(); } } From aaf72b41e7706cc6d775fb8c60197dcb05acb9ad Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 24 Aug 2020 16:36:29 +0200 Subject: [PATCH 0172/1284] Add a --help/-h option that prints the available options and commands --- src/sqlancer/Main.java | 2 +- src/sqlancer/MainOptions.java | 7 +++++++ test/sqlancer/TestParameterFormat.java | 7 +++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 4380d4a01..c1f1bb56b 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -416,7 +416,7 @@ public static int executeMain(String... args) throws AssertionError { JCommander jc = commandBuilder.programName("SQLancer").build(); jc.parse(args); - if (jc.getParsedCommand() == null) { + if (jc.getParsedCommand() == null || options.isHelp()) { jc.usage(); return options.getErrorExitCode(); } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 91a0d76c1..9cda38f8b 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -8,6 +8,9 @@ @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { + @Parameter(names = { "--help", "-h" }, description = "Lists all supported options and commands", help = true) + private boolean help; // NOPMD + @Parameter(names = { "--num-threads" }, description = "How many threads should run concurrently to test separate databases") private int nrConcurrentThreads = 16; // NOPMD @@ -190,4 +193,8 @@ public int getConstantCacheSize() { return constantCacheSize; } + public boolean isHelp() { + return help; + } + } diff --git a/test/sqlancer/TestParameterFormat.java b/test/sqlancer/TestParameterFormat.java index fbdd3ad81..6d9522ca0 100644 --- a/test/sqlancer/TestParameterFormat.java +++ b/test/sqlancer/TestParameterFormat.java @@ -17,7 +17,7 @@ */ public class TestParameterFormat { - private final static String OPTION_REGEX = "--[a-z0-9-]*"; + private final static String OPTION_REGEX = "(-)?-[a-z0-9-]*"; @Test public void testOptionFormat() throws Exception { @@ -38,7 +38,10 @@ public void testOptionFormat() throws Exception { parameterDescriptions.addAll(command.getParameters()); } for (ParameterDescription parameter : parameterDescriptions) { - assertTrue(Pattern.matches(OPTION_REGEX, parameter.getNames()), parameter.getNames()); + String[] names = parameter.getNames().split(", "); + for (String name : names) { + assertTrue(Pattern.matches(OPTION_REGEX, name), name); + } } } From a2e2d4303655f26a04119f3ce7ac9df5519efc6c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 17:29:30 +0000 Subject: [PATCH 0173/1284] Bump plexus-compiler-eclipse from 2.8.7 to 2.8.8 Bumps plexus-compiler-eclipse from 2.8.7 to 2.8.8. Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7c77d81f4..dcbf1d49b 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.8.7 + 2.8.8 org.eclipse.jdt From 982c0701316110ee506ec5e3937030d34ac9ef26 Mon Sep 17 00:00:00 2001 From: nukoyluoglu Date: Tue, 25 Aug 2020 11:32:18 +0300 Subject: [PATCH 0174/1284] Add repartition flag detail to Citus README --- docs/Citus README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/Citus README.md b/docs/Citus README.md index bde865684..7bd3ffb49 100644 --- a/docs/Citus README.md +++ b/docs/Citus README.md @@ -25,6 +25,8 @@ java -jar SQLancer-0.0.1-SNAPSHOT.jar --num-threads 4 citus --oracle QUERY_PARTI How to configure the run and how to find the output logs is explained in [SQLancer - Using SQLancer](https://github.com/sqlancer/sqlancer#using-sqlancer). +The `--repartition` flag is a boolean optional argument specific to the Citus implementation (and therefore should be used after `citus` on the command line) that enables [repartition joins](https://docs.citusdata.com/en/v9.3/develop/api_guc.html?highlight=repartition%20join#citus-enable-repartitioned-insert-select-boolean). It is set to `true` by default. + ## Interpreting output logs ### Current logs @@ -62,7 +64,7 @@ The `CitusBugs` class in `CitusBugs.java` is an interface between [issues](https ### What to do: new bug found If the bug found is a panic error, i.e. NOT a logic bug (mismatch in result sets identified by the TLP Oracle), this error should be added to the `CitusBugs` class and the `addCitusErrors()` method. -1. Open an issue for the bug in the [Citus GitHub repository](https://github.com/citusdata/citus/issues). +1. Open an issue for the bug in the [Citus GitHub repository](https://github.com/citusdata/citus/issues?q=is%3Aissue+label%3Asqlancer+), and tag the issue with the `sqlancer` label. 2. Add a boolean variable associated with this issue to the `CitusBugs` class and set it to `true`. 3. Add the error message to the `addCitusErrors()` method wrapped inside an if-statement referring to the boolean created in the `CitusBugs` class. From 3a293f4e5cde23a84bf32830e776c74f427cf988 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 11:50:21 +0200 Subject: [PATCH 0175/1284] [Postgres] Use the ExpectedErrors class in the PQS implementation --- .../PostgresPivotedQuerySynthesisOracle.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index e9b54e8ba..0b85de1f6 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -26,6 +26,7 @@ import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; public class PostgresPivotedQuerySynthesisOracle @@ -39,6 +40,8 @@ public PostgresPivotedQuerySynthesisOracle(PostgresGlobalState globalState) thro super(globalState); options = globalState.getOptions(); logger = globalState.getLogger(); + PostgresCommon.addCommonExpressionErrors(errors); + PostgresCommon.addCommonFetchErrors(errors); } @Override @@ -153,20 +156,14 @@ protected boolean isContainedIn(Query query) throws SQLException { if (options.logEachSelect()) { logger.writeCurrent(resultingQueryString); } + globalState.getState().getLocalState().log(resultingQueryString); + QueryAdapter finalQuery = new QueryAdapter(resultingQueryString, errors); try (ResultSet result = createStatement.executeQuery(resultingQueryString)) { boolean isContainedIn = result.next(); createStatement.close(); return isContainedIn; } catch (PSQLException e) { - if (e.getMessage().contains("out of range") || e.getMessage().contains("cannot cast") - || e.getMessage().contains("invalid input syntax for ") || e.getMessage().contains("must be type") - || e.getMessage().contains("operator does not exist") - || e.getMessage().contains("Could not choose a best candidate function.") - || e.getMessage().contains("division by zero") - || e.getMessage().contains("zero raised to a negative power is undefined") - || e.getMessage().contains("canceling statement due to statement timeout") - || e.getMessage().contains("operator is not unique") - || e.getMessage().contains("could not determine which collation to use for string comparison")) { + if (finalQuery.getExpectedErrors().errorIsExpected(e.getMessage())) { return true; } else { throw e; From de41168a3c8b0d4c95f2c4992ebd7942938bcdb8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 11:51:27 +0200 Subject: [PATCH 0176/1284] [Postgres] Prevent name collisions in the PQS class --- .../PostgresPivotedQuerySynthesisOracle.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 0b85de1f6..bac64d78d 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -57,7 +57,8 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { selectStatement.setFromList(randomFromTables.getTables().stream().map(t -> new PostgresFromTable(t, false)) .collect(Collectors.toList())); selectStatement.setFetchColumns(fetchColumns.stream() - .map(c -> new PostgresColumnValue(c, pivotRow.getValues().get(c))).collect(Collectors.toList())); + .map(c -> new PostgresColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); PostgresExpression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); selectStatement.setWhereClause(whereClause); List groupByClause = generateGroupByClause(columns, pivotRow); @@ -96,6 +97,16 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { return new QueryAdapter(visitor.get()); } + /* + * Prevent name collisions by aliasing the column. + */ + private PostgresColumn getFetchValueAliasedColumn(PostgresColumn c) { + PostgresColumn aliasedColumn = new PostgresColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType()); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + private List generateGroupByClause(List columns, PostgresRowValue rw) { if (Randomly.getBoolean()) { return columns.stream().map(c -> PostgresColumnValue.create(c, rw.getValues().get(c))) @@ -134,7 +145,11 @@ protected boolean isContainedIn(Query query) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions - sb.append(query.getQueryString()); + if (query.getQueryString().endsWith(";")) { + sb.append(query.getQueryString().substring(0, query.getQueryString().length() - 1)); + } else { + sb.append(query.getQueryString()); + } sb.append(") as result WHERE "); int i = 0; for (PostgresColumn c : fetchColumns) { From 644725714a787e06a5dc82743e44485b078d2281 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 11:52:26 +0200 Subject: [PATCH 0177/1284] Add RANGE, MONEY, and BIT to the list of data types not supported by PQS --- src/sqlancer/postgres/PostgresSchema.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 3a028dace..860047adc 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -30,12 +30,15 @@ public enum PostgresDataType { INT, BOOLEAN, TEXT, DECIMAL, FLOAT, REAL, RANGE, MONEY, BIT, INET; public static PostgresDataType getRandomType() { - List dataTypes = Arrays.asList(values()); + List dataTypes = new ArrayList<>(Arrays.asList(values())); if (PostgresProvider.generateOnlyKnown) { dataTypes.remove(PostgresDataType.DECIMAL); dataTypes.remove(PostgresDataType.FLOAT); dataTypes.remove(PostgresDataType.REAL); dataTypes.remove(PostgresDataType.INET); + dataTypes.remove(PostgresDataType.RANGE); + dataTypes.remove(PostgresDataType.MONEY); + dataTypes.remove(PostgresDataType.BIT); } return Randomly.fromList(dataTypes); } From 205b3949b0b49f6d26a5ab657b2942d89445e369 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 12:24:36 +0200 Subject: [PATCH 0178/1284] [Postgres] Let getExpectedValue() return null when no PQS implementation is available --- .../ast/PostgresBinaryArithmeticOperation.java | 7 ++++--- .../ast/PostgresBinaryComparisonOperation.java | 7 ++++++- .../ast/PostgresBinaryLogicalOperation.java | 7 ++++++- .../postgres/ast/PostgresCastOperation.java | 6 +++++- .../postgres/ast/PostgresConcatOperation.java | 11 ++++++++--- src/sqlancer/postgres/ast/PostgresConstant.java | 9 ++++----- .../postgres/ast/PostgresExpression.java | 4 ++-- src/sqlancer/postgres/ast/PostgresFunction.java | 7 ++++++- .../postgres/ast/PostgresInOperation.java | 16 ++++++++++++---- .../postgres/ast/PostgresLikeOperation.java | 3 +++ .../postgres/ast/PostgresPostfixOperation.java | 6 +++++- .../postgres/ast/PostgresPrefixOperation.java | 12 ++++++++++-- 12 files changed, 71 insertions(+), 24 deletions(-) diff --git a/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java index 2c362efcd..a36173743 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java @@ -48,12 +48,10 @@ public PostgresConstant apply(PostgresConstant left, PostgresConstant right) { } }, - // TODO no implementation EXPONENTIATION("^") { @Override public PostgresConstant apply(PostgresConstant left, PostgresConstant right) { - // return applyBitOperation(left, right, (l, r) -> (long) Math.pow(l, r)); - throw new AssertionError(); + return null; } }; @@ -97,6 +95,9 @@ public PostgresBinaryArithmeticOperation(PostgresExpression left, PostgresExpres public PostgresConstant getExpectedValue() { PostgresConstant leftExpected = getLeft().getExpectedValue(); PostgresConstant rightExpected = getRight().getExpectedValue(); + if (leftExpected == null || rightExpected == null) { + return null; + } return getOp().apply(leftExpected, rightExpected); } diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 8f17be89c..95efe5b72 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -123,7 +123,12 @@ public PostgresBinaryComparisonOperation(PostgresExpression left, PostgresExpres @Override public PostgresConstant getExpectedValue() { - return getOp().getExpectedValue(getLeft().getExpectedValue(), getRight().getExpectedValue()); + PostgresConstant leftExpectedValue = getLeft().getExpectedValue(); + PostgresConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); } @Override diff --git a/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java index 5809ced09..9c104670e 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java @@ -77,7 +77,12 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - return getOp().apply(getLeft().getExpectedValue(), getRight().getExpectedValue()); + PostgresConstant leftExpectedValue = getLeft().getExpectedValue(); + PostgresConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); } } diff --git a/src/sqlancer/postgres/ast/PostgresCastOperation.java b/src/sqlancer/postgres/ast/PostgresCastOperation.java index dcd2d3e56..521b2cd97 100644 --- a/src/sqlancer/postgres/ast/PostgresCastOperation.java +++ b/src/sqlancer/postgres/ast/PostgresCastOperation.java @@ -23,7 +23,11 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - return expression.getExpectedValue().cast(type.getDataType()); + PostgresConstant expectedValue = expression.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type.getDataType()); } public PostgresExpression getExpression() { diff --git a/src/sqlancer/postgres/ast/PostgresConcatOperation.java b/src/sqlancer/postgres/ast/PostgresConcatOperation.java index b83633e1d..cbbe59085 100644 --- a/src/sqlancer/postgres/ast/PostgresConcatOperation.java +++ b/src/sqlancer/postgres/ast/PostgresConcatOperation.java @@ -16,11 +16,16 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - if (getLeft().getExpectedValue().isNull() || getRight().getExpectedValue().isNull()) { + PostgresConstant leftExpectedValue = getLeft().getExpectedValue(); + PostgresConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + if (leftExpectedValue.isNull() || rightExpectedValue.isNull()) { return PostgresConstant.createNullConstant(); } - String leftStr = getLeft().getExpectedValue().cast(PostgresDataType.TEXT).getUnquotedTextRepresentation(); - String rightStr = getRight().getExpectedValue().cast(PostgresDataType.TEXT).getUnquotedTextRepresentation(); + String leftStr = leftExpectedValue.cast(PostgresDataType.TEXT).getUnquotedTextRepresentation(); + String rightStr = rightExpectedValue.cast(PostgresDataType.TEXT).getUnquotedTextRepresentation(); return PostgresConstant.createTextConstant(leftStr + rightStr); } diff --git a/src/sqlancer/postgres/ast/PostgresConstant.java b/src/sqlancer/postgres/ast/PostgresConstant.java index d5bc3b671..b60e8a96b 100644 --- a/src/sqlancer/postgres/ast/PostgresConstant.java +++ b/src/sqlancer/postgres/ast/PostgresConstant.java @@ -75,7 +75,7 @@ public PostgresConstant cast(PostgresDataType type) { case TEXT: return PostgresConstant.createTextConstant(value ? "true" : "false"); default: - throw new AssertionError(); + return null; } } @@ -212,7 +212,7 @@ public PostgresConstant cast(PostgresDataType type) { case TEXT: return this; default: - throw new AssertionError(this); + return null; } } @@ -307,7 +307,7 @@ public PostgresConstant cast(PostgresDataType type) { case TEXT: return PostgresConstant.createTextConstant(String.valueOf(val)); default: - throw new AssertionError(type); + return null; } } @@ -406,8 +406,7 @@ protected PostgresConstant isLessThan(PostgresConstant rightVal) { @Override public PostgresConstant cast(PostgresDataType type) { - throw new AssertionError(); - + return null; } } diff --git a/src/sqlancer/postgres/ast/PostgresExpression.java b/src/sqlancer/postgres/ast/PostgresExpression.java index ed4516ffa..96ddfa433 100644 --- a/src/sqlancer/postgres/ast/PostgresExpression.java +++ b/src/sqlancer/postgres/ast/PostgresExpression.java @@ -5,10 +5,10 @@ public interface PostgresExpression { default PostgresDataType getExpressionType() { - throw new AssertionError("operator does not support PQS evaluation!"); + return null; } default PostgresConstant getExpectedValue() { - throw new AssertionError("operator does not support PQS evaluation!"); + return null; } } diff --git a/src/sqlancer/postgres/ast/PostgresFunction.java b/src/sqlancer/postgres/ast/PostgresFunction.java index 00c4c4810..24a1b3426 100644 --- a/src/sqlancer/postgres/ast/PostgresFunction.java +++ b/src/sqlancer/postgres/ast/PostgresFunction.java @@ -286,10 +286,15 @@ public boolean checkArguments(PostgresExpression... constants) { @Override public PostgresConstant getExpectedValue() { - assert functionWithKnownResult != null; + if (functionWithKnownResult == null) { + return null; + } PostgresConstant[] constants = new PostgresConstant[args.length]; for (int i = 0; i < constants.length; i++) { constants[i] = args[i].getExpectedValue(); + if (constants[i] == null) { + return null; + } } return functionWithKnownResult.apply(constants, args); } diff --git a/src/sqlancer/postgres/ast/PostgresInOperation.java b/src/sqlancer/postgres/ast/PostgresInOperation.java index 73bd8ec57..68f4b38cc 100644 --- a/src/sqlancer/postgres/ast/PostgresInOperation.java +++ b/src/sqlancer/postgres/ast/PostgresInOperation.java @@ -26,15 +26,23 @@ public List getListElements() { @Override public PostgresConstant getExpectedValue() { - if (expr.getExpectedValue().isNull()) { + PostgresConstant leftValue = expr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { return PostgresConstant.createNullConstant(); } boolean isNull = false; for (PostgresExpression expr : getListElements()) { - if (expr.getExpectedValue().isNull()) { + PostgresConstant rightExpectedValue = expr.getExpectedValue(); + if (rightExpectedValue == null) { + return null; + } + if (rightExpectedValue.isNull()) { isNull = true; - } else if (expr.getExpectedValue().isEquals(this.expr.getExpectedValue()).isBoolean() - && expr.getExpectedValue().isEquals(this.expr.getExpectedValue()).asBoolean()) { + } else if (rightExpectedValue.isEquals(this.expr.getExpectedValue()).isBoolean() + && rightExpectedValue.isEquals(this.expr.getExpectedValue()).asBoolean()) { return PostgresConstant.createBooleanConstant(isTrue); } } diff --git a/src/sqlancer/postgres/ast/PostgresLikeOperation.java b/src/sqlancer/postgres/ast/PostgresLikeOperation.java index 6971b1b11..8e80e4817 100644 --- a/src/sqlancer/postgres/ast/PostgresLikeOperation.java +++ b/src/sqlancer/postgres/ast/PostgresLikeOperation.java @@ -19,6 +19,9 @@ public PostgresDataType getExpressionType() { public PostgresConstant getExpectedValue() { PostgresConstant leftVal = getLeft().getExpectedValue(); PostgresConstant rightVal = getRight().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } if (leftVal.isNull() || rightVal.isNull()) { return PostgresConstant.createNullConstant(); } else { diff --git a/src/sqlancer/postgres/ast/PostgresPostfixOperation.java b/src/sqlancer/postgres/ast/PostgresPostfixOperation.java index 4eea1fe90..78c735030 100644 --- a/src/sqlancer/postgres/ast/PostgresPostfixOperation.java +++ b/src/sqlancer/postgres/ast/PostgresPostfixOperation.java @@ -129,7 +129,11 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - return op.apply(expr.getExpectedValue()); + PostgresConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.apply(expectedValue); } public String getOperatorTextRepresentation() { diff --git a/src/sqlancer/postgres/ast/PostgresPrefixOperation.java b/src/sqlancer/postgres/ast/PostgresPrefixOperation.java index f517c57f2..eefbcc47f 100644 --- a/src/sqlancer/postgres/ast/PostgresPrefixOperation.java +++ b/src/sqlancer/postgres/ast/PostgresPrefixOperation.java @@ -51,7 +51,11 @@ protected PostgresConstant getExpectedValue(PostgresConstant expectedValue) { // TODO throw new IgnoreMeException(); } - return PostgresConstant.createIntConstant(-expectedValue.asInt()); + try { + return PostgresConstant.createIntConstant(-expectedValue.asInt()); + } catch (UnsupportedOperationException e) { + return null; + } } }; @@ -90,7 +94,11 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - return op.getExpectedValue(expr.getExpectedValue()); + PostgresConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.getExpectedValue(expectedValue); } public PostgresDataType[] getInputDataTypes() { From 500fb9e10dc8b448c1c3024d4842211749597b8c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 13:07:06 +0200 Subject: [PATCH 0179/1284] [Postgres] Add a method to the expression generator to generate expressions with expected non-null values --- .../postgres/gen/PostgresExpressionGenerator.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index a1b105141..0c112d6b9 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -462,7 +462,7 @@ public static PostgresExpression generateConstant(Randomly r) { public static PostgresExpression generateTrueCondition(List columns, PostgresRowValue rw, PostgresGlobalState globalState) { PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) - .expectedResult().generateExpression(0, PostgresDataType.BOOLEAN); + .generateExpressionWithExpectedResult(PostgresDataType.BOOLEAN); if (expr.getExpectedValue().isNull()) { return PostgresPostfixOperation.create(expr, PostfixOperator.IS_NULL); } @@ -470,6 +470,17 @@ public static PostgresExpression generateTrueCondition(List colu ? PostfixOperator.IS_TRUE : PostfixOperator.IS_FALSE); } + private PostgresExpression generateExpressionWithExpectedResult(PostgresDataType type) { + this.expectedResult = true; + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns) + .setRowValue(rw); + PostgresExpression expr; + do { + expr = gen.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + public static PostgresExpression generateConstant(Randomly r, PostgresDataType type) { if (Randomly.getBooleanWithSmallProbability()) { return PostgresConstant.createNullConstant(); From dae8fb518bb820508c5f675d0ceaa24ce6b454e7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:50:08 +0200 Subject: [PATCH 0180/1284] [Postgres] Bailout when encountering an unknown type when selecting a pivot row --- src/sqlancer/postgres/PostgresSchema.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 860047adc..947e286f9 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -11,6 +11,9 @@ import java.util.List; import java.util.Map; +import org.postgresql.util.PSQLException; + +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; @@ -93,13 +96,15 @@ public PostgresRowValue getRandomRowValue(Connection con) throws SQLException { constant = PostgresConstant.createTextConstant(randomRowValues.getString(columnIndex)); break; default: - throw new AssertionError(column.getType()); + throw new IgnoreMeException(); } } values.put(column, constant); } assert !randomRowValues.next(); return new PostgresRowValue(this, values); + } catch (PSQLException e) { + throw new IgnoreMeException(); } } From 900ba793513d41c8f6c3748f207066ac77e7a8e9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:51:19 +0200 Subject: [PATCH 0181/1284] [Postgres] Set a flag when using PQS to indicate that only expressions with known result should be generated --- configs/spotbugs-exclude.xml | 5 +++++ src/sqlancer/postgres/PostgresProvider.java | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/configs/spotbugs-exclude.xml b/configs/spotbugs-exclude.xml index 1b146bf10..366e8ac55 100644 --- a/configs/spotbugs-exclude.xml +++ b/configs/spotbugs-exclude.xml @@ -7,4 +7,9 @@ + + + + + diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 7100bdd5a..52312699c 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -17,6 +17,7 @@ import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; import sqlancer.postgres.gen.PostgresAlterTableGenerator; import sqlancer.postgres.gen.PostgresAnalyzeGenerator; import sqlancer.postgres.gen.PostgresClusterGenerator; @@ -44,6 +45,9 @@ // IN public class PostgresProvider extends ProviderAdapter { + /** + * Generate only data types and expressions that are understood by PQS. + */ public static boolean generateOnlyKnown; private PostgresGlobalState globalState; @@ -195,6 +199,11 @@ public void generateDatabase(PostgresGlobalState globalState) throws SQLExceptio @Override public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { + if (globalState.getDmbsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == PostgresOracleFactory.PQS)) { + generateOnlyKnown = true; + } + username = globalState.getOptions().getUserName(); password = globalState.getOptions().getPassword(); entryPath = "/test"; From ec6d727053e34d2bd21035459892a5eaf475d627 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:52:05 +0200 Subject: [PATCH 0182/1284] [Postgres] Do not generate COLLATEs for table columns when using PQS --- src/sqlancer/postgres/gen/PostgresCommon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 111b60ae0..62a8a811b 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -151,7 +151,7 @@ public static boolean appendDataType(PostgresDataType type, StringBuilder sb, bo } else { sb.append("name"); } - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() && !PostgresProvider.generateOnlyKnown) { sb.append(" COLLATE "); sb.append('"'); sb.append(Randomly.fromList(opClasses)); From 39fc2c862eae29588ce2ab77f535fd6ccf89ebce Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:53:04 +0200 Subject: [PATCH 0183/1284] [Postgres] Do not generate size specifications for types when using PQS --- src/sqlancer/postgres/gen/PostgresExpressionGenerator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 0c112d6b9..4ca27dcff 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -333,7 +333,10 @@ private static PostgresCompoundDataType getCompoundDataType(PostgresDataType typ return PostgresCompoundDataType.create(type); case TEXT: // TODO case BIT: - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() || PostgresProvider.generateOnlyKnown /* + * The PQS implementation does not check for + * size specifications + */) { return PostgresCompoundDataType.create(type); } else { return PostgresCompoundDataType.create(type, (int) Randomly.getNotCachedInteger(1, 1000)); From 2ff65983da3a33bc462440cfa097e68614e97a6f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:53:47 +0200 Subject: [PATCH 0184/1284] [Postgres] Do not generate strings representing booleans when using PQS --- src/sqlancer/postgres/gen/PostgresExpressionGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 4ca27dcff..af46660e4 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -499,7 +499,7 @@ public static PostgresExpression generateConstant(Randomly r, PostgresDataType t return PostgresConstant.createIntConstant(r.getInteger()); } case BOOLEAN: - if (Randomly.getBooleanWithSmallProbability()) { + if (Randomly.getBooleanWithSmallProbability() && !PostgresProvider.generateOnlyKnown) { return PostgresConstant .createTextConstant(Randomly.fromOptions("TR", "TRUE", "FA", "FALSE", "0", "1", "ON", "off")); } else { From 23230a36ebf8162788ce4a8586cdfbbfb154223d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:55:06 +0200 Subject: [PATCH 0185/1284] [Postgres] Move the method for generating the rectified expression into the PQS implementation --- .../gen/PostgresExpressionGenerator.java | 13 +----------- .../PostgresPivotedQuerySynthesisOracle.java | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index af46660e4..f8c7bf539 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -462,18 +462,7 @@ public static PostgresExpression generateConstant(Randomly r) { return generateConstant(r, Randomly.fromOptions(PostgresDataType.values())); } - public static PostgresExpression generateTrueCondition(List columns, PostgresRowValue rw, - PostgresGlobalState globalState) { - PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) - .generateExpressionWithExpectedResult(PostgresDataType.BOOLEAN); - if (expr.getExpectedValue().isNull()) { - return PostgresPostfixOperation.create(expr, PostfixOperator.IS_NULL); - } - return PostgresPostfixOperation.create(expr, expr.getExpectedValue().cast(PostgresDataType.BOOLEAN).asBoolean() - ? PostfixOperator.IS_TRUE : PostfixOperator.IS_FALSE); - } - - private PostgresExpression generateExpressionWithExpectedResult(PostgresDataType type) { + public PostgresExpression generateExpressionWithExpectedResult(PostgresDataType type) { this.expectedResult = true; PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns) .setRowValue(rw); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index bac64d78d..d7cb2782a 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -17,6 +17,7 @@ import sqlancer.common.query.QueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresRowValue; import sqlancer.postgres.PostgresSchema.PostgresTables; import sqlancer.postgres.PostgresToStringVisitor; @@ -24,6 +25,8 @@ import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresPostfixOperation; +import sqlancer.postgres.ast.PostgresPostfixOperation.PostfixOperator; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.gen.PostgresCommon; @@ -97,6 +100,22 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { return new QueryAdapter(visitor.get()); } + public PostgresExpression generateTrueCondition(List columns, PostgresRowValue rw, + PostgresGlobalState globalState) { + PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) + .generateExpressionWithExpectedResult(PostgresDataType.BOOLEAN); + PostgresExpression result; + if (expr.getExpectedValue().isNull()) { + result = PostgresPostfixOperation.create(expr, PostfixOperator.IS_NULL); + } else { + result = PostgresPostfixOperation.create(expr, + expr.getExpectedValue().cast(PostgresDataType.BOOLEAN).asBoolean() ? PostfixOperator.IS_TRUE + : PostfixOperator.IS_FALSE); + } + rectifiedPredicates.add(result); + return result; + } + /* * Prevent name collisions by aliasing the column. */ @@ -135,7 +154,7 @@ private PostgresExpression generateOffset() { private PostgresExpression generateWhereClauseThatContainsRowValue(List columns, PostgresRowValue rw) { - return PostgresExpressionGenerator.generateTrueCondition(columns, rw, globalState); + return generateTrueCondition(columns, rw, globalState); } @Override From 1a73e597273858c9ea9604373d930c28dd36ece9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:57:01 +0200 Subject: [PATCH 0186/1284] [Postgres] Implement a method to visit PostgresBinaryLogicalOperations --- .../postgres/PostgresExpectedValueVisitor.java | 8 ++++++++ src/sqlancer/postgres/PostgresToStringVisitor.java | 12 ++++++++++++ src/sqlancer/postgres/PostgresVisitor.java | 5 +++++ 3 files changed, 25 insertions(+) diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index 549efdeec..4930b676a 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -2,6 +2,7 @@ import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresBetweenOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; import sqlancer.postgres.ast.PostgresColumnValue; @@ -161,4 +162,11 @@ public void visit(PostgresSubquery subquery) { print(subquery); } + @Override + public void visit(PostgresBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + } diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 121fa1426..70adf838c 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -7,6 +7,7 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresBetweenOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; import sqlancer.postgres.ast.PostgresColumnValue; @@ -337,4 +338,15 @@ public void visit(PostgresCollate op) { sb.append(")"); } + @Override + public void visit(PostgresBinaryLogicalOperation op) { + sb.append("("); + visit(op.getLeft()); + sb.append(")"); + sb.append(op.getOperatorRepresentation()); + sb.append("("); + visit(op.getRight()); + sb.append(")"); + } + } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 0cc1d53bc..54fb992ff 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -6,6 +6,7 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresBetweenOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; import sqlancer.postgres.ast.PostgresColumnValue; @@ -60,6 +61,8 @@ public interface PostgresVisitor { void visit(PostgresSubquery subquery); + void visit(PostgresBinaryLogicalOperation op); + default void visit(PostgresExpression expression) { if (expression instanceof PostgresConstant) { visit((PostgresConstant) expression); @@ -95,6 +98,8 @@ default void visit(PostgresExpression expression) { visit((PostgresFromTable) expression); } else if (expression instanceof PostgresSubquery) { visit((PostgresSubquery) expression); + } else if (expression instanceof PostgresBinaryLogicalOperation) { + visit((PostgresBinaryLogicalOperation) expression); } else { throw new AssertionError(expression); } From b46d39983aac7f62f27b1edb209d56a2a4bbaeaf Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:57:29 +0200 Subject: [PATCH 0187/1284] [Postgres] Address newly appearing expected errors --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 ++ src/sqlancer/postgres/gen/PostgresUpdateGenerator.java | 1 + 2 files changed, 3 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 9f474ceb9..8174d3852 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -52,6 +52,8 @@ public PostgresTableGenerator(String tableName, PostgresSchema newSchema, boolea errors.add("does not accept data type"); errors.add("but default expression is of type text"); errors.add("has pseudo-type unknown"); + errors.add("no collation was derived for partition key column"); + errors.add("inherits from generated column but specifies identity"); PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonTableErrors(errors); } diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index e184c144b..253a86c03 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -32,6 +32,7 @@ public static Query create(PostgresGlobalState globalState) { "View columns that are not columns of their base relation are not updatable"); errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced // table multiple times + errors.add("new row violates check option for view"); List columns = randomTable.getRandomNonEmptyColumnSubset(); PostgresCommon.addCommonInsertUpdateErrors(errors); From d4ae16d746524a0207703f5ca79fa147461b9748 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:58:10 +0200 Subject: [PATCH 0188/1284] Use the asExpectedValues() method when printing the rectified expression --- src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 2 +- .../postgres/oracle/PostgresPivotedQuerySynthesisOracle.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 0c0c0c1f9..661eaf97b 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -195,6 +195,6 @@ protected boolean isContainedIn(Query query) throws SQLException { @Override protected String asString(MySQLExpression expr) { - return MySQLVisitor.asString(expr); + return MySQLVisitor.asExpectedValues(expr); } } diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index d7cb2782a..aaf5f4f59 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -207,7 +207,7 @@ protected boolean isContainedIn(Query query) throws SQLException { @Override protected String asString(PostgresExpression expr) { - return PostgresVisitor.asString(expr); + return PostgresVisitor.asExpectedValues(expr); } } From cbda6793346d48ad326bafdb197b7b6f7f0a1f4f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 14:58:35 +0200 Subject: [PATCH 0189/1284] Check for invalid views in the ProviderAdapter base class --- src/sqlancer/ProviderAdapter.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 6435e8c74..772bd142a 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -7,6 +7,8 @@ import sqlancer.StateToReproduce.OracleRunReproductionState; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.QueryAdapter; +import sqlancer.common.schema.AbstractTable; public abstract class ProviderAdapter, O extends DBMSSpecificOptions>> implements DatabaseProvider { @@ -38,6 +40,7 @@ public Class getOptionClass() { public void generateAndTestDatabase(G globalState) throws SQLException { try { generateDatabase(globalState); + checkViewsAreValid(globalState); globalState.getManager().incrementCreateDatabase(); TestOracle oracle = getTestOracle(globalState); @@ -59,6 +62,18 @@ public void generateAndTestDatabase(G globalState) throws SQLException { } } + private void checkViewsAreValid(G globalState) { + List> views = globalState.getSchema().getViews(); + for (AbstractTable view : views) { + QueryAdapter q = new QueryAdapter("SELECT 1 FROM " + view.getName() + " LIMIT 1"); + try { + q.execute(globalState); + } catch (Throwable t) { + throw new IgnoreMeException(); + } + } + } + protected TestOracle getTestOracle(G globalState) throws SQLException { List> testOracleFactory = globalState.getDmbsSpecificOptions() .getTestOracleFactory(); From dc681278f611901b668c72d0b58523cb74215936 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 20:18:25 +0200 Subject: [PATCH 0190/1284] [MySQL] Return null for unknown expected values --- src/sqlancer/mysql/ast/MySQLBinaryLogicalOperation.java | 4 +++- src/sqlancer/mysql/ast/MySQLComputableFunction.java | 3 +++ src/sqlancer/mysql/ast/MySQLConstant.java | 7 ++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/ast/MySQLBinaryLogicalOperation.java b/src/sqlancer/mysql/ast/MySQLBinaryLogicalOperation.java index 4971ccc49..102e01b0d 100644 --- a/src/sqlancer/mysql/ast/MySQLBinaryLogicalOperation.java +++ b/src/sqlancer/mysql/ast/MySQLBinaryLogicalOperation.java @@ -101,7 +101,9 @@ public String getTextRepresentation() { public MySQLConstant getExpectedValue() { MySQLConstant leftExpected = left.getExpectedValue(); MySQLConstant rightExpected = right.getExpectedValue(); - + if (left.getExpectedValue() == null || right.getExpectedValue() == null) { + return null; + } return op.apply(leftExpected, rightExpected); } diff --git a/src/sqlancer/mysql/ast/MySQLComputableFunction.java b/src/sqlancer/mysql/ast/MySQLComputableFunction.java index 2adcae95b..8e790b462 100644 --- a/src/sqlancer/mysql/ast/MySQLComputableFunction.java +++ b/src/sqlancer/mysql/ast/MySQLComputableFunction.java @@ -209,6 +209,9 @@ public MySQLConstant getExpectedValue() { MySQLConstant[] constants = new MySQLConstant[args.length]; for (int i = 0; i < constants.length; i++) { constants[i] = args[i].getExpectedValue(); + if (constants[i].getExpectedValue() == null) { + return null; + } } return func.apply(constants, args); } diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index e04b54c40..5c639d721 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -30,8 +30,7 @@ private RuntimeException throwException() { @Override public MySQLConstant isEquals(MySQLConstant rightVal) { - throw throwException(); - + return null; } @Override @@ -199,7 +198,9 @@ protected MySQLConstant isLessThan(MySQLConstant rightVal) { return castAs(rightVal.isSigned() ? CastType.SIGNED : CastType.UNSIGNED).isLessThan(rightVal); } else if (rightVal.isString()) { // unexpected result for '-' < "!"; - // return MySQLConstant.createBoolean(value.compareToIgnoreCase(rightVal.getString()) < 0); + // return + // MySQLConstant.createBoolean(value.compareToIgnoreCase(rightVal.getString()) < + // 0); throw new IgnoreMeException(); } else { throw new AssertionError(rightVal); From f03cb0ea242af09f61e0ec7931b3aadf023e31cc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 20:19:23 +0200 Subject: [PATCH 0191/1284] [MySQL] Do not use FALSE/TRUE as synonyms for 0/1 since this does not work for PQS --- src/sqlancer/mysql/ast/MySQLConstant.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index 5c639d721..6988f74c0 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -218,16 +218,10 @@ public static class MySQLIntConstant extends MySQLConstant { public MySQLIntConstant(long value, boolean isSigned) { this.value = value; this.isSigned = isSigned; - if (value == 0 && Randomly.getBoolean()) { - stringRepresentation = "FALSE"; - } else if (value == 1 && Randomly.getBoolean()) { - stringRepresentation = "TRUE"; + if (isSigned) { + stringRepresentation = String.valueOf(value); } else { - if (isSigned) { - stringRepresentation = String.valueOf(value); - } else { - stringRepresentation = Long.toUnsignedString(value); - } + stringRepresentation = Long.toUnsignedString(value); } } From 3054a8ec180a3d88925fd8a089fb3e98f58b5fe6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 20:20:29 +0200 Subject: [PATCH 0192/1284] [MySQL] Add various improvemements and fixes to the PQS class --- .../MySQLPivotedQuerySynthesisOracle.java | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 661eaf97b..72d1f07bb 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -9,9 +9,9 @@ import sqlancer.Randomly; import sqlancer.common.oracle.PivotedQuerySynthesisBase; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; @@ -38,6 +38,8 @@ public class MySQLPivotedQuerySynthesisOracle public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { super(globalState); + MySQLErrors.addExpressionErrors(errors); + errors.add("in 'order clause'"); // e.g., Unknown column '2067708013' in 'order clause' } @Override @@ -104,7 +106,8 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { if (i++ != 0) { sb2.append(" AND "); } - sb2.append(c.getFullQualifiedName()); + sb2.append("ref"); + sb2.append(i - 1); if (pivotRow.getValues().get(c).isNull()) { sb2.append(" IS NULL"); } else { @@ -116,7 +119,7 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { MySQLToStringVisitor visitor = new MySQLToStringVisitor(); visitor.visit(selectStatement); - return new QueryAdapter(visitor.get(), ExpectedErrors.from("BIGINT value is out of range")); + return new QueryAdapter(visitor.get(), errors); } private List generateGroupByClause(List columns, MySQLRowValue rw) { @@ -149,13 +152,16 @@ private MySQLExpression generateWhereClauseThatContainsRowValue(List Date: Tue, 25 Aug 2020 20:23:36 +0200 Subject: [PATCH 0193/1284] [MySQL] When using PQS, only use the supported types --- src/sqlancer/mysql/MySQLGlobalState.java | 5 +++++ src/sqlancer/mysql/MySQLProvider.java | 5 ++--- src/sqlancer/mysql/MySQLSchema.java | 8 ++++++-- .../mysql/gen/MySQLExpressionGenerator.java | 15 +++++++++++++-- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 15 +++++++++------ 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index c464fb4de..5df20bb50 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import sqlancer.GlobalState; +import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; public class MySQLGlobalState extends GlobalState { @@ -12,4 +13,8 @@ protected void updateSchema() throws SQLException { setSchema(MySQLSchema.fromConnection(getConnection(), getDatabaseName())); } + public boolean usesPQS() { + return getDmbsSpecificOptions().oracles.stream().anyMatch(o -> o == MySQLOracleFactory.PQS); + } + } diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 4069b0aab..3eef6a739 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -56,7 +56,7 @@ enum Action implements AbstractAction { CREATE_TABLE((g) -> { // TODO refactor String tableName = SQLite3Common.createTableName(g.getSchema().getDatabaseTables().size()); - return MySQLTableGenerator.generate(tableName, g.getRandomly(), g.getSchema()); + return MySQLTableGenerator.generate(g, tableName); }), // DELETE(MySQLDeleteGenerator::delete), // DROP_INDEX(MySQLDropIndex::generate); @@ -134,10 +134,9 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { @Override public void generateDatabase(MySQLGlobalState globalState) throws SQLException { - Randomly r = globalState.getRandomly(); while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = MySQLTableGenerator.generate(tableName, r, globalState.getSchema()); + Query createTable = MySQLTableGenerator.generate(globalState, tableName); globalState.executeStatement(createTable); } diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 0173dcd2c..66edf7fc2 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -29,8 +29,12 @@ public class MySQLSchema extends AbstractSchema { public enum MySQLDataType { INT, VARCHAR, FLOAT, DOUBLE, DECIMAL; - public static MySQLDataType getRandom() { - return Randomly.fromOptions(values()); + public static MySQLDataType getRandom(MySQLGlobalState globalState) { + if (globalState.usesPQS()) { + return Randomly.fromOptions(MySQLDataType.INT, MySQLDataType.VARCHAR); + } else { + return Randomly.fromOptions(values()); + } } public boolean isNumeric() { diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 6c41d5dd7..508d2b1c7 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -134,11 +134,21 @@ private MySQLExpression getComputableFunction(int depth) { private enum ConstantType { INT, NULL, STRING, DOUBLE; + + public static ConstantType[] valuesPQS() { + return new ConstantType[] { INT, NULL, STRING }; + } } @Override public MySQLExpression generateConstant() { - switch (Randomly.fromOptions(ConstantType.values())) { + ConstantType[] values; + if (state.usesPQS()) { + values = ConstantType.valuesPQS(); + } else { + values = ConstantType.values(); + } + switch (Randomly.fromOptions(values)) { case INT: return MySQLConstant.createIntConstant((int) state.getRandomly().getInteger()); case NULL: @@ -155,7 +165,8 @@ public MySQLExpression generateConstant() { } MySQLConstant createStringConstant = MySQLConstant.createStringConstant(string); // if (Randomly.getBoolean()) { - // return new MySQLCollate(createStringConstant, Randomly.fromOptions("ascii_bin", "binary")); + // return new MySQLCollate(createStringConstant, + // Randomly.fromOptions("ascii_bin", "binary")); // } if (string.startsWith("1e")) { // https://bugs.mysql.com/bug.php?id=99146 diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 9f4414180..e99897497 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -12,6 +12,7 @@ import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.mysql.MySQLBugs; +import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; @@ -30,16 +31,18 @@ public class MySQLTableGenerator { private int keysSpecified; private final List columns = new ArrayList<>(); private final MySQLSchema schema; + private final MySQLGlobalState globalState; - public MySQLTableGenerator(String tableName, Randomly r, MySQLSchema schema) { + public MySQLTableGenerator(MySQLGlobalState globalState, String tableName) { this.tableName = tableName; - this.r = r; - this.schema = schema; + this.r = globalState.getRandomly(); + this.schema = globalState.getSchema(); allowPrimaryKey = Randomly.getBoolean(); + this.globalState = globalState; } - public static Query generate(String tableName, Randomly r, MySQLSchema schema) { - return new MySQLTableGenerator(tableName, r, schema).create(); + public static Query generate(MySQLGlobalState globalState, String tableName) { + return new MySQLTableGenerator(globalState, tableName).create(); } private Query create() { @@ -257,7 +260,7 @@ private enum ColumnOptions { private void appendColumnDefinition() { sb.append(" "); - MySQLDataType randomType = MySQLDataType.getRandom(); + MySQLDataType randomType = MySQLDataType.getRandom(globalState); boolean isTextType = randomType == MySQLDataType.VARCHAR; appendTypeString(randomType); sb.append(" "); From d4165fc27d052a9600664a14a8918e4de91e2b56 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 20:24:13 +0200 Subject: [PATCH 0194/1284] [MySQL] Do not generate ZEROFILL columns when using PQS --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index e99897497..68f7be5a7 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -264,11 +264,6 @@ private void appendColumnDefinition() { boolean isTextType = randomType == MySQLDataType.VARCHAR; appendTypeString(randomType); sb.append(" "); - // TODO: this was commented out since it makes the implementation of LIKE more - // difficult - // if (Randomly.getBoolean()) { - // sb.append(" ZEROFILL"); - // } boolean isNull = false; boolean columnHasPrimaryKey = false; @@ -363,7 +358,7 @@ private void appendTypeString(MySQLDataType randomType) { if (Randomly.getBoolean() && randomType != MySQLDataType.INT && !MySQLBugs.bug99127) { sb.append(" UNSIGNED"); } - if (Randomly.getBoolean()) { + if (!globalState.usesPQS() && Randomly.getBoolean()) { sb.append(" ZEROFILL"); } } From e8eea660a3acceaf317ce6c82ab701b9c8743bbe Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 20:26:12 +0200 Subject: [PATCH 0195/1284] [MySQL] Remove commented-out code --- src/sqlancer/mysql/MySQLSchema.java | 13 ----------- .../MySQLPivotedQuerySynthesisOracle.java | 22 ------------------- 2 files changed, 35 deletions(-) diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 66edf7fc2..a23ad7990 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -105,15 +105,7 @@ public MySQLRowValue getRandomRowValue(Connection con) throws SQLException { Object value; int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); assert columnIndex == i + 1; - // String typeString = randomRowValues.getString(columnIndex + getColumns().size()); - // MySQLDataType valueType = getColumnType(typeString); MySQLConstant constant; - // if (randomRowValues.getString(columnIndex) == null) { - // value = null; - // constant = MySQLConstant.createNullConstant(); - // } else { - // switch (valueType) { - // case INT: if (randomRowValues.getString(columnIndex) == null) { constant = MySQLConstant.createNullConstant(); } else { @@ -130,11 +122,6 @@ public MySQLRowValue getRandomRowValue(Connection con) throws SQLException { throw new AssertionError(column.getType()); } } - // break; - // default: - // throw new AssertionError(valueType); - // } - // } values.put(column, constant); } assert !randomRowValues.next(); diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 72d1f07bb..75d0582fd 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -50,30 +50,8 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { MySQLSelect selectStatement = new MySQLSelect(); selectStatement.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); columns = randomFromTables.getColumns(); - // for (MySQLTable t : tables) { - // if (t.getRowid() != null) { - // columns.add(t.getRowid()); - // } - // } pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); - // List joinStatements = new ArrayList<>(); - // for (int i = 1; i < tables.size(); i++) { - // SQLite3Expression joinClause = generateWhereClauseThatContainsRowValue(columns, rw); - // Table table = Randomly.fromList(tables); - // tables.remove(table); - // JoinType options; - // if (tables.size() == 2) { - // // allow outer with arbitrary column order (see error: ON clause references - // // tables to its right) - // options = Randomly.fromOptions(JoinType.INNER, JoinType.CROSS, JoinType.OUTER); - // } else { - // options = Randomly.fromOptions(JoinType.INNER, JoinType.CROSS); - // } - // Join j = new SQLite3Expression.Join(table, joinClause, options); - // joinStatements.add(j); - // } - // selectStatement.setJoinClauses(joinStatements); selectStatement.setFromList(tables.stream().map(t -> new MySQLTableReference(t)).collect(Collectors.toList())); fetchColumns = columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); From bbe6efd9fe1cd0f68e391923a5b25770f8500771 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 15:57:49 +0200 Subject: [PATCH 0196/1284] [Postgres] Fix a PQS minus Long.MAX_VALUE corner case --- src/sqlancer/postgres/ast/PostgresPrefixOperation.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/postgres/ast/PostgresPrefixOperation.java b/src/sqlancer/postgres/ast/PostgresPrefixOperation.java index eefbcc47f..8a19f7ac9 100644 --- a/src/sqlancer/postgres/ast/PostgresPrefixOperation.java +++ b/src/sqlancer/postgres/ast/PostgresPrefixOperation.java @@ -51,6 +51,9 @@ protected PostgresConstant getExpectedValue(PostgresConstant expectedValue) { // TODO throw new IgnoreMeException(); } + if (expectedValue.isInt() && expectedValue.asInt() == Long.MIN_VALUE) { + throw new IgnoreMeException(); + } try { return PostgresConstant.createIntConstant(-expectedValue.asInt()); } catch (UnsupportedOperationException e) { From 9d8eb18a188fcc43938786dddc919275a6fa52ce Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 15:58:22 +0200 Subject: [PATCH 0197/1284] [Postgres] Test PQS --- test/sqlancer/dbms/TestPostgres.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/sqlancer/dbms/TestPostgres.java b/test/sqlancer/dbms/TestPostgres.java index 3bb64ef80..344a47bf7 100644 --- a/test/sqlancer/dbms/TestPostgres.java +++ b/test/sqlancer/dbms/TestPostgres.java @@ -9,10 +9,11 @@ public class TestPostgres { + String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); + boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); + @Test public void testPostgres() { - String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); - boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); assumeTrue(postgresIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, @@ -20,4 +21,13 @@ public void testPostgres() { "false" })); } + @Test + public void testPQS() { + assumeTrue(postgresIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "postgres", "--test-collations", "false", "--oracle", "pqs" })); + } + } From f5e64859372e3fdfe57c82357a49b6b64d117739 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 16:38:11 +0200 Subject: [PATCH 0198/1284] [Postgres] Wrap all strings in COLLATE 'C' for PQS --- .../postgres/gen/PostgresExpressionGenerator.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index f8c7bf539..eb5d772bb 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -271,8 +271,21 @@ public PostgresExpression generateExpression(int depth, PostgresDataType origina dataType = PostgresDataType.INT; } if (!filterColumns(dataType).isEmpty() && Randomly.getBoolean()) { - return createColumnOfType(dataType); + return potentiallyWrapInCollate(dataType, createColumnOfType(dataType)); } + PostgresExpression exprInternal = generateExpressionInternal(depth, dataType); + return potentiallyWrapInCollate(dataType, exprInternal); + } + + private PostgresExpression potentiallyWrapInCollate(PostgresDataType dataType, PostgresExpression exprInternal) { + if (dataType == PostgresDataType.TEXT && PostgresProvider.generateOnlyKnown) { + return new PostgresCollate(exprInternal, "C"); + } else { + return exprInternal; + } + } + + private PostgresExpression generateExpressionInternal(int depth, PostgresDataType dataType) throws AssertionError { if (allowAggregateFunctions && Randomly.getBoolean()) { allowAggregateFunctions = false; // aggregate function calls cannot be nested return getAggregate(dataType); From f642b8a4057e394c9fb5810d29c77b051569e92d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 16:39:16 +0200 Subject: [PATCH 0199/1284] [Postgres] Implement the expected value visitor for LIKE --- .../postgres/PostgresExpectedValueVisitor.java | 8 ++++++++ .../postgres/PostgresToStringVisitor.java | 15 ++++++++------- src/sqlancer/postgres/PostgresVisitor.java | 7 +++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index 4930b676a..d75aa4a5e 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -10,6 +10,7 @@ import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresFunction; import sqlancer.postgres.ast.PostgresInOperation; +import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -169,4 +170,11 @@ public void visit(PostgresBinaryLogicalOperation op) { visit(op.getRight()); } + @Override + public void visit(PostgresLikeOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + } diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 70adf838c..1e95ae745 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -3,6 +3,7 @@ import java.util.Optional; import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; import sqlancer.common.visitor.ToStringVisitor; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; @@ -17,6 +18,7 @@ import sqlancer.postgres.ast.PostgresInOperation; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; +import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -340,13 +342,12 @@ public void visit(PostgresCollate op) { @Override public void visit(PostgresBinaryLogicalOperation op) { - sb.append("("); - visit(op.getLeft()); - sb.append(")"); - sb.append(op.getOperatorRepresentation()); - sb.append("("); - visit(op.getRight()); - sb.append(")"); + super.visit((BinaryOperation) op); + } + + @Override + public void visit(PostgresLikeOperation op) { + super.visit((BinaryOperation) op); } } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 54fb992ff..7b3bf2853 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -14,6 +14,7 @@ import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresFunction; import sqlancer.postgres.ast.PostgresInOperation; +import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -63,6 +64,8 @@ public interface PostgresVisitor { void visit(PostgresBinaryLogicalOperation op); + void visit(PostgresLikeOperation op); + default void visit(PostgresExpression expression) { if (expression instanceof PostgresConstant) { visit((PostgresConstant) expression); @@ -98,8 +101,8 @@ default void visit(PostgresExpression expression) { visit((PostgresFromTable) expression); } else if (expression instanceof PostgresSubquery) { visit((PostgresSubquery) expression); - } else if (expression instanceof PostgresBinaryLogicalOperation) { - visit((PostgresBinaryLogicalOperation) expression); + } else if (expression instanceof PostgresLikeOperation) { + visit((PostgresLikeOperation) expression); } else { throw new AssertionError(expression); } From 1033352e23e04bcba3b65d3e49a501061ef925c5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 16:40:27 +0200 Subject: [PATCH 0200/1284] [Postgres] Ignore a new error --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 8174d3852..874893a7e 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -54,6 +54,7 @@ public PostgresTableGenerator(String tableName, PostgresSchema newSchema, boolea errors.add("has pseudo-type unknown"); errors.add("no collation was derived for partition key column"); errors.add("inherits from generated column but specifies identity"); + errors.add("inherits from generated column but specifies default"); PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonTableErrors(errors); } From 738f54a85e817036ea90b4b5d182f547987a7f8a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 17:12:06 +0200 Subject: [PATCH 0201/1284] Move the Postgres tests to the front in Travis-CI since they run the longest --- .travis.yml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9132ee046..819255d6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,28 @@ branches: matrix: include: + - name: PostgreSQL + jdk : openjdk8 + before_install: + - sudo apt-get update + - sudo apt-get --yes remove postgresql\* + - sudo apt-get install -y postgresql-12 postgresql-client-12 + - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/12/main/postgresql.conf + - sudo cp /etc/postgresql/{10,12}/main/pg_hba.conf + - sudo service postgresql restart 12 + addons: + postgresql: "12.3" + env: + global: + - PGPORT=5432 + services: + - postgresql + before_script: + - sudo apt-get install locales + - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -U postgres + - createdb test -U postgres + script: + - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test - name: MariaDB jdk : openjdk8 before_install: @@ -73,28 +95,6 @@ matrix: jdk : openjdk8 script: - mvn '-Dtest=!sqlancer.dbms.**' test - - name: PostgreSQL - jdk : openjdk8 - before_install: - - sudo apt-get update - - sudo apt-get --yes remove postgresql\* - - sudo apt-get install -y postgresql-12 postgresql-client-12 - - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/12/main/postgresql.conf - - sudo cp /etc/postgresql/{10,12}/main/pg_hba.conf - - sudo service postgresql restart 12 - addons: - postgresql: "12.3" - env: - global: - - PGPORT=5432 - services: - - postgresql - before_script: - - sudo apt-get install locales - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -U postgres - - createdb test -U postgres - script: - - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test - name: Citus (PostgreSQL extension) jdk : openjdk8 before_install: From b24af26bd989b19582e852a0a32836e45cc05022 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 21:36:42 +0200 Subject: [PATCH 0202/1284] Work around connections not being closed by adding a new database prefix option --- src/sqlancer/Main.java | 2 +- src/sqlancer/MainOptions.java | 7 +++++++ test/sqlancer/dbms/TestPostgres.java | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index c1f1bb56b..b92821b86 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -453,7 +453,7 @@ private String formatInteger(long intValue) { ExecutorService execService = Executors.newFixedThreadPool(options.getNumberConcurrentThreads()); DBMSExecutorFactory executorFactory = nameToProvider.get(jc.getParsedCommand()); for (int i = 0; i < options.getTotalNumberTries(); i++) { - final String databaseName = "database" + i; + final String databaseName = options.getDatabasePrefix() + i; final long seed; if (options.getRandomSeed() == -1) { seed = System.currentTimeMillis() + i; diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 9cda38f8b..f61c9adb5 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -88,6 +88,9 @@ public class MainOptions { @Parameter(names = "--constant-cache-size", description = "Specifies the size of the constant cache. This option only takes effect when constant caching is enabled") private int constantCacheSize = 100; // NOPMD + @Parameter(names = "--database-prefix", description = "The prefix used for each database created") + private String databasePrefix = "database"; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -197,4 +200,8 @@ public boolean isHelp() { return help; } + public String getDatabasePrefix() { + return databasePrefix; + } + } diff --git a/test/sqlancer/dbms/TestPostgres.java b/test/sqlancer/dbms/TestPostgres.java index 344a47bf7..5eeb1f858 100644 --- a/test/sqlancer/dbms/TestPostgres.java +++ b/test/sqlancer/dbms/TestPostgres.java @@ -27,7 +27,9 @@ public void testPQS() { assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", - "ALPHANUMERIC_SPECIALCHAR", "postgres", "--test-collations", "false", "--oracle", "pqs" })); + "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", + "pqsdb" /* Workaround for connections not being closed */, "postgres", "--test-collations", + "false", "--oracle", "pqs" })); } } From 3893b78a42bb1f7c6d24465f10dc50d8ed378e9e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 25 Aug 2020 22:23:55 +0200 Subject: [PATCH 0203/1284] [MySQL] Add tests for the PQS implementation --- test/sqlancer/dbms/TestMySQL.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/sqlancer/dbms/TestMySQL.java b/test/sqlancer/dbms/TestMySQL.java index 9d7c2d8e3..da6e80cac 100644 --- a/test/sqlancer/dbms/TestMySQL.java +++ b/test/sqlancer/dbms/TestMySQL.java @@ -9,15 +9,26 @@ public class TestMySQL { + String mysqlAvailable = System.getenv("MYSQL_AVAILABLE"); + boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); + @Test public void testMySQL() { - String mysqlAvailable = System.getenv("MYSQL_AVAILABLE"); - boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); assumeTrue(mysqlIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--max-expression-depth", "1", "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, + "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "mysql" })); } + @Test + public void testPQS() { + assumeTrue(mysqlIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC", "--database-prefix", + "pqsdb" /* Workaround for connections not being closed */, "--num-queries", "100000", "mysql", + "--oracle", "PQS" })); + } + } From a6d2245f9a210ded0fda3dafd2b10096d3bd6003 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2020 06:45:46 +0000 Subject: [PATCH 0204/1284] Bump formatter-maven-plugin from 2.12.1 to 2.12.2 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.12.1 to 2.12.2. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.12.1...formatter-maven-plugin-2.12.2) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dcbf1d49b..bfc1e63d7 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.12.1 + 2.12.2 eclipseformat From 29169268a4d4ab86ef403ba343aa894dd63ba655 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 10:50:36 +0200 Subject: [PATCH 0205/1284] [SQLite] Readd the old PQS cast tests These were previously removed in commit 36ede0c0c68b3856e03ef5ba802a7c2575bb3f12. --- .../pqs/sqlite/cast/TestCastToBlob.java | 44 +++++++ .../pqs/sqlite/cast/TestCastToBoolean.java | 17 +++ .../pqs/sqlite/cast/TestCastToInt.java | 89 +++++++++++++ .../pqs/sqlite/cast/TestCastToNumeric.java | 117 ++++++++++++++++++ .../pqs/sqlite/cast/TestCastToText.java | 84 +++++++++++++ 5 files changed, 351 insertions(+) create mode 100644 test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java create mode 100644 test/sqlancer/pqs/sqlite/cast/TestCastToBoolean.java create mode 100644 test/sqlancer/pqs/sqlite/cast/TestCastToInt.java create mode 100644 test/sqlancer/pqs/sqlite/cast/TestCastToNumeric.java create mode 100644 test/sqlancer/pqs/sqlite/cast/TestCastToText.java diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java b/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java new file mode 100644 index 000000000..bc70e3417 --- /dev/null +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java @@ -0,0 +1,44 @@ +package sqlancer.pqs.sqlite.cast; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.sqlite3.ast.SQLite3Cast; +import sqlancer.sqlite3.ast.SQLite3Constant; + +public class TestCastToBlob { + + @Test + public void testNull() { + SQLite3Constant nullVal = SQLite3Constant.createNullConstant(); + SQLite3Constant castNullVal = SQLite3Cast.castToBlob(nullVal); + assertTrue(castNullVal.isNull()); + } + + @Test + public void testEmptyString() { + SQLite3Constant emptyBinary = SQLite3Constant.createTextConstant(""); + SQLite3Constant castVal = SQLite3Cast.castToBlob(emptyBinary); + assertArrayEquals(new byte[0], castVal.asBinary()); + } + + @Test + public void testString1() { + assertCastStringToBlob("0x12", "x'30783132'"); + } + + @Test + public void testString2() { + assertCastStringToBlob("123", "x'313233'"); + } + + void assertCastStringToBlob(String val, String expectedBlob) { + SQLite3Constant c = SQLite3Constant.createTextConstant(val); + SQLite3Constant binVal = SQLite3Cast.castToBlob(c); + assertEquals(binVal.getValue(), expectedBlob); + } + +} diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToBoolean.java b/test/sqlancer/pqs/sqlite/cast/TestCastToBoolean.java new file mode 100644 index 000000000..ba6b9895a --- /dev/null +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToBoolean.java @@ -0,0 +1,17 @@ +package sqlancer.pqs.sqlite.cast; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.sqlite3.ast.SQLite3Constant; + +public class TestCastToBoolean { + + @Test + void nan() { + SQLite3Constant text = SQLite3Constant.createTextConstant("NaN"); + assertEquals(text.castToBoolean().asInt(), 0); + } + +} diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToInt.java b/test/sqlancer/pqs/sqlite/cast/TestCastToInt.java new file mode 100644 index 000000000..54f338ca3 --- /dev/null +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToInt.java @@ -0,0 +1,89 @@ +package sqlancer.pqs.sqlite.cast; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.sqlite3.ast.SQLite3Cast; +import sqlancer.sqlite3.ast.SQLite3Constant; + +class TestCastToInt { + + @Test + void test1() { + assertBinaryCastToInt("dbb25259", 0); + } + + @Test + void test2() { + assertBinaryCastToInt("d9a3", 0); + } + + @Test + void test3() { + assertCastStringToInt("1231231922047954197746780200000", Long.MAX_VALUE); + } + + @Test + void test4() { + assertCastStringToInt("1231231922047954197746780200000.5", Long.MAX_VALUE); + } + + @Test + void test5() { + assertCastStringToInt("-1231231922047954197746780200000.5", Long.MIN_VALUE); + } + + @Test + void testSign1() { + assertCastStringToInt("++123", 0); + } + + @Test + void testSign2() { + assertCastStringToInt("+123", 123); + } + + @Test + void testSign3() { + assertCastStringToInt("-123", -123); + } + + @Test + void testSign4() { + assertCastStringToInt("-+123", 0); + } + + @Test + void testSign5() { + assertCastStringToInt("+-123", 0); + } + + @Test + void testInfinity1() { + assertCastStringToInt("Infinity", 0); + } + + @Test + void testInfinity2() { + assertCastStringToInt("-Infinity", 0); + } + + @Test + void testNan() { + assertCastStringToInt("NaN", 0); + } + + void assertCastStringToInt(String val, long expectedLong) { + SQLite3Constant c = SQLite3Constant.createTextConstant(val); + SQLite3Constant intVal = SQLite3Cast.castToInt(c); + assertEquals(intVal.asInt(), expectedLong); + } + + void assertBinaryCastToInt(String val, long expectedLong) { + SQLite3Constant c = SQLite3Constant.createBinaryConstant(val); + SQLite3Constant intVal = SQLite3Cast.castToInt(c); + assertEquals(intVal.asInt(), expectedLong); + } + +} diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToNumeric.java b/test/sqlancer/pqs/sqlite/cast/TestCastToNumeric.java new file mode 100644 index 000000000..369e7390a --- /dev/null +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToNumeric.java @@ -0,0 +1,117 @@ +package sqlancer.pqs.sqlite.cast; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Cast; +import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.schema.SQLite3DataType; + +class TestCastToNumeric { + + @Test + void testLong() { + long numbers[] = new long[] { 0, 1, 123, Long.MAX_VALUE, Long.MIN_VALUE }; + for (long number : numbers) { + assertEquals(castLongConstant(number), number); + } + } + + class StringTestTriple { + String value; + SQLite3DataType type; + Object expectedCastValue; + + public StringTestTriple(String value, SQLite3DataType type, Object expectedCastValue) { + this.value = value; + this.type = type; + this.expectedCastValue = expectedCastValue; + } + } + + @Test + void testString() { + List triples = new ArrayList<>(); + triples.add(new StringTestTriple("-3.0", SQLite3DataType.INT, -3L)); + triples.add(new StringTestTriple("8.2250617031974513E18", SQLite3DataType.REAL, 8.2250617031974513E18)); + + triples.add(new StringTestTriple("-2277224522334683278", SQLite3DataType.INT, -2277224522334683278L)); + + triples.add(new StringTestTriple("123a", SQLite3DataType.INT, 123L)); + triples.add(new StringTestTriple("", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("a", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("3", SQLite3DataType.INT, 3L)); + triples.add(new StringTestTriple("-3", SQLite3DataType.INT, -3L)); + triples.add(new StringTestTriple("0.0", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("+0", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("+9", SQLite3DataType.INT, 9L)); + triples.add(new StringTestTriple("++9", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("+-9", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("-1748799336", SQLite3DataType.INT, -1748799336L)); + triples.add(new StringTestTriple("-0", SQLite3DataType.INT, 0L)); + + triples.add(new StringTestTriple("4E ", SQLite3DataType.INT, 4L)); + triples.add(new StringTestTriple("3.0e+5", SQLite3DataType.INT, 300000L)); + triples.add(new StringTestTriple("-3.2", SQLite3DataType.REAL, -3.2d)); + triples.add(new StringTestTriple("10e9", SQLite3DataType.INT, 10000000000L)); + // triples.add(new StringTestTriple("-0.0", SQLite3DataType.REAL, 0.0d)); + triples.add(new StringTestTriple("9223372036854775807", SQLite3DataType.INT, 9223372036854775807L)); + triples.add(new StringTestTriple("4337561223119921152", SQLite3DataType.INT, 4337561223119921152L)); + triples.add(new StringTestTriple("7839344951195291815", SQLite3DataType.INT, 7839344951195291815L)); + + // infinities + triples.add(new StringTestTriple("-Infinity", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("Infinity", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("Inf", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("-Inf", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("NaN", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("1e500", SQLite3DataType.REAL, Double.POSITIVE_INFINITY)); // + triples.add(new StringTestTriple("-1e500", SQLite3DataType.REAL, Double.NEGATIVE_INFINITY)); // + + for (StringTestTriple triple : triples) { + SQLite3Constant castVal = SQLite3Cast.castToNumeric(SQLite3Constant.createTextConstant(triple.value)); + assertEquals(triple.expectedCastValue, castVal.getValue(), triple.value.toString()); + } + } + + @Test + void testBinary() { + List triples = new ArrayList<>(); + triples.add(new StringTestTriple("112B3980", SQLite3DataType.INT, 0L)); // +9� + triples.add(new StringTestTriple("0936", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0C36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0a36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0b36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0c36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0d36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0e36", SQLite3DataType.INT, 0L)); // 6 + triples.add(new StringTestTriple("1a347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("1b347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("1C32", SQLite3DataType.INT, 0L)); // FS2 + triples.add(new StringTestTriple("1D32", SQLite3DataType.INT, 0L)); // GS2 + triples.add(new StringTestTriple("1e32", SQLite3DataType.INT, 0L)); // RS2 + triples.add(new StringTestTriple("1f32", SQLite3DataType.INT, 0L)); // RS2 + triples.add(new StringTestTriple("2032", SQLite3DataType.INT, 2L)); // RS2 + triples.add(new StringTestTriple("09013454", SQLite3DataType.INT, 0L)); // RS2 + triples.add(new StringTestTriple("2016347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("2017347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("2018347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("2019347C", SQLite3DataType.INT, 0L)); // + + for (StringTestTriple triple : triples) { + SQLite3Constant castVal = SQLite3Cast.castToNumeric( + SQLite3Constant.createBinaryConstant(SQLite3Visitor.hexStringToByteArray(triple.value))); + assertEquals(triple.expectedCastValue, castVal.getValue(), triple.value.toString()); + } + } + + private long castLongConstant(long constant) { + return SQLite3Cast.castToNumeric(SQLite3Constant.createIntConstant(constant)).asInt(); + } + +} diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToText.java b/test/sqlancer/pqs/sqlite/cast/TestCastToText.java new file mode 100644 index 000000000..7622ea929 --- /dev/null +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToText.java @@ -0,0 +1,84 @@ +package sqlancer.pqs.sqlite.cast; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.sqlite3.ast.SQLite3Cast; +import sqlancer.sqlite3.ast.SQLite3Constant; + +public class TestCastToText { + + @Test + void test0() { + assertBinaryCastToText("3dca", "=�"); + } + + @Test + void test1() { + assertBinaryCastToText("7e0fa8", "~�"); + } + + @Test + void test2() { + assertBinaryCastToText("a4ee", "��"); + } + + @Test + void test3() { + assertBinaryCastToText("2D8A", "-�"); + } + + void assertBinaryCastToText(String val, String expected) { + SQLite3Constant c = SQLite3Constant.createBinaryConstant(val); + SQLite3Constant intVal = SQLite3Cast.castToText(c); + assertEquals(intVal.asString(), expected); + } + + @Test + void testString1() { + assertRealCastToText(1562730931.0, "1562730931.0"); + } + + @Test + void testString2() { + assertRealCastToText(1.834665208E9, "1834665208.0"); + } + + @Test + void testString3() { + assertRealCastToText(-1.5, "-1.5"); + } + + @Test + void testString4() { + assertRealCastToText(0.8205349286718593, "0.8205349286718593"); + } + + @Test + void testString5() { + assertRealCastToText(-0.6792529217385632, "-0.679252921738563"); + } + + @Test + void testString6() { + assertRealCastToText(0.6918798430590762, "0.691879843059076"); + } + + @Test + void testString7() { + assertRealCastToText(0.021848023722833787, "0.0218480237228338"); + } + + @Test + void testString8() { + assertRealCastToText(-1.6391052705683897E308, "-1.63910527056839e+308"); + } + + void assertRealCastToText(double val, String expected) { + SQLite3Constant c = SQLite3Constant.createRealConstant(val); + SQLite3Constant intVal = SQLite3Cast.castToText(c); + assertEquals(expected, intVal.asString()); + } + +} From fd9011dd82bb6f878c47844f598f61de1b1bd03c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 10:52:46 +0200 Subject: [PATCH 0206/1284] [SQLite] Remove the tests for casting to text, since they are no longer supported --- .../pqs/sqlite/cast/TestCastToText.java | 84 ------------------- 1 file changed, 84 deletions(-) delete mode 100644 test/sqlancer/pqs/sqlite/cast/TestCastToText.java diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToText.java b/test/sqlancer/pqs/sqlite/cast/TestCastToText.java deleted file mode 100644 index 7622ea929..000000000 --- a/test/sqlancer/pqs/sqlite/cast/TestCastToText.java +++ /dev/null @@ -1,84 +0,0 @@ -package sqlancer.pqs.sqlite.cast; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -import sqlancer.sqlite3.ast.SQLite3Cast; -import sqlancer.sqlite3.ast.SQLite3Constant; - -public class TestCastToText { - - @Test - void test0() { - assertBinaryCastToText("3dca", "=�"); - } - - @Test - void test1() { - assertBinaryCastToText("7e0fa8", "~�"); - } - - @Test - void test2() { - assertBinaryCastToText("a4ee", "��"); - } - - @Test - void test3() { - assertBinaryCastToText("2D8A", "-�"); - } - - void assertBinaryCastToText(String val, String expected) { - SQLite3Constant c = SQLite3Constant.createBinaryConstant(val); - SQLite3Constant intVal = SQLite3Cast.castToText(c); - assertEquals(intVal.asString(), expected); - } - - @Test - void testString1() { - assertRealCastToText(1562730931.0, "1562730931.0"); - } - - @Test - void testString2() { - assertRealCastToText(1.834665208E9, "1834665208.0"); - } - - @Test - void testString3() { - assertRealCastToText(-1.5, "-1.5"); - } - - @Test - void testString4() { - assertRealCastToText(0.8205349286718593, "0.8205349286718593"); - } - - @Test - void testString5() { - assertRealCastToText(-0.6792529217385632, "-0.679252921738563"); - } - - @Test - void testString6() { - assertRealCastToText(0.6918798430590762, "0.691879843059076"); - } - - @Test - void testString7() { - assertRealCastToText(0.021848023722833787, "0.0218480237228338"); - } - - @Test - void testString8() { - assertRealCastToText(-1.6391052705683897E308, "-1.63910527056839e+308"); - } - - void assertRealCastToText(double val, String expected) { - SQLite3Constant c = SQLite3Constant.createRealConstant(val); - SQLite3Constant intVal = SQLite3Cast.castToText(c); - assertEquals(expected, intVal.asString()); - } - -} From 1a22cf9c2380543a5efc69c5f47aad60b71677be Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 10:58:25 +0200 Subject: [PATCH 0207/1284] [SQLite] Fix the cast-to-blob tests --- test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java b/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java index bc70e3417..b257a529b 100644 --- a/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java @@ -27,18 +27,18 @@ public void testEmptyString() { @Test public void testString1() { - assertCastStringToBlob("0x12", "x'30783132'"); + assertCastStringToBlob("0x12", "(BINARY) x'30783132'"); } @Test public void testString2() { - assertCastStringToBlob("123", "x'313233'"); + assertCastStringToBlob("123", "(BINARY) x'313233'"); } void assertCastStringToBlob(String val, String expectedBlob) { SQLite3Constant c = SQLite3Constant.createTextConstant(val); SQLite3Constant binVal = SQLite3Cast.castToBlob(c); - assertEquals(binVal.getValue(), expectedBlob); + assertEquals(binVal.toString(), expectedBlob); } } From 4e2634169e2461e75a8874fc97d7a016e6b27bee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 12:25:02 +0200 Subject: [PATCH 0208/1284] [SQLite] Prevent creating views whose columns have affinities or collates when using PQS --- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 07b7190d1..dc7c8c55c 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -7,9 +7,11 @@ import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; +import sqlancer.sqlite3.ast.SQLite3Select; import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.oracle.SQLite3RandomQuerySynthesizer; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -45,12 +47,39 @@ public static Query generate(SQLite3GlobalState globalState) throws SQLException errors.add("The database file is locked"); int size = 1 + Randomly.smallNumber(); columnNamesAs(sb, size); - SQLite3Expression randomQuery = SQLite3RandomQuerySynthesizer.generate(globalState, size); + SQLite3Expression randomQuery; + do { + randomQuery = SQLite3RandomQuerySynthesizer.generate(globalState, size); + } while (globalState.getDmbsSpecificOptions().oracles == SQLite3OracleFactory.PQS + && !checkAffinity(randomQuery)); sb.append(SQLite3Visitor.asString(randomQuery)); return new QueryAdapter(sb.toString(), errors, true); } + /** + * The affinity of columns in a view cannot be determined using features of the DBMS - this would need to be parsed + * from the CREATE TABLE and CREATE VIEW statements. This is non-trivial, and currently not implemented. Rather, we + * avoid generating expressions with an affinity or view. + * + * @see http://sqlite.1065341.n5.nabble.com/Determining-column-collating-functions-td108157.html#a108159 + * + * @param randomQuery + * + * @return true if the query can be used for PQS + */ + private static boolean checkAffinity(SQLite3Expression randomQuery) { + if (randomQuery instanceof SQLite3Select) { + for (SQLite3Expression expr : ((SQLite3Select) randomQuery).getFetchColumns()) { + if (expr.getExpectedValue() == null || expr.getAffinity() != null + || expr.getImplicitCollateSequence() != null || expr.getExplicitCollateSequence() != null) { + return false; + } + } + } + return true; + } + private static void columnNamesAs(StringBuilder sb, int size) { sb.append("("); for (int i = 0; i < size; i++) { From aa234558375d9d3b1345276b05b97fdc81a8aa8a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 12:34:26 +0200 Subject: [PATCH 0209/1284] [SQLite] Add an option to disable generating the IN operator To ignore https://www.sqlite.org/src/tktview?name=f3ff147288 until the fix is included in the latest JDBC driver --- src/sqlancer/sqlite3/SQLite3Options.java | 3 +++ src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index e5be902a6..fdf438672 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -69,6 +69,9 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--test-match" }, description = "Allow the generation of the MATCH operator", arity = 1) public boolean testMatch = true; + @Parameter(names = { "--test-in-operator" }, description = "Allow the generation of the IN operator", arity = 1) + public boolean testIn = true; + @Parameter(names = { "--test-distinct-in-view" }, description = "DISTINCT in views might cause occasional false positives in NoREC and TLP", arity = 1) public boolean testDistinctInView; diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 230f94ce5..9c4e4ce51 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -239,6 +239,9 @@ public SQLite3Expression getRandomExpression(int depth) { if (!globalState.getDmbsSpecificOptions().testMatch) { list.remove(ExpressionType.MATCH); } + if (!globalState.getDmbsSpecificOptions().testIn) { + list.remove(ExpressionType.IN_OPERATOR); + } ExpressionType randomExpressionType = Randomly.fromList(list); switch (randomExpressionType) { case AND_OR_CHAIN: From 8e77193b6699c9238e755844f222144202baf528 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 12:36:13 +0200 Subject: [PATCH 0210/1284] [SQLite] Do not generate CHECK constraints when using PQS since they can be misinterpreted by the parser --- src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index f121c25e0..da04a219e 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -10,6 +10,7 @@ import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; import sqlancer.sqlite3.gen.SQLite3Common; @@ -113,7 +114,13 @@ public void start() { addForeignKey(); } - if (globalState.getDmbsSpecificOptions().testCheckConstraints + if (globalState.getDmbsSpecificOptions().testCheckConstraints && globalState + .getDmbsSpecificOptions().oracles != SQLite3OracleFactory.PQS /* + * we are currently lacking a parser to + * read column definitions, and would + * interpret a COLLATE in the check + * constraint as belonging to the column + */ && Randomly.getBooleanWithRatherLowProbability()) { sb.append(SQLite3Common.getCheckConstraint(globalState, columns)); } From d8e0270257058cfffbadfa104349bea671a6d35b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 12:36:46 +0200 Subject: [PATCH 0211/1284] [SQLite] Do not generate rowid columns since they are difficult to reduce --- .../SQLite3PivotedQuerySynthesisOracle.java | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 274becd3f..0d39c930e 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -72,37 +72,31 @@ public SQLite3Select getQuery() throws SQLException { List tables = randomFromTables.getTables(); pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); - globalState.getState().queryTargetedTablesString = randomFromTables.tableNamesAsString(); SQLite3Select selectStatement = new SQLite3Select(); selectStatement.setSelectType(Randomly.fromOptions(SQLite3Select.SelectType.values())); List columns = randomFromTables.getColumns(); - for (SQLite3Table t : tables) { - if (t.getRowid() != null) { - columns.add(t.getRowid()); - } - } - - List joinStatements = getJoinStatements(globalState, tables, columns); - selectStatement.setJoinClauses(joinStatements); - selectStatement.setFromTables(SQLite3Common.getTableRefs(tables, globalState.getSchema())); - // TODO: also implement a wild-card check (*) // filter out row ids from the select because the hinder the reduction process // once a bug is found List columnsWithoutRowid = columns.stream() .filter(c -> !SQLite3Schema.ROWID_STRINGS.contains(c.getName())).collect(Collectors.toList()); + List joinStatements = getJoinStatements(globalState, tables, columnsWithoutRowid); + selectStatement.setJoinClauses(joinStatements); + selectStatement.setFromTables(SQLite3Common.getTableRefs(tables, globalState.getSchema())); + fetchColumns = Randomly.nonEmptySubset(columnsWithoutRowid); List allTables = new ArrayList<>(); allTables.addAll(tables); allTables.addAll(joinStatements.stream().map(join -> join.getTable()).collect(Collectors.toList())); boolean allTablesContainOneRow = allTables.stream().allMatch(t -> t.getNrRows(globalState) == 1); boolean testAggregateFunctions = allTablesContainOneRow && globalState.getOptions().testAggregateFunctionsPQS(); - pivotRowExpression = getColExpressions(testAggregateFunctions, columns, columnsWithoutRowid); + pivotRowExpression = getColExpressions(testAggregateFunctions, columnsWithoutRowid, columnsWithoutRowid); selectStatement.setFetchColumns(pivotRowExpression); - SQLite3Expression whereClause = generateRectifiedExpression(columns, pivotRow, false); + SQLite3Expression whereClause = generateRectifiedExpression(columnsWithoutRowid, pivotRow, false); selectStatement.setWhereClause(whereClause); - List groupByClause = generateGroupByClause(columns, pivotRow, allTablesContainOneRow); + List groupByClause = generateGroupByClause(columnsWithoutRowid, pivotRow, + allTablesContainOneRow); selectStatement.setGroupByClause(groupByClause); SQLite3Expression limitClause = generateLimit((long) (Math.pow(globalState.getOptions().getMaxNumberInserts(), joinStatements.size() + randomFromTables.getTables().size()))); From bd46b132e033c3635f793700f7f4523dac3b9956 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 17:18:20 +0200 Subject: [PATCH 0212/1284] [SQLite] Fix a PQS COLLATE issue in the CASE operator --- src/sqlancer/sqlite3/ast/SQLite3Case.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Case.java b/src/sqlancer/sqlite3/ast/SQLite3Case.java index 4f5aef30c..da73cfe18 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Case.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Case.java @@ -120,15 +120,15 @@ public SQLite3Constant getExpectedValue() { SQLite3Constant whenComparisonValue = c.getCond().getExpectedValue(); if (whenComparisonValue == null) { return null; - } // TODO collate + } SQLite3CollateSequence seq; if (baseExpr.getExplicitCollateSequence() != null) { seq = baseExpr.getExplicitCollateSequence(); - } else if (whenComparisonValue.getExplicitCollateSequence() != null) { - seq = whenComparisonValue.getExplicitCollateSequence(); + } else if (c.getCond().getExplicitCollateSequence() != null) { + seq = c.getCond().getExplicitCollateSequence(); } else if (c.getCond().getImplicitCollateSequence() != null) { seq = c.getCond().getImplicitCollateSequence(); - } else if (whenComparisonValue.getImplicitCollateSequence() != null) { + } else if (c.getCond().getImplicitCollateSequence() != null) { seq = c.getCond().getImplicitCollateSequence(); } else { seq = SQLite3CollateSequence.BINARY; From c6ef403acad10610abca78ebeac3ac123dc4ded0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 17:19:46 +0200 Subject: [PATCH 0213/1284] [SQLite] Work around a JDBC driver issue --- .../sqlite3/schema/SQLite3Schema.java | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index ad8f0d62a..e36c2879d 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -115,31 +115,32 @@ public static SQLite3Constant getConstant(ResultSet randomRowValues, int columnI throws SQLException, AssertionError { Object value; SQLite3Constant constant; - if (randomRowValues.getString(columnIndex) == null) { - value = null; - constant = SQLite3Constant.createNullConstant(); - } else { - switch (valueType) { - case INT: - value = randomRowValues.getLong(columnIndex); - constant = SQLite3Constant.createIntConstant((long) value); - break; - case REAL: - value = randomRowValues.getDouble(columnIndex); - constant = SQLite3Constant.createRealConstant((double) value); - break; - case TEXT: - case NONE: - value = randomRowValues.getString(columnIndex); - constant = SQLite3Constant.createTextConstant((String) value); - break; - case BINARY: - value = randomRowValues.getBytes(columnIndex); - constant = SQLite3Constant.createBinaryConstant((byte[]) value); - break; - default: - throw new AssertionError(valueType); + switch (valueType) { + case INT: + value = randomRowValues.getLong(columnIndex); + constant = SQLite3Constant.createIntConstant((long) value); + break; + case REAL: + value = randomRowValues.getDouble(columnIndex); + if (!Double.isFinite((double) value)) { + // TODO: the JDBC driver seems to sometimes return infinity for NULL values + throw new IgnoreMeException(); } + constant = SQLite3Constant.createRealConstant((double) value); + break; + case TEXT: + case NONE: + value = randomRowValues.getString(columnIndex); + constant = SQLite3Constant.createTextConstant((String) value); + break; + case BINARY: + value = randomRowValues.getBytes(columnIndex); + constant = SQLite3Constant.createBinaryConstant((byte[]) value); + break; + case NULL: + return SQLite3Constant.createNullConstant(); + default: + throw new AssertionError(valueType); } return constant; } From 9f75bda99e36e77548bde71169f6bc7703d48cdb Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 17:21:00 +0200 Subject: [PATCH 0214/1284] [SQLite] Remove the PQS implementations of arithmetic operators They are tedious to maintain and did not find any bugs. --- .../sqlite3/ast/SQLite3Expression.java | 204 +----------------- 1 file changed, 5 insertions(+), 199 deletions(-) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index b256ddf87..7f7207219 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1129,50 +1129,7 @@ public SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { MULTIPLY("*") { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant result = compute(left, right); - if (result.getDataType() == SQLite3DataType.REAL) { - double val = result.asDouble(); - if (Double.isNaN(val)) { - return SQLite3Constant.createNullConstant(); - } - } - return result; - } - - private SQLite3Constant compute(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - try { - long intResult = Math.multiplyExact(leftInt, rightInt); - return SQLite3Constant.createIntConstant(intResult); - } catch (ArithmeticException e) { - double realResult = (double) leftInt * (double) rightInt; - return SQLite3Constant.createRealConstant(realResult); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftInt * rightDouble); - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - return SQLite3Constant.createRealConstant(leftReal * rightInt); - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftReal * rightReal); - } - } + return null; } }, @@ -1180,100 +1137,14 @@ private SQLite3Constant compute(SQLite3Constant left, SQLite3Constant right) { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - if (rightInt == 0) { - return SQLite3Constant.createNullConstant(); - } else if (leftInt == Long.MIN_VALUE && rightInt == -1) { - return SQLite3Constant.createRealConstant(9.22337203685478e+18); - } else { - return SQLite3Constant.createIntConstant(leftInt / rightInt); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - if (rightDouble == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant(leftInt / rightDouble); - } - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - if (rightInt == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant(leftReal / rightInt); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - if (rightReal == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant(leftReal / rightReal); - } - } - } + return null; } }, // division by zero results in zero REMAINDER("%") { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - if (rightInt == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createIntConstant(leftInt % rightInt); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - if ((long) rightDouble == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant(leftInt % (long) rightDouble); - } - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - if (rightInt == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant((long) leftReal % rightInt); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - if ((long) rightReal == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant((long) leftReal % (long) rightReal); - } - } - } + return null; } }, @@ -1282,80 +1153,15 @@ SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - try { - long intResult = Math.addExact(leftInt, rightInt); - return SQLite3Constant.createIntConstant(intResult); - } catch (ArithmeticException e) { - double realResult = (double) leftInt + (double) rightInt; - return SQLite3Constant.createRealConstant(realResult); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftInt + rightDouble); - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - return SQLite3Constant.createRealConstant(leftReal + rightInt); - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftReal + rightReal); - } - } + return null; } - }, MINUS("-") { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - try { - long intResult = Math.subtractExact(leftInt, rightInt); - return SQLite3Constant.createIntConstant(intResult); - } catch (ArithmeticException e) { - double realResult = (double) leftInt - (double) rightInt; - return SQLite3Constant.createRealConstant(realResult); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftInt - rightDouble); - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - return SQLite3Constant.createRealConstant(leftReal - rightInt); - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftReal - rightReal); - } - } + return null; } }, From 10c855d44f272e8cea1b6c533eebd740a8fe0107 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 17:22:05 +0200 Subject: [PATCH 0215/1284] [SQLite] Check for COLLATE at an additional place --- src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index 0d6dad577..1625c92a5 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -4,6 +4,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; @@ -38,6 +39,10 @@ public boolean isContainsPrimaryKey() { } public String createColumn(String columnName, SQLite3GlobalState globalState, List columns) { + if (globalState.getDmbsSpecificOptions().oracles == SQLite3OracleFactory.PQS + || !globalState.getDmbsSpecificOptions().testCheckConstraints) { + allowCheck = false; + } sb.append(columnName); sb.append(" "); String dataType = Randomly.fromOptions("INT", "TEXT", "BLOB", "REAL", "INTEGER"); From e1c4a21cfa58b6c93d6a6cf8540ea9a0b7f2b7bb Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 17:23:32 +0200 Subject: [PATCH 0216/1284] [SQLite] Also prevent generating UNION clauses with affinities in views when using PQS --- src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index dc7c8c55c..bb5a8b1d2 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -76,8 +76,10 @@ private static boolean checkAffinity(SQLite3Expression randomQuery) { return false; } } + return true; + } else { + return false; // the columns in UNION clauses can also have affinities } - return true; } private static void columnNamesAs(StringBuilder sb, int size) { From e1b268e45ccc85f72cca91ccf0f52a5f9ede7d4a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 17:25:45 +0200 Subject: [PATCH 0217/1284] [SQLite] Do not use non-default encodings when using PQS --- src/sqlancer/sqlite3/SQLite3Provider.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 4586c9b94..54cd6233a 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -331,7 +331,8 @@ private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws SQ // the PQS implementation currently assumes the default behavior of LIKE pragmasToExecute.add("PRAGMA case_sensitive_like=ON;"); } - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { + // the encoding has an influence how binary strings are cast pragmasToExecute.add(String.format("PRAGMA encoding = '%s';", Randomly.fromOptions("UTF-8", "UTF-16", "UTF-16le", "UTF-16be"))); } From 941a5c83e7f9ddb2fdc9cfecdc79a1fac1f61abf Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 18:20:48 +0200 Subject: [PATCH 0218/1284] Scramble bits only when using the sophisticated random string generator --- src/sqlancer/Randomly.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 14b747c14..461d7aae2 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -70,7 +70,8 @@ private String getFromStringCache() { return String.valueOf(Randomly.fromList(cachedLongs)); } else if (Randomly.getBoolean() && !cachedDoubles.isEmpty()) { return String.valueOf(Randomly.fromList(cachedDoubles)); - } else if (Randomly.getBoolean() && !cachedBytes.isEmpty()) { + } else if (Randomly.getBoolean() && !cachedBytes.isEmpty() + && stringGenerationStrategy == StringGenerationStrategy.SOPHISTICATED) { return new String(Randomly.fromList(cachedBytes)); } else if (!cachedStrings.isEmpty()) { String randomString = Randomly.fromList(cachedStrings); From c95c9bbcf5b8bee6be16575b4ff80ea570649fff Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 20:18:41 +0200 Subject: [PATCH 0219/1284] [MySQL] Use the default number of queries --- test/sqlancer/dbms/TestMySQL.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/sqlancer/dbms/TestMySQL.java b/test/sqlancer/dbms/TestMySQL.java index da6e80cac..4ab6576ef 100644 --- a/test/sqlancer/dbms/TestMySQL.java +++ b/test/sqlancer/dbms/TestMySQL.java @@ -13,22 +13,22 @@ public class TestMySQL { boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); @Test - public void testMySQL() { + public void testPQS() { assumeTrue(mysqlIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, - "mysql" })); + "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC", "--database-prefix", + "pqsdb" /* Workaround for connections not being closed */, "--num-queries", + TestConfig.NUM_QUERIES, "mysql", "--oracle", "PQS" })); } @Test - public void testPQS() { + public void testMySQL() { assumeTrue(mysqlIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC", "--database-prefix", - "pqsdb" /* Workaround for connections not being closed */, "--num-queries", "100000", "mysql", - "--oracle", "PQS" })); + "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, + "mysql" })); } } From b478b7db12481bc718c0e23015dc7bf3c4d5de26 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 22:12:02 +0200 Subject: [PATCH 0220/1284] [ClickHouse] Remove an unused import in the test suite --- test/sqlancer/dbms/TestClickHouse.java | 1 - 1 file changed, 1 deletion(-) diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 1421137c3..86d23a9b7 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import sqlancer.Main; From 40dad1bfdd5f1888f98968ba0c3b67baaf5ff8ce Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 26 Aug 2020 22:32:35 +0200 Subject: [PATCH 0221/1284] Remove the umlauts from the alphabet for the random generation --- src/sqlancer/Randomly.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 461d7aae2..171356839 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -290,7 +290,7 @@ public String transformCachedString(Randomly r, String randomString) { }; - private static final String ALPHANUMERIC_SPECIALCHAR_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzöß!#<>/.,~-+'*()[]{} ^*?%_\t\n\r|&\\"; + private static final String ALPHANUMERIC_SPECIALCHAR_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#<>/.,~-+'*()[]{} ^*?%_\t\n\r|&\\"; private static final String ALPHANUMERIC_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; private static final String NUMERIC_ALPHABET = "0123456789"; From d3f06f54a4242887224c9102929c5e7cc1f66e61 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 09:47:35 +0200 Subject: [PATCH 0222/1284] [MySQL] Explicitly specify the TLP test oracle in the tests --- test/sqlancer/dbms/TestMySQL.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sqlancer/dbms/TestMySQL.java b/test/sqlancer/dbms/TestMySQL.java index 4ab6576ef..c1b89c5da 100644 --- a/test/sqlancer/dbms/TestMySQL.java +++ b/test/sqlancer/dbms/TestMySQL.java @@ -28,7 +28,7 @@ public void testMySQL() { assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, - "mysql" })); + "mysql", "--oracle", "TLP_WHERE" })); } } From f572a4bb271b7656f47a29d30b7f9e9dd7c33cbc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 11:18:54 +0200 Subject: [PATCH 0223/1284] [MySQL] Use NUMERIC as a string generation strategy --- test/sqlancer/dbms/TestMySQL.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/sqlancer/dbms/TestMySQL.java b/test/sqlancer/dbms/TestMySQL.java index c1b89c5da..dad9d9ad1 100644 --- a/test/sqlancer/dbms/TestMySQL.java +++ b/test/sqlancer/dbms/TestMySQL.java @@ -16,8 +16,13 @@ public class TestMySQL { public void testPQS() { assumeTrue(mysqlIsAvailable); assertEquals(0, + /* + * While the MySQL generation supports ALPHANUMERIC as string generation strategy, the Travis CI gate + * seems to fail due to special characters that are not supposed to be generated, and which cannot be + * reproduced locally. + */ Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC", "--database-prefix", + "--num-threads", "4", "--random-string-generation", "NUMERIC", "--database-prefix", "pqsdb" /* Workaround for connections not being closed */, "--num-queries", TestConfig.NUM_QUERIES, "mysql", "--oracle", "PQS" })); } From eea8ead0ad1982075043ed34f155e840c13d9a91 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 13:32:02 +0200 Subject: [PATCH 0224/1284] [SQLite] Address encoding issues --- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 3 ++- src/sqlancer/sqlite3/ast/SQLite3Cast.java | 16 +++++++++++++--- .../SQLite3PivotedQuerySynthesisOracle.java | 7 ++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 379e24885..4d0820b25 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -9,6 +9,7 @@ import sqlancer.sqlite3.ast.SQLite3Case.CasePair; import sqlancer.sqlite3.ast.SQLite3Case.SQLite3CaseWithBaseExpression; import sqlancer.sqlite3.ast.SQLite3Case.SQLite3CaseWithoutBaseExpression; +import sqlancer.sqlite3.ast.SQLite3Cast; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.BetweenOperation; @@ -209,7 +210,7 @@ public void visit(SQLite3Constant c) { if (c.getValue() instanceof byte[]) { arr = c.asBinary(); } else { - arr = c.asString().getBytes(); + arr = c.asString().getBytes(SQLite3Cast.DEFAULT_ENCODING); } sb.append(SQLite3Visitor.byteArrayToHex(arr)); sb.append("'"); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Cast.java b/src/sqlancer/sqlite3/ast/SQLite3Cast.java index 99db3c9cb..25ac45cff 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Cast.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Cast.java @@ -1,16 +1,19 @@ package sqlancer.sqlite3.ast; import java.math.BigDecimal; +import java.nio.charset.Charset; import java.sql.Connection; import java.util.Optional; import java.util.regex.Pattern; +import sqlancer.IgnoreMeException; import sqlancer.sqlite3.schema.SQLite3DataType; public final class SQLite3Cast { private static final double MAX_INT_FOR_WHICH_CONVERSION_TO_INT_IS_TRIED = Math.pow(2, 51 - 1) - 1; private static final double MIN_INT_FOR_WHICH_CONVERSION_TO_INT_IS_TRIED = -Math.pow(2, 51 - 1); + public static final Charset DEFAULT_ENCODING = Charset.forName("UTF-8"); private static final byte FILE_SEPARATOR = 0x1c; private static final byte GROUP_SEPARATOR = 0x1d; @@ -45,11 +48,18 @@ public static Optional isTrue(SQLite3Constant value) { } } + public static void checkDoubleIsInsideDangerousRange(double doubleVal) { + // high double-values might result in small rounding differences between Java and SQLite + if (Math.abs(doubleVal) > 1e15) { + throw new IgnoreMeException(); + } + } + // SELECT CAST('-1.370998801E9' AS INTEGER) == -1 public static SQLite3Constant castToInt(SQLite3Constant originalCons) { SQLite3Constant cons = originalCons; if (cons.getDataType() == SQLite3DataType.BINARY) { - String text = new String(cons.asBinary()); + String text = new String(cons.asBinary(), DEFAULT_ENCODING); cons = SQLite3Constant.createTextConstant(text); } switch (cons.getDataType()) { @@ -124,7 +134,7 @@ private static SQLite3Constant convertInternal(SQLite3Constant originalValue, bo boolean noNumIsRealZero, boolean convertIntToReal) throws AssertionError { SQLite3Constant value = originalValue; if (value.getDataType() == SQLite3DataType.BINARY) { - String text = new String(value.asBinary()); + String text = new String(value.asBinary(), DEFAULT_ENCODING); value = SQLite3Constant.createTextConstant(text); } switch (value.getDataType()) { @@ -261,7 +271,7 @@ public static SQLite3Constant castToBlob(SQLite3Constant cons) { if (stringVal == null) { return null; } else { - return SQLite3Constant.createBinaryConstant(stringVal.asString().getBytes()); + return SQLite3Constant.createBinaryConstant(stringVal.asString().getBytes(DEFAULT_ENCODING)); } } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 0d39c930e..7acd65f67 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -226,7 +226,12 @@ private String getGeneralizedPivotRowValues() { sb.append(", "); } SQLite3Constant expectedValue = pivotRowExpression.get(i).getExpectedValue(); - sb.append(SQLite3Visitor.asString(expectedValue)); + String value = SQLite3Visitor.asString(expectedValue); + if (value.contains("�")) { + // encoding issues + throw new IgnoreMeException(); + } + sb.append(value); } return sb.toString(); } From 1ce76fe2ab6420f5655fca21f334661966a0d9a0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 13:33:33 +0200 Subject: [PATCH 0225/1284] [SQLite] Check for more doubles that might result in rounding errors --- src/sqlancer/sqlite3/ast/SQLite3Cast.java | 5 ++++- src/sqlancer/sqlite3/ast/SQLite3Expression.java | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Cast.java b/src/sqlancer/sqlite3/ast/SQLite3Cast.java index 25ac45cff..32f83381c 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Cast.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Cast.java @@ -68,6 +68,7 @@ public static SQLite3Constant castToInt(SQLite3Constant originalCons) { case INT: return cons; case REAL: + checkDoubleIsInsideDangerousRange(cons.asDouble()); return SQLite3Constant.createIntConstant((long) cons.asDouble()); case TEXT: String asString = cons.asString(); @@ -109,7 +110,9 @@ public static SQLite3Constant castToInt(SQLite3Constant originalCons) { public static SQLite3Constant castToReal(SQLite3Constant cons) { SQLite3Constant numericValue = castToNumeric(cons); if (numericValue.getDataType() == SQLite3DataType.INT) { - return SQLite3Constant.createRealConstant(numericValue.asInt()); + double val = numericValue.asInt(); + checkDoubleIsInsideDangerousRange(val); + return SQLite3Constant.createRealConstant(val); } else { return numericValue; } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 7f7207219..7bfbb7ea7 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1342,8 +1342,12 @@ public SQLite3Constant getExpectedValue() { if (left.getExpectedValue() == null || right.getExpectedValue() == null) { return null; } - return operation.applyOperand(left.getExpectedValue(), left.getAffinity(), right.getExpectedValue(), - right.getAffinity()); + SQLite3Constant result = operation.applyOperand(left.getExpectedValue(), left.getAffinity(), + right.getExpectedValue(), right.getAffinity()); + if (result != null && result.isReal()) { + SQLite3Cast.checkDoubleIsInsideDangerousRange(result.asDouble()); + } + return result; } public static Sqlite3BinaryOperation create(SQLite3Expression leftVal, SQLite3Expression rightVal, From efef8d53ec272c9823a58ce234244de38bea5f23 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 13:53:04 +0200 Subject: [PATCH 0226/1284] [SQLite] Work around a JDBC driver issue --- src/sqlancer/sqlite3/schema/SQLite3Schema.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index e36c2879d..4870ae109 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -136,6 +136,10 @@ public static SQLite3Constant getConstant(ResultSet randomRowValues, int columnI case BINARY: value = randomRowValues.getBytes(columnIndex); constant = SQLite3Constant.createBinaryConstant((byte[]) value); + if (((byte[]) value).length == 0) { + // TODO: the JDBC driver seems to sometimes return a zero-length array for NULL values + throw new IgnoreMeException(); + } break; case NULL: return SQLite3Constant.createNullConstant(); From d97055fd368916c76fa486d02140b54d7de5f332 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 14:48:52 +0200 Subject: [PATCH 0227/1284] [SQLite] Enable SQLite PQS tests --- .travis.yml | 9 +++++---- test/sqlancer/dbms/TestSQLitePQS.java | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 test/sqlancer/dbms/TestSQLitePQS.java diff --git a/.travis.yml b/.travis.yml index 819255d6a..dcfa76ccb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,11 @@ branches: matrix: include: + - name: SQLite3 + jdk : openjdk8 + script: + - mvn -Dtest=TestSQLitePQS test + - mvn -Dtest=TestSQLite3 test - name: PostgreSQL jdk : openjdk8 before_install: @@ -75,10 +80,6 @@ matrix: - sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" script: - TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test - - name: SQLite3 - jdk : openjdk8 - script: - - mvn -Dtest=TestSQLite3 test - name: DuckDB jdk : openjdk8 script: diff --git a/test/sqlancer/dbms/TestSQLitePQS.java b/test/sqlancer/dbms/TestSQLitePQS.java new file mode 100644 index 000000000..976b305b6 --- /dev/null +++ b/test/sqlancer/dbms/TestSQLitePQS.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLitePQS { + + @Test + public void testSqlitePQS() { + // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", "1000", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", + "--database-prefix", "pqsdb" /* Workaround for connections not being closed */, "sqlite3", "--oracle", + "PQS", "--test-fts", "false", "--test-rtree", "false", "--test-check-constraints", "false", + "--test-in-operator", "false" })); + } + +} From bd0f094ffc9341141c1e8a185c2d87c2a53baba5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 17:11:29 +0200 Subject: [PATCH 0228/1284] [SQLite] Address an issue which causes Java do not completely print a string --- .../sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 7acd65f67..e9976c6b6 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -227,8 +227,8 @@ private String getGeneralizedPivotRowValues() { } SQLite3Constant expectedValue = pivotRowExpression.get(i).getExpectedValue(); String value = SQLite3Visitor.asString(expectedValue); - if (value.contains("�")) { - // encoding issues + if (value.contains("�") || value.contains("\0")) { + // encoding issues || Java does not completely strings with \0 characters throw new IgnoreMeException(); } sb.append(value); From a9f2df508f8a8adcdec7f6d0251c49329b1480e6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 12:58:28 +0200 Subject: [PATCH 0229/1284] [MySQL] Ee-enable ALPHANUMERIC string generation for PQS by splitting up the test classes --- .travis.yml | 3 ++- .../mysql/gen/MySQLExpressionGenerator.java | 3 ++- .../{TestMySQL.java => TestMySQLPQS.java} | 18 ++------------ test/sqlancer/dbms/TestMySQLTLP.java | 24 +++++++++++++++++++ 4 files changed, 30 insertions(+), 18 deletions(-) rename test/sqlancer/dbms/{TestMySQL.java => TestMySQLPQS.java} (51%) create mode 100644 test/sqlancer/dbms/TestMySQLTLP.java diff --git a/.travis.yml b/.travis.yml index dcfa76ccb..5546d12bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,8 @@ matrix: - sudo apt-get update && sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 - sudo apt-get remove mysql-* && wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && yes | sudo dpkg -i *.deb - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - - MYSQL_AVAILABLE=true mvn -Dtest=TestMySQL test + - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS + - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP - name: CockroachDB jdk : openjdk8 before_install: diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 508d2b1c7..7911a0f3e 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -154,7 +154,8 @@ public MySQLExpression generateConstant() { case NULL: return MySQLConstant.createNullConstant(); case STRING: - String string = state.getRandomly().getString(); + /* Replace characters that still trigger open bugs in MySQL */ + String string = state.getRandomly().getString().replace("\\", "").replace("\n", ""); if (string.startsWith("\n")) { // workaround for https://bugs.mysql.com/bug.php?id=99130 throw new IgnoreMeException(); diff --git a/test/sqlancer/dbms/TestMySQL.java b/test/sqlancer/dbms/TestMySQLPQS.java similarity index 51% rename from test/sqlancer/dbms/TestMySQL.java rename to test/sqlancer/dbms/TestMySQLPQS.java index dad9d9ad1..ba6a9c501 100644 --- a/test/sqlancer/dbms/TestMySQL.java +++ b/test/sqlancer/dbms/TestMySQLPQS.java @@ -7,7 +7,7 @@ import sqlancer.Main; -public class TestMySQL { +public class TestMySQLPQS { String mysqlAvailable = System.getenv("MYSQL_AVAILABLE"); boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); @@ -16,24 +16,10 @@ public class TestMySQL { public void testPQS() { assumeTrue(mysqlIsAvailable); assertEquals(0, - /* - * While the MySQL generation supports ALPHANUMERIC as string generation strategy, the Travis CI gate - * seems to fail due to special characters that are not supposed to be generated, and which cannot be - * reproduced locally. - */ Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--random-string-generation", "NUMERIC", "--database-prefix", + "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC", "--database-prefix", "pqsdb" /* Workaround for connections not being closed */, "--num-queries", TestConfig.NUM_QUERIES, "mysql", "--oracle", "PQS" })); } - @Test - public void testMySQL() { - assumeTrue(mysqlIsAvailable); - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, - "mysql", "--oracle", "TLP_WHERE" })); - } - } diff --git a/test/sqlancer/dbms/TestMySQLTLP.java b/test/sqlancer/dbms/TestMySQLTLP.java new file mode 100644 index 000000000..3d3e7d107 --- /dev/null +++ b/test/sqlancer/dbms/TestMySQLTLP.java @@ -0,0 +1,24 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMySQLTLP { + + String mysqlAvailable = System.getenv("MYSQL_AVAILABLE"); + boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); + + @Test + public void testMySQL() { + assumeTrue(mysqlIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, + "mysql", "--oracle", "TLP_WHERE" })); + } + +} From 3e3a8eb70459fe0d1c6d1b51405ae3f3a419ec31 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 18:22:18 +0200 Subject: [PATCH 0230/1284] [MySQL] Ignore a table-creation error --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 68f7be5a7..e514cfbf7 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -98,6 +98,7 @@ private void addCommonErrors(ExpectedErrors list) { list.add("doesn't support BLOB/TEXT columns"); list.add("A BLOB field is not allowed in partition function"); list.add("Too many keys specified; max 1 keys allowed"); + list.add("The total length of the partitioning fields is too large"); } private enum PartitionOptions { From 48186ce9d669c07f62670baf7ce499e57beea5b8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 19:05:19 +0200 Subject: [PATCH 0231/1284] [SQLite] Move the stat-manipulation table generator to its own class --- src/sqlancer/sqlite3/SQLite3Provider.java | 63 +------------ .../gen/dml/SQLite3StatTableGenerator.java | 88 +++++++++++++++++++ 2 files changed, 90 insertions(+), 61 deletions(-) create mode 100644 src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 54cd6233a..5c658478a 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -3,9 +3,7 @@ import java.io.File; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -41,11 +39,10 @@ import sqlancer.sqlite3.gen.ddl.SQLite3ViewGenerator; import sqlancer.sqlite3.gen.dml.SQLite3DeleteGenerator; import sqlancer.sqlite3.gen.dml.SQLite3InsertGenerator; +import sqlancer.sqlite3.gen.dml.SQLite3StatTableGenerator; import sqlancer.sqlite3.gen.dml.SQLite3UpdateGenerator; import sqlancer.sqlite3.schema.SQLite3Schema; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table.TableKind; public class SQLite3Provider extends ProviderAdapter { @@ -101,63 +98,7 @@ public boolean canBeRetried() { VIRTUAL_TABLE_ACTION(SQLite3VirtualFTSTableCommandGenerator::create), // CREATE_VIEW(SQLite3ViewGenerator::generate), // CREATE_TRIGGER(SQLite3CreateTriggerGenerator::create), // - MANIPULATE_STAT_TABLE((g) -> { - List columns = new ArrayList<>(); - SQLite3Table t = new SQLite3Table("sqlite_stat1", columns, TableKind.MAIN, false, false, false, false); - if (Randomly.getBoolean()) { - return SQLite3DeleteGenerator.deleteContent(g, t); - } else { - StringBuilder sb = new StringBuilder(); - sb.append("INSERT OR IGNORE INTO sqlite_stat1"); - String indexName; - try (Statement stat = g.getConnection().createStatement()) { - try (ResultSet rs = stat.executeQuery( - "SELECT name FROM sqlite_master WHERE type='index' ORDER BY RANDOM() LIMIT 1;")) { - if (rs.isClosed()) { - throw new IgnoreMeException(); - } - indexName = rs.getString("name"); - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - sb.append(" VALUES"); - sb.append("('"); - sb.append(g.getSchema().getRandomTable().getName()); - sb.append("', "); - sb.append("'"); - if (Randomly.getBoolean()) { - sb.append(indexName); - } else { - sb.append(g.getSchema().getRandomTable().getName()); - } - sb.append("'"); - sb.append(", '"); - for (int i = 0; i < Randomly.smallNumber(); i++) { - if (i != 0) { - sb.append(" "); - } - if (Randomly.getBoolean()) { - sb.append(g.getRandomly().getInteger()); - } else { - sb.append(Randomly.smallNumber()); - } - } - if (Randomly.getBoolean()) { - sb.append(" sz="); - sb.append(g.getRandomly().getInteger()); - } - if (Randomly.getBoolean()) { - sb.append(" unordered"); - } - if (Randomly.getBoolean()) { - sb.append(" noskipscan"); - } - sb.append("')"); - return new QueryAdapter(sb.toString(), - ExpectedErrors.from("no such table", "The database file is locked")); - } - }); + MANIPULATE_STAT_TABLE(SQLite3StatTableGenerator::getQuery); private final QueryProvider queryProvider; diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java new file mode 100644 index 000000000..d953afb32 --- /dev/null +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java @@ -0,0 +1,88 @@ +package sqlancer.sqlite3.gen.dml; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; +import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table.TableKind; + +public final class SQLite3StatTableGenerator { + + private final SQLite3GlobalState globalState; + + private SQLite3StatTableGenerator(SQLite3GlobalState globalState) { + this.globalState = globalState; + } + + public static Query getQuery(SQLite3GlobalState globalState) { + return new SQLite3StatTableGenerator(globalState).getQuery(); + } + + private Query getQuery() { + List columns = new ArrayList<>(); + SQLite3Table t = new SQLite3Table("sqlite_stat1", columns, TableKind.MAIN, false, false, false, false); + if (Randomly.getBoolean()) { + return SQLite3DeleteGenerator.deleteContent(globalState, t); + } else { + StringBuilder sb = new StringBuilder(); + sb.append("INSERT OR IGNORE INTO sqlite_stat1"); + String indexName; + try (Statement stat = globalState.getConnection().createStatement()) { + try (ResultSet rs = stat + .executeQuery("SELECT name FROM sqlite_master WHERE type='index' ORDER BY RANDOM() LIMIT 1;")) { + if (rs.isClosed()) { + throw new IgnoreMeException(); + } + indexName = rs.getString("name"); + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + sb.append(" VALUES"); + sb.append("('"); + sb.append(globalState.getSchema().getRandomTable().getName()); + sb.append("', "); + sb.append("'"); + if (Randomly.getBoolean()) { + sb.append(indexName); + } else { + sb.append(globalState.getSchema().getRandomTable().getName()); + } + sb.append("'"); + sb.append(", '"); + for (int i = 0; i < Randomly.smallNumber(); i++) { + if (i != 0) { + sb.append(" "); + } + if (Randomly.getBoolean()) { + sb.append(globalState.getRandomly().getInteger()); + } else { + sb.append(Randomly.smallNumber()); + } + } + if (Randomly.getBoolean()) { + sb.append(" sz="); + sb.append(globalState.getRandomly().getInteger()); + } + if (Randomly.getBoolean()) { + sb.append(" unordered"); + } + if (Randomly.getBoolean()) { + sb.append(" noskipscan"); + } + sb.append("')"); + return new QueryAdapter(sb.toString(), ExpectedErrors.from("no such table", "The database file is locked")); + } + } + +} From 1231dfe487eee665305424c460df50f192866a91 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 19:07:49 +0200 Subject: [PATCH 0232/1284] Remove unused code to generate PQS debug information --- .../MySQLPivotedQuerySynthesisOracle.java | 20 ------------------- .../PostgresPivotedQuerySynthesisOracle.java | 20 ------------------- 2 files changed, 40 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 75d0582fd..87c318bc6 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -75,26 +75,6 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { .generateOrderBys(); selectStatement.setOrderByExpressions(orderBy); - StringBuilder sb2 = new StringBuilder(); - sb2.append("SELECT * FROM (SELECT 1 FROM "); - sb2.append(randomFromTables.tableNamesAsString()); - sb2.append(" WHERE "); - int i = 0; - for (MySQLColumn c : columns) { - if (i++ != 0) { - sb2.append(" AND "); - } - sb2.append("ref"); - sb2.append(i - 1); - if (pivotRow.getValues().get(c).isNull()) { - sb2.append(" IS NULL"); - } else { - sb2.append(" = "); - sb2.append(pivotRow.getValues().get(c).getTextRepresentation()); - } - } - sb2.append(") as result;"); - MySQLToStringVisitor visitor = new MySQLToStringVisitor(); visitor.visit(selectStatement); return new QueryAdapter(visitor.get(), errors); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index aaf5f4f59..cb346cf88 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -75,26 +75,6 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { List orderBy = new PostgresExpressionGenerator(globalState).setColumns(columns) .generateOrderBy(); selectStatement.setOrderByExpressions(orderBy); - - StringBuilder sb2 = new StringBuilder(); - sb2.append("SELECT * FROM (SELECT 1 FROM "); - sb2.append(randomFromTables.tableNamesAsString()); - sb2.append(" WHERE "); - int i = 0; - for (PostgresColumn c : fetchColumns) { - if (i++ != 0) { - sb2.append(" AND "); - } - sb2.append(c.getFullQualifiedName()); - if (pivotRow.getValues().get(c).isNull()) { - sb2.append(" IS NULL"); - } else { - sb2.append(" = "); - sb2.append(pivotRow.getValues().get(c).getTextRepresentation()); - } - } - sb2.append(") as result;"); - PostgresToStringVisitor visitor = new PostgresToStringVisitor(); visitor.visit(selectStatement); return new QueryAdapter(visitor.get()); From 6e854a8b5a1411c9e6caccc7e98b88bfa33d2c87 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 19:23:04 +0200 Subject: [PATCH 0233/1284] Use a SQLancerResultSet in the PQS classes --- .../MySQLPivotedQuerySynthesisOracle.java | 21 ++++++------------ .../PostgresPivotedQuerySynthesisOracle.java | 22 +++++-------------- .../SQLite3PivotedQuerySynthesisOracle.java | 18 +++++---------- 3 files changed, 18 insertions(+), 43 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 87c318bc6..ad5bf81aa 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -1,16 +1,16 @@ package sqlancer.mysql.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; @@ -124,9 +124,6 @@ private MySQLExpression generateWhereClauseThatContainsRowValue(List Date: Thu, 27 Aug 2020 19:31:51 +0200 Subject: [PATCH 0234/1284] Factor out common functionality to check whether the pivot row is contained --- .../oracle/PivotedQuerySynthesisBase.java | 20 ++++++++++++++-- .../MySQLPivotedQuerySynthesisOracle.java | 16 ++----------- .../PostgresPivotedQuerySynthesisOracle.java | 23 ++----------------- .../SQLite3PivotedQuerySynthesisOracle.java | 12 ++-------- 4 files changed, 24 insertions(+), 47 deletions(-) diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index 279e31ae1..781a064d6 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -5,8 +5,10 @@ import java.util.List; import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractRowValue; public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E> @@ -28,12 +30,26 @@ public final void check() throws SQLException { if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(pivotRowQuery.getQueryString()); } - boolean isContainedIn = isContainedIn(pivotRowQuery); + Query isContainedQuery = getContainedInQuery(pivotRowQuery); + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(isContainedQuery.getQueryString()); + } + globalState.getState().getLocalState().log(isContainedQuery.getQueryString()); + boolean isContainedIn = isPivotRowContainedIn(isContainedQuery); if (!isContainedIn) { reportMissingPivotRow(pivotRowQuery); } } + private boolean isPivotRowContainedIn(Query isContainedQuery) throws SQLException { + try (SQLancerResultSet result = isContainedQuery.executeAndGet(globalState)) { + if (result == null) { + throw new IgnoreMeException(); + } + return !result.isClosed(); + } + } + protected void reportMissingPivotRow(Query query) { globalState.getState().getLocalState().log("-- " + "pivot row values:\n"); String expectedPivotRowString = pivotRow.asStringGroupedByTables(); @@ -48,7 +64,7 @@ protected void reportMissingPivotRow(Query query) { throw new AssertionError(query); } - protected abstract boolean isContainedIn(Query pivotRowQuery) throws SQLException; + protected abstract Query getContainedInQuery(Query pivotRowQuery) throws SQLException; protected abstract Query getQueryThatContainsAtLeastOneRow() throws SQLException; diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index ad5bf81aa..4284c8a69 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -5,12 +5,10 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; -import sqlancer.common.query.SQLancerResultSet; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; @@ -123,7 +121,7 @@ private MySQLExpression generateWhereClauseThatContainsRowValue(List { private List fetchColumns; - private final MainOptions options; - private final StateLogger logger; public PostgresPivotedQuerySynthesisOracle(PostgresGlobalState globalState) throws SQLException { super(globalState); - options = globalState.getOptions(); - logger = globalState.getLogger(); PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonFetchErrors(errors); } @@ -136,7 +128,7 @@ private PostgresExpression generateWhereClauseThatContainsRowValue(List Date: Thu, 27 Aug 2020 19:38:06 +0200 Subject: [PATCH 0235/1284] Introduce a method to get the unterminated query string to simplify the PQS implementations --- src/sqlancer/common/query/Query.java | 12 ++++++++++++ src/sqlancer/common/query/QueryAdapter.java | 12 ++++++++++++ .../oracle/MySQLPivotedQuerySynthesisOracle.java | 6 +----- .../oracle/PostgresPivotedQuerySynthesisOracle.java | 6 +----- .../oracle/SQLite3PivotedQuerySynthesisOracle.java | 7 +------ 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/sqlancer/common/query/Query.java b/src/sqlancer/common/query/Query.java index 1dad78f94..430c93cad 100644 --- a/src/sqlancer/common/query/Query.java +++ b/src/sqlancer/common/query/Query.java @@ -6,8 +6,20 @@ public abstract class Query { + /** + * Gets the query string, which is guaranteed to be terminated with a semicolon. + * + * @return the query string. + */ public abstract String getQueryString(); + /** + * Gets the query string without trailing semicolons. + * + * @return the query string that does not end with a ";". + */ + public abstract String getUnterminatedQueryString(); + /** * Whether the query could affect the schema (i.e., by add/deleting columns or tables). * diff --git a/src/sqlancer/common/query/QueryAdapter.java b/src/sqlancer/common/query/QueryAdapter.java index e7e7be4ee..8934bdc14 100644 --- a/src/sqlancer/common/query/QueryAdapter.java +++ b/src/sqlancer/common/query/QueryAdapter.java @@ -55,6 +55,18 @@ public String getQueryString() { return query; } + @Override + public String getUnterminatedQueryString() { + String result; + if (query.endsWith(";")) { + result = query.substring(0, query.length() - 1); + } else { + result = query; + } + assert !result.endsWith(";"); + return result; + } + @Override public boolean execute(GlobalState globalState, String... fills) throws SQLException { Statement s; diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 4284c8a69..b5b661447 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -124,11 +124,7 @@ private MySQLExpression generateWhereClauseThatContainsRowValue(List Date: Thu, 27 Aug 2020 22:13:23 +0200 Subject: [PATCH 0236/1284] Improve the documentation and naming of the PQS methods --- .../oracle/PivotedQuerySynthesisBase.java | 64 ++++++++++++++++--- .../MySQLPivotedQuerySynthesisOracle.java | 11 ++-- .../PostgresPivotedQuerySynthesisOracle.java | 11 ++-- .../SQLite3PivotedQuerySynthesisOracle.java | 12 ++-- 4 files changed, 66 insertions(+), 32 deletions(-) diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index 781a064d6..2a2f2e7d1 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -15,6 +15,10 @@ public abstract class PivotedQuerySynthesisBase, R e implements TestOracle { protected final ExpectedErrors errors = new ExpectedErrors(); + + /** + * The predicates used in WHERE and JOIN clauses, which yield TRUE for the pivot row + */ protected final List rectifiedPredicates = new ArrayList<>(); protected final S globalState; protected R pivotRow; @@ -26,23 +30,34 @@ public PivotedQuerySynthesisBase(S globalState) { @Override public final void check() throws SQLException { rectifiedPredicates.clear(); - Query pivotRowQuery = getQueryThatContainsAtLeastOneRow(); + Query pivotRowQuery = getRectifiedQuery(); if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(pivotRowQuery.getQueryString()); } - Query isContainedQuery = getContainedInQuery(pivotRowQuery); + Query isContainedQuery = getContainmentCheckQuery(pivotRowQuery); if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(isContainedQuery.getQueryString()); } globalState.getState().getLocalState().log(isContainedQuery.getQueryString()); - boolean isContainedIn = isPivotRowContainedIn(isContainedQuery); - if (!isContainedIn) { + // combines step 6 and 7 described in the PQS paper + boolean pivotRowIsContained = containsRows(isContainedQuery); + if (!pivotRowIsContained) { reportMissingPivotRow(pivotRowQuery); } } - private boolean isPivotRowContainedIn(Query isContainedQuery) throws SQLException { - try (SQLancerResultSet result = isContainedQuery.executeAndGet(globalState)) { + /** + * Checks whether the result set contains at least a single row. + * + * @param query + * the query for which to check whether its result set contains any rows + * + * @return true if at least one row is contained, false otherwise + * + * @throws SQLException + */ + private boolean containsRows(Query query) throws SQLException { + try (SQLancerResultSet result = query.executeAndGet(globalState)) { if (result == null) { throw new IgnoreMeException(); } @@ -58,16 +73,45 @@ protected void reportMissingPivotRow(Query query) { StringBuilder sb = new StringBuilder("-- rectified predicates and their expected values:\n"); for (E rectifiedPredicate : rectifiedPredicates) { sb.append("--"); - sb.append(asString(rectifiedPredicate).replace("\n", "\n-- ")); + sb.append(getExpectedValues(rectifiedPredicate).replace("\n", "\n-- ")); } globalState.getState().getLocalState().log(sb.toString()); throw new AssertionError(query); } - protected abstract Query getContainedInQuery(Query pivotRowQuery) throws SQLException; + /** + * Gets a query that checks whether the pivot row is contained in the result. If the pivot row is contained, the + * query will fetch at least one row. If the pivot row is not contained, no rows will be fetched. This corresponds + * to step 7 described in the PQS paper. + * + * @param pivotRowQuery + * the query that is guaranteed to fetch the pivot row, potentially among other rows + * + * @return a query that checks whether the pivot row is contained in pivotRowQuery + * + * @throws SQLException + */ + protected abstract Query getContainmentCheckQuery(Query pivotRowQuery) throws SQLException; - protected abstract Query getQueryThatContainsAtLeastOneRow() throws SQLException; + /** + * Obtains a rectified query (i.e., a query that is guaranteed to fetch the pivot row. This corresponds to steps 2-5 + * of the PQS paper. + * + * @return the rectified query + * + * @throws SQLException + */ + protected abstract Query getRectifiedQuery() throws SQLException; - protected abstract String asString(E expr); + /** + * Prints the value to which the expression is expected to evaluate, and then recursively prints the subexpressions' + * expected values. + * + * @param expr + * the expression whose expected value should be printed + * + * @return a string representing the expected value of the expression and its subexpressions + */ + protected abstract String getExpectedValues(E expr); } diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index b5b661447..b73ae1f87 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -15,7 +15,6 @@ import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLSchema.MySQLTables; -import sqlancer.mysql.MySQLToStringVisitor; import sqlancer.mysql.MySQLVisitor; import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLConstant; @@ -41,7 +40,7 @@ public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQL } @Override - public Query getQueryThatContainsAtLeastOneRow() throws SQLException { + public Query getRectifiedQuery() throws SQLException { MySQLTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); List tables = randomFromTables.getTables(); @@ -73,9 +72,7 @@ public Query getQueryThatContainsAtLeastOneRow() throws SQLException { .generateOrderBys(); selectStatement.setOrderByExpressions(orderBy); - MySQLToStringVisitor visitor = new MySQLToStringVisitor(); - visitor.visit(selectStatement); - return new QueryAdapter(visitor.get(), errors); + return new QueryAdapter(MySQLVisitor.asString(selectStatement), errors); } private List generateGroupByClause(List columns, MySQLRowValue rw) { @@ -121,7 +118,7 @@ private MySQLExpression generateWhereClauseThatContainsRowValue(List orderBy = new PostgresExpressionGenerator(globalState).setColumns(columns) .generateOrderBy(); selectStatement.setOrderByExpressions(orderBy); - PostgresToStringVisitor visitor = new PostgresToStringVisitor(); - visitor.visit(selectStatement); - return new QueryAdapter(visitor.get()); + return new QueryAdapter(PostgresVisitor.asString(selectStatement)); } public PostgresExpression generateTrueCondition(List columns, PostgresRowValue rw, @@ -128,7 +125,7 @@ private PostgresExpression generateWhereClauseThatContainsRowValue(List Date: Thu, 27 Aug 2020 22:37:04 +0200 Subject: [PATCH 0237/1284] Fix the formatting of the comments that are printed when PQS detects a bug --- src/sqlancer/Main.java | 6 +----- .../common/oracle/PivotedQuerySynthesisBase.java | 6 +++--- src/sqlancer/common/schema/AbstractRowValue.java | 16 +++++++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index b92821b86..52b41c6d5 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -229,11 +229,7 @@ private void printState(FileWriter writer, StateToReproduce state) { sb.append("-- Database version: " + state.getDatabaseVersion() + "\n"); sb.append("-- seed value: " + state.getSeedValue() + "\n"); for (Query s : state.getStatements()) { - if (s.getQueryString().endsWith(";")) { - sb.append(s.getQueryString()); - } else { - sb.append(s.getQueryString() + ";"); - } + sb.append(s.getQueryString()); sb.append('\n'); } try { diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index 2a2f2e7d1..c1a9a1f96 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -17,7 +17,7 @@ public abstract class PivotedQuerySynthesisBase, R e protected final ExpectedErrors errors = new ExpectedErrors(); /** - * The predicates used in WHERE and JOIN clauses, which yield TRUE for the pivot row + * The predicates used in WHERE and JOIN clauses, which yield TRUE for the pivot row. */ protected final List rectifiedPredicates = new ArrayList<>(); protected final S globalState; @@ -66,11 +66,11 @@ private boolean containsRows(Query query) throws SQLException { } protected void reportMissingPivotRow(Query query) { - globalState.getState().getLocalState().log("-- " + "pivot row values:\n"); + globalState.getState().getLocalState().log("-- pivot row values:"); String expectedPivotRowString = pivotRow.asStringGroupedByTables(); globalState.getState().getLocalState().log(expectedPivotRowString); - StringBuilder sb = new StringBuilder("-- rectified predicates and their expected values:\n"); + StringBuilder sb = new StringBuilder("--\n-- rectified predicates and their expected values:\n"); for (E rectifiedPredicate : rectifiedPredicates) { sb.append("--"); sb.append(getExpectedValues(rectifiedPredicate).replace("\n", "\n-- ")); diff --git a/src/sqlancer/common/schema/AbstractRowValue.java b/src/sqlancer/common/schema/AbstractRowValue.java index 29bfb0c86..7714b860b 100644 --- a/src/sqlancer/common/schema/AbstractRowValue.java +++ b/src/sqlancer/common/schema/AbstractRowValue.java @@ -58,16 +58,22 @@ public String asStringGroupedByTables() { List columnList = getValues().keySet().stream().collect(Collectors.toList()); List> tableList = columnList.stream().map(c -> c.getTable()).distinct().sorted() .collect(Collectors.toList()); - for (AbstractTable t : tableList) { + for (int j = 0; j < tableList.size(); j++) { + if (j != 0) { + sb.append("\n"); + } + AbstractTable t = tableList.get(j); sb.append("-- " + t.getName() + "\n"); List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) .collect(Collectors.toList()); - for (C c : columnsForTable) { + for (int i = 0; i < columnsForTable.size(); i++) { + if (i != 0) { + sb.append("\n"); + } sb.append("--\t"); - sb.append(c); + sb.append(columnsForTable.get(i)); sb.append("="); - sb.append(getValues().get(c)); - sb.append("\n"); + sb.append(getValues().get(columnsForTable.get(i))); } } return sb.toString(); From a5482da6f3543be5ce9a93561ed769869e9cec25 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 22:49:39 +0200 Subject: [PATCH 0238/1284] [SQLite] Also print the expected values of the pivot row expressions --- .../oracle/PivotedQuerySynthesisBase.java | 26 ++++++++++++++++--- .../SQLite3PivotedQuerySynthesisOracle.java | 1 - 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index c1a9a1f96..f4ea9a191 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -20,6 +20,12 @@ public abstract class PivotedQuerySynthesisBase, R e * The predicates used in WHERE and JOIN clauses, which yield TRUE for the pivot row. */ protected final List rectifiedPredicates = new ArrayList<>(); + + /** + * The generalization of a pivot row, as explained in the "Checking arbitrary expressions" paragraph of the PQS + * paper. + */ + protected List pivotRowExpression = new ArrayList<>(); protected final S globalState; protected R pivotRow; @@ -70,10 +76,22 @@ protected void reportMissingPivotRow(Query query) { String expectedPivotRowString = pivotRow.asStringGroupedByTables(); globalState.getState().getLocalState().log(expectedPivotRowString); - StringBuilder sb = new StringBuilder("--\n-- rectified predicates and their expected values:\n"); - for (E rectifiedPredicate : rectifiedPredicates) { - sb.append("--"); - sb.append(getExpectedValues(rectifiedPredicate).replace("\n", "\n-- ")); + StringBuilder sb = new StringBuilder(); + if (!rectifiedPredicates.isEmpty()) { + sb.append("--\n-- rectified predicates and their expected values:\n"); + for (E rectifiedPredicate : rectifiedPredicates) { + sb.append("--"); + sb.append(getExpectedValues(rectifiedPredicate).replace("\n", "\n-- ")); + } + sb.append("\n"); + } + if (!pivotRowExpression.isEmpty()) { + sb.append("-- pivot row expressions and their expected values:\n"); + for (E pivotRowExpression : pivotRowExpression) { + sb.append("--"); + sb.append(getExpectedValues(pivotRowExpression).replace("\n", "\n--")); + sb.append("\n"); + } } globalState.getState().getLocalState().log(sb.toString()); throw new AssertionError(query); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index a7de0d431..baf1d210a 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -44,7 +44,6 @@ public class SQLite3PivotedQuerySynthesisOracle extends PivotedQuerySynthesisBase { private List fetchColumns; - private List pivotRowExpression; private OracleRunReproductionState localState; public SQLite3PivotedQuerySynthesisOracle(SQLite3GlobalState globalState) { From f2e057f3352d91e07cb9f24fb267e8e654cd3bc9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 22:50:48 +0200 Subject: [PATCH 0239/1284] [SQLite] Remove a duplicate argument --- .../oracle/SQLite3PivotedQuerySynthesisOracle.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index baf1d210a..85ee07494 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -84,7 +84,7 @@ public SQLite3Select getQuery() throws SQLException { allTables.addAll(joinStatements.stream().map(join -> join.getTable()).collect(Collectors.toList())); boolean allTablesContainOneRow = allTables.stream().allMatch(t -> t.getNrRows(globalState) == 1); boolean testAggregateFunctions = allTablesContainOneRow && globalState.getOptions().testAggregateFunctionsPQS(); - pivotRowExpression = getColExpressions(testAggregateFunctions, columnsWithoutRowid, columnsWithoutRowid); + pivotRowExpression = getColExpressions(testAggregateFunctions, columnsWithoutRowid); selectStatement.setFetchColumns(pivotRowExpression); SQLite3Expression whereClause = generateRectifiedExpression(columnsWithoutRowid, pivotRow, false); selectStatement.setWhereClause(whereClause); @@ -122,8 +122,7 @@ private List getJoinStatements(SQLite3GlobalState globalState, List getColExpressions(boolean testAggregateFunctions, List columns, - List columnsWithoutRowid) { + private List getColExpressions(boolean testAggregateFunctions, List columns) { List colExpressions = new ArrayList<>(); for (SQLite3Column c : fetchColumns) { @@ -156,8 +155,8 @@ private List getColExpressions(boolean testAggregateFunctions } } if (testAggregateFunctions) { - SQLite3WindowFunction windowFunction = SQLite3WindowFunction.getRandom(columnsWithoutRowid, globalState); - SQLite3Expression windowExpr = generateWindowFunction(columnsWithoutRowid, windowFunction, false); + SQLite3WindowFunction windowFunction = SQLite3WindowFunction.getRandom(columns, globalState); + SQLite3Expression windowExpr = generateWindowFunction(columns, windowFunction, false); colExpressions.add(windowExpr); } for (SQLite3Expression expr : colExpressions) { From 9b1f962c3cb0d66fcd8c0c593618036d3f4de664 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 23:01:15 +0200 Subject: [PATCH 0240/1284] Remove obsolete comments --- .../mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 6 +----- .../oracle/PostgresPivotedQuerySynthesisOracle.java | 1 - .../sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java | 6 +----- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index b73ae1f87..1e9e101a7 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -63,10 +63,7 @@ public Query getRectifiedQuery() throws SQLException { MySQLExpression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } - List modifiers = Randomly.subset("STRAIGHT_JOIN", "SQL_SMALL_RESULT", "SQL_BIG_RESULT", "SQL_NO_CACHE"); // "SQL_BUFFER_RESULT", - // "SQL_CALC_FOUND_ROWS", - // "HIGH_PRIORITY" - // TODO: Incorrect usage/placement of 'SQL_BUFFER_RESULT' + List modifiers = Randomly.subset("STRAIGHT_JOIN", "SQL_SMALL_RESULT", "SQL_BIG_RESULT", "SQL_NO_CACHE"); selectStatement.setModifiers(modifiers); List orderBy = new MySQLExpressionGenerator(globalState).setColumns(columns) .generateOrderBys(); @@ -94,7 +91,6 @@ private MySQLConstant generateLimit() { private MySQLExpression generateOffset() { if (Randomly.getBoolean()) { - // OFFSET 0 return MySQLConstant.createIntConstantNotAsBoolean(0); } else { return null; diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 8aae2df62..eaf0a0abe 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -112,7 +112,6 @@ private PostgresConstant generateLimit() { private PostgresExpression generateOffset() { if (Randomly.getBoolean()) { - // OFFSET 0 return PostgresConstant.createIntConstant(0); } else { return null; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 85ee07494..f746eecac 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -69,7 +69,6 @@ public SQLite3Select getQuery() throws SQLException { SQLite3Select selectStatement = new SQLite3Select(); selectStatement.setSelectType(Randomly.fromOptions(SQLite3Select.SelectType.values())); List columns = randomFromTables.getColumns(); - // TODO: also implement a wild-card check (*) // filter out row ids from the select because the hinder the reduction process // once a bug is found List columnsWithoutRowid = columns.stream() @@ -161,7 +160,7 @@ private List getColExpressions(boolean testAggregateFunctions } for (SQLite3Expression expr : colExpressions) { if (expr.getExpectedValue() == null) { - throw new IgnoreMeException(); // TODO: aggregates + throw new IgnoreMeException(); } } return colExpressions; @@ -169,7 +168,6 @@ private List getColExpressions(boolean testAggregateFunctions private SQLite3Expression generateOffset() { if (Randomly.getBoolean()) { - // OFFSET 0 return SQLite3Constant.createIntConstant(0); } else { return null; @@ -310,10 +308,8 @@ private SQLite3Expression generateWindowFunction(List columns, SQ default: throw new AssertionError(); } - // sb.append(" BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING"); if (Randomly.getBoolean()) { sb.append(" EXCLUDE "); - // "CURRENT ROW", "GROUP" sb.append(Randomly.fromOptions("NO OTHERS", "TIES")); } } From ad3c4ab17b5a818d14182dae53b296b22a76b409 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 23:04:45 +0200 Subject: [PATCH 0241/1284] Unify the name of the method for generating the rectified expression --- .../MySQLPivotedQuerySynthesisOracle.java | 4 +-- .../PostgresPivotedQuerySynthesisOracle.java | 34 ++++++++----------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 1e9e101a7..68cb39b2e 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -53,7 +53,7 @@ public Query getRectifiedQuery() throws SQLException { fetchColumns = columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); selectStatement.setFetchColumns(fetchColumns); - MySQLExpression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); + MySQLExpression whereClause = generateRectifiedExpression(columns, pivotRow); selectStatement.setWhereClause(whereClause); List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); @@ -97,7 +97,7 @@ private MySQLExpression generateOffset() { } } - private MySQLExpression generateWhereClauseThatContainsRowValue(List columns, MySQLRowValue rw) { + private MySQLExpression generateRectifiedExpression(List columns, MySQLRowValue rw) { MySQLExpression expression = new MySQLExpressionGenerator(globalState).setRowVal(rw).setColumns(columns) .generateExpression(); MySQLConstant expectedValue = expression.getExpectedValue(); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index eaf0a0abe..77b86eb9c 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -51,7 +51,7 @@ public Query getRectifiedQuery() throws SQLException { selectStatement.setFetchColumns(fetchColumns.stream() .map(c -> new PostgresColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) .collect(Collectors.toList())); - PostgresExpression whereClause = generateWhereClauseThatContainsRowValue(columns, pivotRow); + PostgresExpression whereClause = generateRectifiedExpression(columns, pivotRow); selectStatement.setWhereClause(whereClause); List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); @@ -67,22 +67,6 @@ public Query getRectifiedQuery() throws SQLException { return new QueryAdapter(PostgresVisitor.asString(selectStatement)); } - public PostgresExpression generateTrueCondition(List columns, PostgresRowValue rw, - PostgresGlobalState globalState) { - PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) - .generateExpressionWithExpectedResult(PostgresDataType.BOOLEAN); - PostgresExpression result; - if (expr.getExpectedValue().isNull()) { - result = PostgresPostfixOperation.create(expr, PostfixOperator.IS_NULL); - } else { - result = PostgresPostfixOperation.create(expr, - expr.getExpectedValue().cast(PostgresDataType.BOOLEAN).asBoolean() ? PostfixOperator.IS_TRUE - : PostfixOperator.IS_FALSE); - } - rectifiedPredicates.add(result); - return result; - } - /* * Prevent name collisions by aliasing the column. */ @@ -118,9 +102,19 @@ private PostgresExpression generateOffset() { } } - private PostgresExpression generateWhereClauseThatContainsRowValue(List columns, - PostgresRowValue rw) { - return generateTrueCondition(columns, rw, globalState); + private PostgresExpression generateRectifiedExpression(List columns, PostgresRowValue rw) { + PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) + .generateExpressionWithExpectedResult(PostgresDataType.BOOLEAN); + PostgresExpression result; + if (expr.getExpectedValue().isNull()) { + result = PostgresPostfixOperation.create(expr, PostfixOperator.IS_NULL); + } else { + result = PostgresPostfixOperation.create(expr, + expr.getExpectedValue().cast(PostgresDataType.BOOLEAN).asBoolean() ? PostfixOperator.IS_TRUE + : PostfixOperator.IS_FALSE); + } + rectifiedPredicates.add(result); + return result; } @Override From 2f5ebb2726e86bdc538c8e5b34d15fc425736cca Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 27 Aug 2020 23:45:17 +0200 Subject: [PATCH 0242/1284] Update the PQS bib entry --- docs/PAPERS.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/PAPERS.md b/docs/PAPERS.md index dcd0162ed..704608b61 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -8,10 +8,13 @@ This paper describes PQS, a testing approach to detect various kinds of logic bu ``` @inproceedings{Rigger2020PQS, - author={Manuel Rigger and Zhendong Su}, - title={Testing Database Engines via Pivoted Query Synthesis}, - year={2020}, - url={https://arxiv.org/abs/2001.04174} + title = {Testing Database Engines via Pivoted Query Synthesis}, + booktitle = {14th {USENIX} Symposium on Operating Systems Design and Implementation ({OSDI} 20)}, + year = {2020}, + address = {Banff, Alberta}, + url = {https://www.usenix.org/conference/osdi20/presentation/rigger}, + publisher = {{USENIX} Association}, + month = nov, } ``` From f842a75a48afcb627482ff1a40aa8ad05613b0e4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 28 Aug 2020 00:37:13 +0200 Subject: [PATCH 0243/1284] [Postgres] Fix a PQS issue --- src/sqlancer/postgres/ast/PostgresConstant.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/ast/PostgresConstant.java b/src/sqlancer/postgres/ast/PostgresConstant.java index b60e8a96b..c898ff95e 100644 --- a/src/sqlancer/postgres/ast/PostgresConstant.java +++ b/src/sqlancer/postgres/ast/PostgresConstant.java @@ -390,18 +390,17 @@ public abstract static class PostgresConstantBase extends PostgresConstant { @Override public String getUnquotedTextRepresentation() { - throw new AssertionError(); + return null; } @Override public PostgresConstant isEquals(PostgresConstant rightVal) { - throw new AssertionError(); + return null; } @Override protected PostgresConstant isLessThan(PostgresConstant rightVal) { - throw new AssertionError(); - + return null; } @Override From b1816d994661270717fc9e48e75204bc49b2d882 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 28 Aug 2020 09:53:58 +0200 Subject: [PATCH 0244/1284] [MySQL] Add an expected error to the CREATE TABLE statement --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index e514cfbf7..eddf39b40 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -80,7 +80,6 @@ private Query create() { } else if ((tableHasNullableColumn || keysSpecified > 1) && engine == MySQLEngine.ARCHIVE) { errors.add("Too many keys specified; max 1 keys allowed"); errors.add("Table handler doesn't support NULL in given index"); - errors.add("Got error -1 - 'Unknown error -1' from storage engine"); addCommonErrors(errors); return new QueryAdapter(sb.toString(), errors, true); } @@ -99,6 +98,7 @@ private void addCommonErrors(ExpectedErrors list) { list.add("A BLOB field is not allowed in partition function"); list.add("Too many keys specified; max 1 keys allowed"); list.add("The total length of the partitioning fields is too large"); + list.add("Got error -1 - 'Unknown error -1' from storage engine"); } private enum PartitionOptions { From 3ef97c28858c5e5285fbd983446d7b4f5b14c712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 28 Aug 2020 13:03:01 +0200 Subject: [PATCH 0245/1284] decimal support for sqlancer and duckdb --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + src/sqlancer/duckdb/DuckDBSchema.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 751c30ce0..91cb6b0c7 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -87,6 +87,7 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if // the table has a primary key + errors.add("Out of Range: Could not cast value"); } public static void addGroupByErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index aea5839da..b743420c6 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -162,6 +162,9 @@ public DuckDBTables getRandomTableNonEmptyTables() { private static DuckDBCompositeDataType getColumnType(String typeString) { DuckDBDataType primitiveType; int size = -1; + if (typeString.startsWith("DECIMAL")) { // Ugly hack + return new DuckDBCompositeDataType(DuckDBDataType.FLOAT, 8); + } switch (typeString) { case "INTEGER": primitiveType = DuckDBDataType.INT; From 81ce28dc0b2fb18471098e9451e00b35e9333897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 28 Aug 2020 13:06:59 +0200 Subject: [PATCH 0246/1284] another error message suppressed --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 91cb6b0c7..50d781cc8 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -47,6 +47,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("does not have a column named"); // TODO: this only happens for views whose underlying table has a // removed column errors.add("Contents of view were altered: types don't match!"); + errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); } private static void addRegexErrors(ExpectedErrors errors) { From ddd0461dfdfb2ed811eb10a5fe2215f9e3a598ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 28 Aug 2020 13:14:26 +0200 Subject: [PATCH 0247/1284] increased epsilon for double comparisions --- src/sqlancer/ComparatorHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index e60c0a46c..6e87900dc 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -32,7 +32,7 @@ static boolean equals(double a, double b) { return true; } // If the difference is less than epsilon, treat as equal. - return Math.abs(a - b) < 0.0001 * Math.max(Math.abs(a), Math.abs(b)); + return Math.abs(a - b) < 0.001 * Math.max(Math.abs(a), Math.abs(b)); } public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, From 69f58f7de041b2c4377a04260665a711b4f7af2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Fri, 28 Aug 2020 13:23:01 +0200 Subject: [PATCH 0248/1284] more epsilon fun --- src/sqlancer/ComparatorHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 6e87900dc..3f595a721 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -32,7 +32,7 @@ static boolean equals(double a, double b) { return true; } // If the difference is less than epsilon, treat as equal. - return Math.abs(a - b) < 0.001 * Math.max(Math.abs(a), Math.abs(b)); + return Math.abs(a - b) < 0.001 * Math.max(Math.abs(a), Math.abs(b)) + 0.001; } public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, From b0354dee9431d1a02fa666e1ce2abe883e1b5d7a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 17 Jul 2020 00:07:01 +0200 Subject: [PATCH 0249/1284] Provide a useful error message when SQLancer fails to connect to the DBMS --- src/sqlancer/Main.java | 38 +++++++++++++++++++ src/sqlancer/MainOptions.java | 7 ++++ .../postgres/PostgresGlobalState.java | 7 ++-- src/sqlancer/postgres/PostgresProvider.java | 12 ++---- 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 52b41c6d5..47c7bc442 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -316,6 +316,13 @@ public O getCommand() { return command; } + public void testConnection() throws SQLException { + G state = getInitializedGlobalState(options.getRandomSeed()); + try (Connection con = provider.createDatabase(state)) { + return; + } + } + public void run() throws SQLException { G state = createGlobalState(); stateToRepro = provider.getStateToReproduce(databaseName); @@ -350,6 +357,20 @@ public void run() throws SQLException { } } + private G getInitializedGlobalState(long seed) { + G state = createGlobalState(); + stateToRepro = provider.getStateToReproduce(databaseName); + stateToRepro.seedValue = seed; + state.setState(stateToRepro); + logger = new StateLogger(databaseName, provider, options); + Randomly r = new Randomly(seed); + state.setRandomly(r); + state.setDatabaseName(databaseName); + state.setMainOptions(options); + state.setDmbsSpecificOptions(command); + return state; + } + public StateLogger getLogger() { return logger; } @@ -393,6 +414,10 @@ public DBMSExecutor getDBMSExecutor(String databaseName, Randomly r) { } } + public DatabaseProvider getProvider() { + return provider; + } + } public static int executeMain(String... args) throws AssertionError { @@ -448,6 +473,19 @@ private String formatInteger(long intValue) { ExecutorService execService = Executors.newFixedThreadPool(options.getNumberConcurrentThreads()); DBMSExecutorFactory executorFactory = nameToProvider.get(jc.getParsedCommand()); + + if (options.performConnectionTest()) { + try { + executorFactory.getDBMSExecutor(options.getDatabasePrefix() + "connectiontest", new Randomly()) + .testConnection(); + } catch (SQLException e) { + System.err.println( + "SQLancer failed creating a test database, indicating that SQLancer might have failed connecting to the DBMS. In order to change the username and password, you can use the --username and --password options. Currently, SQLancer does not yet support passing a host and port (see https://github.com/sqlancer/sqlancer/issues/95).\n\n"); + e.printStackTrace(); + return options.getErrorExitCode(); + } + } + for (int i = 0; i < options.getTotalNumberTries(); i++) { final String databaseName = options.getDatabasePrefix() + i; final long seed; diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index f61c9adb5..45867edb4 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -85,6 +85,9 @@ public class MainOptions { @Parameter(names = "--use-constant-caching", description = "Specifies whether constants should be cached and re-used with a certain probability") private boolean useConstantCaching = true; // NOPMD + @Parameter(names = "--use-connection-test", description = "Test whether the DBMS is accessible before trying to connect using multiple threads") + private boolean useConnectionTest = true; // NOPMD + @Parameter(names = "--constant-cache-size", description = "Specifies the size of the constant cache. This option only takes effect when constant caching is enabled") private int constantCacheSize = 100; // NOPMD @@ -204,4 +207,8 @@ public String getDatabasePrefix() { return databasePrefix; } + public boolean performConnectionTest() { + return useConnectionTest; + } + } diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index b3ec5cfd6..deb342b40 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -6,6 +6,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -19,9 +20,9 @@ public class PostgresGlobalState extends GlobalState operators; - private List collates; - private List opClasses; + private List operators = Collections.emptyList(); + private List collates = Collections.emptyList(); + private List opClasses = Collections.emptyList(); // store and allow filtering by function volatility classifications private final Map functionsAndTypes = new HashMap<>(); private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 52312699c..0bc9596b0 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -50,8 +50,6 @@ public class PostgresProvider extends ProviderAdapter Date: Thu, 27 Aug 2020 23:47:00 +0200 Subject: [PATCH 0250/1284] Delete unused methods Detected using http://www.ucdetector.org/. --- src/sqlancer/Main.java | 15 --------------- src/sqlancer/Randomly.java | 4 ---- src/sqlancer/StateToReproduce.java | 10 ---------- src/sqlancer/citus/gen/CitusCommon.java | 9 --------- .../cockroachdb/ast/CockroachDBAggregate.java | 4 ---- .../CockroachDBUnaryArithmeticOperation.java | 5 ----- .../ast/CockroachDBUnaryPostfixOperation.java | 5 ----- src/sqlancer/common/schema/AbstractTable.java | 5 ----- .../common/schema/AbstractTables.java | 5 ----- src/sqlancer/duckdb/DuckDBSchema.java | 13 ------------- src/sqlancer/mariadb/ast/MariaDBConstant.java | 4 ---- src/sqlancer/mysql/MySQLProvider.java | 11 ----------- src/sqlancer/mysql/MySQLSchema.java | 4 ---- src/sqlancer/mysql/ast/MySQLConstant.java | 10 ---------- .../mysql/gen/MySQLInsertGenerator.java | 4 ---- src/sqlancer/postgres/PostgresVisitor.java | 7 ------- .../postgres/ast/PostgresAggregate.java | 8 -------- .../postgres/ast/PostgresColumnValue.java | 4 ---- .../postgres/ast/PostgresFunction.java | 19 ------------------- .../gen/PostgresExpressionGenerator.java | 19 ------------------- .../SQLite3PivotedQuerySynthesisOracle.java | 1 - 21 files changed, 166 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 47c7bc442..892df5bbb 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -186,15 +186,6 @@ public void writeCurrentNoLineBreak(String queryString) { write(queryString, ""); } - public void logRowNotFound(StateToReproduce state) { - printState(getLogFileWriter(), state); - try { - getLogFileWriter().flush(); - } catch (IOException e) { - throw new AssertionError(e); - } - } - public void logException(Throwable reduce, StateToReproduce state) { String stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); @@ -275,12 +266,6 @@ public void incrementCreateDatabase() { } - public static void printArray(Object... arr) { - for (Object o : arr) { - System.out.println(o); - } - } - public static void main(String[] args) { System.exit(executeMain(args)); } diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 171356839..c26e8aa17 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -437,10 +437,6 @@ public static long getNonCachedInteger() { return getThreadRandom().get().nextLong(); } - public static long getPositiveNonCachedInteger() { - return getNextLong(1, Long.MAX_VALUE); - } - public static long getPositiveOrZeroNonCachedInteger() { return getNextLong(0, Long.MAX_VALUE); } diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index ea3523624..e77d8a400 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -18,14 +18,8 @@ public class StateToReproduce { protected long seedValue; - public String values; - String exception; - public String queryTargetedTablesString; - - public String queryTargetedColumnsString; - public OracleRunReproductionState localState; public StateToReproduce(String databaseName) { @@ -114,10 +108,6 @@ public void executedWithoutError() { this.success = true; } - public void log(Query q) { - statements.add(q); - } - public void log(String s) { statements.add(new QueryAdapter(s)); } diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index a5a4ca200..f342d89bf 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -2,9 +2,6 @@ import sqlancer.citus.CitusBugs; import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.gen.PostgresCommon; public final class CitusCommon { @@ -73,10 +70,4 @@ public static void addCitusErrors(ExpectedErrors errors) { } } - public static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, - ExpectedErrors errors) { - PostgresCommon.addTableConstraint(sb, table, globalState, errors); - CitusCommon.addCitusErrors(errors); - } - } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBAggregate.java b/src/sqlancer/cockroachdb/ast/CockroachDBAggregate.java index 0bddde8af..b7413b6d6 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBAggregate.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBAggregate.java @@ -78,10 +78,6 @@ public static List getAggregates(CockroachDBDataTy this.supportedReturnTypes = supportedReturnTypes.clone(); } - public static CockroachDBAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - public static CockroachDBAggregateFunction getRandomMetamorphicOracle() { // not: VARIANCE, STDDEV, SQRDIFF return Randomly.fromOptions(SUM, SUM_INT, MIN, MAX, XOR_AGG, BIT_AND, BIT_OR, BOOL_AND, BOOL_OR, COUNT, AVG, diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java index edd6b3c70..b880ea67d 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java @@ -1,6 +1,5 @@ package sqlancer.cockroachdb.ast; -import sqlancer.Randomly; import sqlancer.cockroachdb.ast.CockroachDBUnaryArithmeticOperation.CockroachDBUnaryAritmeticOperator; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.UnaryOperatorNode; @@ -17,10 +16,6 @@ public enum CockroachDBUnaryAritmeticOperator implements Operator { this.textRepr = textRepr; } - public static CockroachDBUnaryAritmeticOperator getRandom() { - return Randomly.fromOptions(values()); - } - @Override public String getTextRepresentation() { return textRepr; diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java index fddc2c315..0b28e4e88 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java @@ -1,6 +1,5 @@ package sqlancer.cockroachdb.ast; -import sqlancer.Randomly; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.UnaryOperatorNode; @@ -21,10 +20,6 @@ public enum CockroachDBUnaryPostfixOperator implements Operator { this.s = s; } - public static CockroachDBUnaryPostfixOperator getRandom() { - return Randomly.fromOptions(values()); - } - @Override public String getTextRepresentation() { return s; diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 74f3ee7da..3c204c908 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -2,7 +2,6 @@ import java.util.Collections; import java.util.List; -import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.GlobalState; @@ -60,10 +59,6 @@ public String getColumnsAsString() { return columns.stream().map(c -> c.getName()).collect(Collectors.joining(", ")); } - public String getColumnsAsString(Function function) { - return columns.stream().map(function).collect(Collectors.joining(", ")); - } - public C getRandomColumn() { return Randomly.fromList(columns); } diff --git a/src/sqlancer/common/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java index c7f44e688..ad80011b0 100644 --- a/src/sqlancer/common/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -30,11 +30,6 @@ public List getColumns() { return columns; } - public String columnNamesAsString() { - return getColumns().stream().map(t -> t.getTable().getName() + "." + t.getName()) - .collect(Collectors.joining(", ")); - } - public String columnNamesAsString(Function function) { return getColumns().stream().map(function).collect(Collectors.joining(", ")); } diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index b743420c6..b0c153625 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -34,11 +34,6 @@ public static class DuckDBCompositeDataType { private final int size; - public DuckDBCompositeDataType(DuckDBDataType dataType) { - this.dataType = dataType; - this.size = -1; - } - public DuckDBCompositeDataType(DuckDBDataType dataType, int size) { this.dataType = dataType; this.size = size; @@ -78,10 +73,6 @@ public static DuckDBCompositeDataType getRandom() { return new DuckDBCompositeDataType(type, size); } - public static DuckDBCompositeDataType getInt(int size) { - return new DuckDBCompositeDataType(DuckDBDataType.INT, size); - } - @Override public String toString() { switch (getPrimitiveDataType()) { @@ -214,10 +205,6 @@ public DuckDBTable(String tableName, List columns, boolean isView) super(tableName, columns, Collections.emptyList(), isView); } - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - } public static DuckDBSchema fromConnection(Connection con, String databaseName) throws SQLException { diff --git a/src/sqlancer/mariadb/ast/MariaDBConstant.java b/src/sqlancer/mariadb/ast/MariaDBConstant.java index 499fe9ded..6e3d77691 100644 --- a/src/sqlancer/mariadb/ast/MariaDBConstant.java +++ b/src/sqlancer/mariadb/ast/MariaDBConstant.java @@ -91,10 +91,6 @@ public static MariaDBTextConstant createTextConstant(String text) { return new MariaDBTextConstant(text); } - public static MariaDBDoubleConstant createDoubleConstant(double val) { - return new MariaDBDoubleConstant(val); - } - public static MariaDBIntConstant createIntConstant(long val) { return new MariaDBIntConstant(val); } diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 3eef6a739..fcdd31672 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -2,7 +2,6 @@ import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -14,7 +13,6 @@ import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.QueryProvider; -import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; import sqlancer.mysql.gen.MySQLDeleteGenerator; import sqlancer.mysql.gen.MySQLDropIndex; @@ -149,15 +147,6 @@ public void generateDatabase(MySQLGlobalState globalState) throws SQLException { se.executeStatements(); } - public static int getNrRows(Connection con, MySQLTable table) throws SQLException { - try (Statement s = con.createStatement()) { - try (ResultSet query = s.executeQuery("SELECT COUNT(*) FROM " + table.getName())) { - query.next(); - return query.getInt(1); - } - } - } - @Override public Connection createDatabase(MySQLGlobalState globalState) throws SQLException { String databaseName = globalState.getDatabaseName(); diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index a23ad7990..112cc72b0 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -177,10 +177,6 @@ public enum MySQLEngine { this.s = s; } - String getTextRepresentation() { - return s; - } - public static MySQLEngine get(String val) { return Stream.of(values()).filter(engine -> engine.s.equalsIgnoreCase(val)).findFirst().get(); } diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index 6988f74c0..11a807841 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -441,16 +441,6 @@ public String toString() { public abstract MySQLConstant isEquals(MySQLConstant rightVal); - public MySQLConstant isEqualsNullSafe(MySQLConstant rightVal) { - if (isNull()) { - return MySQLConstant.createBoolean(rightVal.isNull()); - } else if (rightVal.isNull()) { - return MySQLConstant.createFalse(); - } else { - return isEquals(rightVal); - } - } - public abstract MySQLConstant castAs(CastType type); public abstract String castAsString(); diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index d7597e7e2..788baa5ca 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -17,7 +17,6 @@ public class MySQLInsertGenerator { private final MySQLTable table; private final StringBuilder sb = new StringBuilder(); - boolean canFail; private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; @@ -35,7 +34,6 @@ public static Query insertRow(MySQLGlobalState globalState) throws SQLException } private Query generateReplace() { - canFail = true; sb.append("REPLACE"); if (Randomly.getBoolean()) { sb.append(" "); @@ -53,8 +51,6 @@ private Query generateInsert() { } if (Randomly.getBoolean()) { sb.append(" IGNORE"); - } else { - canFail = true; } return generateInto(); } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 7b3bf2853..d66dd1271 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -120,13 +120,6 @@ static String asExpectedValues(PostgresExpression expr) { return v.get(); } - static String getExpressionAsString(PostgresGlobalState globalState, PostgresDataType type) { - PostgresExpression expression = PostgresExpressionGenerator.generateExpression(globalState, type); - PostgresToStringVisitor visitor = new PostgresToStringVisitor(); - visitor.visit(expression); - return visitor.get(); - } - static String getExpressionAsString(PostgresGlobalState globalState, PostgresDataType type, List columns) { PostgresExpression expression = PostgresExpressionGenerator.generateExpression(globalState, columns, type); diff --git a/src/sqlancer/postgres/ast/PostgresAggregate.java b/src/sqlancer/postgres/ast/PostgresAggregate.java index bcf48d46e..087431756 100644 --- a/src/sqlancer/postgres/ast/PostgresAggregate.java +++ b/src/sqlancer/postgres/ast/PostgresAggregate.java @@ -28,14 +28,6 @@ public enum PostgresAggregateFunction { this.supportedReturnTypes = supportedReturnTypes.clone(); } - public static PostgresAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public static PostgresAggregateFunction getRandom(PostgresDataType type) { - return Randomly.fromOptions(values()); - } - public List getTypes(PostgresDataType returnType) { return Arrays.asList(returnType); } diff --git a/src/sqlancer/postgres/ast/PostgresColumnValue.java b/src/sqlancer/postgres/ast/PostgresColumnValue.java index ccdd5137d..0974ebea8 100644 --- a/src/sqlancer/postgres/ast/PostgresColumnValue.java +++ b/src/sqlancer/postgres/ast/PostgresColumnValue.java @@ -23,10 +23,6 @@ public PostgresConstant getExpectedValue() { return expectedValue; } - public static PostgresColumnValue create(PostgresColumn c) { - return new PostgresColumnValue(c, null); - } - public static PostgresColumnValue create(PostgresColumn c, PostgresConstant expected) { return new PostgresColumnValue(c, expected); } diff --git a/src/sqlancer/postgres/ast/PostgresFunction.java b/src/sqlancer/postgres/ast/PostgresFunction.java index 24a1b3426..5fe2968ab 100644 --- a/src/sqlancer/postgres/ast/PostgresFunction.java +++ b/src/sqlancer/postgres/ast/PostgresFunction.java @@ -1,6 +1,5 @@ package sqlancer.postgres.ast; -import sqlancer.Randomly; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresFunction implements PostgresExpression { @@ -225,26 +224,12 @@ public PostgresDataType[] getRandomTypes(int nr) { return types; } - public PostgresDataType[] getType(int nr, PostgresDataType type) { - PostgresDataType[] types = new PostgresDataType[nr]; - for (int i = 0; i < types.length; i++) { - types[i] = type; - } - return types; - } - PostgresFunctionWithResult(int nrArgs, String functionName) { this.nrArgs = nrArgs; this.functionName = functionName; this.variadic = false; } - PostgresFunctionWithResult(int nrArgs, String functionName, boolean variadic) { - this.nrArgs = nrArgs; - this.functionName = functionName; - this.variadic = variadic; - } - /** * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number * of arguments is returned. @@ -257,10 +242,6 @@ public int getNrArgs() { public abstract PostgresConstant apply(PostgresConstant[] evaluatedArgs, PostgresExpression... args); - public static PostgresFunctionWithResult getRandomFunction() { - return Randomly.fromOptions(values()); - } - @Override public String toString() { return functionName; diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index eb5d772bb..010bb8fec 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -89,15 +89,6 @@ public PostgresExpressionGenerator setRowValue(PostgresRowValue rw) { return this; } - public PostgresExpressionGenerator expectedResult() { - this.expectedResult = true; - return this; - } - - public static PostgresExpression generateExpression(PostgresGlobalState globalState) { - return new PostgresExpressionGenerator(globalState).generateExpression(0); - } - public PostgresExpression generateExpression(int depth) { return generateExpression(depth, PostgresDataType.getRandomType()); } @@ -471,10 +462,6 @@ final List filterColumns(PostgresDataType type) { } } - public static PostgresExpression generateConstant(Randomly r) { - return generateConstant(r, Randomly.fromOptions(PostgresDataType.values())); - } - public PostgresExpression generateExpressionWithExpectedResult(PostgresDataType type) { this.expectedResult = true; PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns) @@ -546,12 +533,6 @@ public static PostgresExpression generateExpression(PostgresGlobalState globalSt return new PostgresExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); } - public static PostgresExpression generateExpression(PostgresGlobalState globalState, List columns, - PostgresDataType type, PostgresRowValue rw) { - return new PostgresExpressionGenerator(globalState).setColumns(columns).setRowValue(rw).generateExpression(0, - type); - } - public static PostgresExpression generateExpression(PostgresGlobalState globalState, List columns) { return new PostgresExpressionGenerator(globalState).setColumns(columns).generateExpression(0); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index f746eecac..353491985 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -65,7 +65,6 @@ public SQLite3Select getQuery() throws SQLException { List tables = randomFromTables.getTables(); pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); - globalState.getState().queryTargetedTablesString = randomFromTables.tableNamesAsString(); SQLite3Select selectStatement = new SQLite3Select(); selectStatement.setSelectType(Randomly.fromOptions(SQLite3Select.SelectType.values())); List columns = randomFromTables.getColumns(); From 97319446137536ce5ed8dd9ef935b9a226237169 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Sep 2020 14:31:10 +0200 Subject: [PATCH 0251/1284] Delete the unused port variable in PostgresProvider --- src/sqlancer/postgres/PostgresProvider.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 0bc9596b0..3e4f3e793 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -55,7 +55,6 @@ public class PostgresProvider extends ProviderAdapter Date: Mon, 14 Sep 2020 06:49:33 +0000 Subject: [PATCH 0252/1284] Bump junit-jupiter-engine from 5.6.2 to 5.7.0 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.6.2 to 5.7.0. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.6.2...r5.7.0) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bfc1e63d7..4b4392ca4 100644 --- a/pom.xml +++ b/pom.xml @@ -216,7 +216,7 @@ org.junit.jupiter junit-jupiter-engine - 5.6.2 + 5.7.0 test From 639f1e7e97e5fdb02bc5bf08a7916d6463a77810 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 14 Sep 2020 17:47:26 +0200 Subject: [PATCH 0253/1284] [Citus] Add an expected error --- src/sqlancer/citus/gen/CitusCommon.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index f342d89bf..5e26285c9 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -44,6 +44,8 @@ public static void addCitusErrors(ExpectedErrors errors) { "complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator"); errors.add("cannot perform distributed planning on this query"); errors.add("cannot pushdown the subquery"); + // see https://github.com/sqlancer/sqlancer/issues/215 + errors.add("direct joins between distributed and local tables are not supported"); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { From 8310ed1a30eb5ae7e8ebb85a476f899edce1280d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2020 06:43:44 +0000 Subject: [PATCH 0254/1284] Bump jacoco-maven-plugin from 0.8.5 to 0.8.6 Bumps [jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.5 to 0.8.6. - [Release notes](https://github.com/jacoco/jacoco/releases) - [Commits](https://github.com/jacoco/jacoco/compare/v0.8.5...v0.8.6) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4b4392ca4..3fd6d51f6 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ org.jacoco jacoco-maven-plugin - 0.8.5 + 0.8.6 From f36b10987c395b526185da653525e4126f3c1d3a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2020 06:39:24 +0000 Subject: [PATCH 0255/1284] Bump ecj from 3.22.0 to 3.23.0 Bumps ecj from 3.22.0 to 3.23.0. Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3fd6d51f6..290cc15c2 100644 --- a/pom.xml +++ b/pom.xml @@ -55,7 +55,7 @@ org.eclipse.jdt ecj - 3.22.0 + 3.23.0 From d0d0d605f244f3d219d39e84e1821e91ced47334 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 23 Sep 2020 22:07:09 +0200 Subject: [PATCH 0256/1284] Simplify the schema update logic --- src/sqlancer/GlobalState.java | 6 +++++- src/sqlancer/citus/CitusGlobalState.java | 4 ++-- src/sqlancer/clickhouse/ClickHouseProvider.java | 4 ++-- src/sqlancer/cockroachdb/CockroachDBProvider.java | 4 ++-- src/sqlancer/duckdb/DuckDBProvider.java | 4 ++-- src/sqlancer/mariadb/MariaDBProvider.java | 4 ++-- src/sqlancer/mysql/MySQLGlobalState.java | 4 ++-- src/sqlancer/postgres/PostgresGlobalState.java | 4 ++-- src/sqlancer/sqlite3/SQLite3Provider.java | 4 ++-- src/sqlancer/tidb/TiDBProvider.java | 4 ++-- 10 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 7255facb6..a7b8ac475 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -164,6 +164,10 @@ protected void setSchema(S schema) { this.schema = schema; } - protected abstract void updateSchema() throws SQLException; + public void updateSchema() throws SQLException { + setSchema(readSchema()); + } + + protected abstract S readSchema() throws SQLException; } diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java index 225f5134d..28bf193a6 100644 --- a/src/sqlancer/citus/CitusGlobalState.java +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -17,8 +17,8 @@ public boolean getRepartition() { } @Override - public void updateSchema() throws SQLException { - setSchema(CitusSchema.fromConnection(getConnection(), getDatabaseName())); + public CitusSchema readSchema() throws SQLException { + return CitusSchema.fromConnection(getConnection(), getDatabaseName()); } } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 84360093b..947d8c90a 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -74,8 +74,8 @@ public String getDatabaseName() { } @Override - protected void updateSchema() throws SQLException { - setSchema(ClickHouseSchema.fromConnection(getConnection(), getDatabaseName())); + protected ClickHouseSchema readSchema() throws SQLException { + return ClickHouseSchema.fromConnection(getConnection(), getDatabaseName()); } } diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index d171ebfbd..b82d5fcdd 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -113,8 +113,8 @@ public Query getQuery(CockroachDBGlobalState state) throws SQLException { public static class CockroachDBGlobalState extends GlobalState { @Override - protected void updateSchema() throws SQLException { - setSchema(CockroachDBSchema.fromConnection(getConnection(), getDatabaseName())); + protected CockroachDBSchema readSchema() throws SQLException { + return CockroachDBSchema.fromConnection(getConnection(), getDatabaseName()); } } diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 5a8517375..2b757a7f5 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -88,8 +88,8 @@ private static int mapActions(DuckDBGlobalState globalState, Action a) { public static class DuckDBGlobalState extends GlobalState { @Override - protected void updateSchema() throws SQLException { - setSchema(DuckDBSchema.fromConnection(getConnection(), getDatabaseName())); + protected DuckDBSchema readSchema() throws SQLException { + return DuckDBSchema.fromConnection(getConnection(), getDatabaseName()); } } diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 6f515dc72..960fd5ae3 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -155,8 +155,8 @@ public void generateDatabase(MariaDBGlobalState globalState) throws SQLException public static class MariaDBGlobalState extends GlobalState { @Override - protected void updateSchema() throws SQLException { - setSchema(MariaDBSchema.fromConnection(getConnection(), getDatabaseName())); + protected MariaDBSchema readSchema() throws SQLException { + return MariaDBSchema.fromConnection(getConnection(), getDatabaseName()); } } diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 5df20bb50..9b79bf3b4 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -9,8 +9,8 @@ public class MySQLGlobalState extends GlobalState { @Override - protected void updateSchema() throws SQLException { - setSchema(MySQLSchema.fromConnection(getConnection(), getDatabaseName())); + protected MySQLSchema readSchema() throws SQLException { + return MySQLSchema.fromConnection(getConnection(), getDatabaseName()); } public boolean usesPQS() { diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index deb342b40..7d7e56d49 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -101,8 +101,8 @@ public String getRandomOpclass() { } @Override - public void updateSchema() throws SQLException { - setSchema(PostgresSchema.fromConnection(getConnection(), getDatabaseName())); + public PostgresSchema readSchema() throws SQLException { + return PostgresSchema.fromConnection(getConnection(), getDatabaseName()); } public void addFunctionAndType(String functionName, Character functionType) { diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 5c658478a..7616718d7 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -115,8 +115,8 @@ public Query getQuery(SQLite3GlobalState state) throws SQLException { public static class SQLite3GlobalState extends GlobalState { @Override - protected void updateSchema() throws SQLException { - setSchema(SQLite3Schema.fromConnection(this)); + protected SQLite3Schema readSchema() throws SQLException { + return SQLite3Schema.fromConnection(this); } } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 0d59d2bea..3cf7e7590 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -69,8 +69,8 @@ public Query getQuery(TiDBGlobalState state) throws SQLException { public static class TiDBGlobalState extends GlobalState { @Override - protected void updateSchema() throws SQLException { - setSchema(TiDBSchema.fromConnection(getConnection(), getDatabaseName())); + protected TiDBSchema readSchema() throws SQLException { + return TiDBSchema.fromConnection(getConnection(), getDatabaseName()); } } From c4273a7d296b84763f7287d84f1ec8edd9fcf978 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2020 06:52:15 +0000 Subject: [PATCH 0257/1284] Bump mariadb-java-client from 2.6.2 to 2.7.0 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.6.2 to 2.7.0. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/2.6.2...2.7.0) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 290cc15c2..26ba4b8f1 100644 --- a/pom.xml +++ b/pom.xml @@ -206,7 +206,7 @@ org.mariadb.jdbc mariadb-java-client - 2.6.2 + 2.7.0 org.duckdb From 6c4da0a17eec339a6075158eb16f2173dc864f21 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2020 06:38:31 +0000 Subject: [PATCH 0258/1284] Bump spotbugs-maven-plugin from 4.0.4 to 4.1.3 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.0.4 to 4.1.3. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.0.4...spotbugs-maven-plugin-4.1.3) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 26ba4b8f1..6722b5d4c 100644 --- a/pom.xml +++ b/pom.xml @@ -164,7 +164,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.0.4 + 4.1.3 spotbugs From 7e133240f0d7d0db8d6027b36efe59427cfe940b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 1 Oct 2020 16:34:15 +0200 Subject: [PATCH 0259/1284] Make the executePrologue and executeEpilogue methods private --- src/sqlancer/GlobalState.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index a7b8ac475..959d14935 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -94,7 +94,7 @@ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } - public ExecutionTimer executePrologue(Query q) throws SQLException { + private ExecutionTimer executePrologue(Query q) throws SQLException { boolean logExecutionTime = getOptions().logExecutionTime(); ExecutionTimer timer = null; if (logExecutionTime) { @@ -113,7 +113,7 @@ public ExecutionTimer executePrologue(Query q) throws SQLException { return timer; } - public void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws SQLException { + private void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws SQLException { boolean logExecutionTime = getOptions().logExecutionTime(); if (success && getOptions().printSucceedingStatements()) { System.out.println(q.getQueryString()); From 4c41df9aca5c798898e93a86ab39adca94eec42c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 1 Oct 2020 17:36:34 +0200 Subject: [PATCH 0260/1284] [SQLite] Copy the expression generator when setting a non-default option --- .../gen/SQLite3ExpressionGenerator.java | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 9c4e4ce51..4d64ea664 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -58,6 +58,19 @@ public class SQLite3ExpressionGenerator implements ExpressionGenerator(other.columns); + this.r = other.r; + this.deterministicOnly = other.deterministicOnly; + this.allowMatchClause = other.allowMatchClause; + this.allowAggregateFunctions = other.allowAggregateFunctions; + this.allowSubqueries = other.allowSubqueries; + this.allowAggreates = other.allowAggreates; + } + private enum LiteralValueType { INTEGER, NUMERIC, STRING, BLOB_LITERAL, NULL } @@ -68,38 +81,45 @@ public SQLite3ExpressionGenerator(SQLite3GlobalState globalState) { } public SQLite3ExpressionGenerator deterministicOnly() { - this.deterministicOnly = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.deterministicOnly = true; + return gen; } public SQLite3ExpressionGenerator allowAggregateFunctions() { - this.allowAggregateFunctions = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.allowAggregateFunctions = true; + return gen; } public SQLite3ExpressionGenerator setColumns(List columns) { - this.columns = columns; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.columns = new ArrayList<>(columns); + return gen; } public SQLite3ExpressionGenerator setRowValue(SQLite3RowValue rw) { - this.rw = rw; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.rw = rw; + return gen; } public SQLite3ExpressionGenerator allowMatchClause() { - this.allowMatchClause = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.allowMatchClause = true; + return gen; } public SQLite3ExpressionGenerator allowSubqueries() { - this.allowSubqueries = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.allowSubqueries = true; + return gen; } public SQLite3ExpressionGenerator tryToGenerateKnownResult() { - this.tryToGenerateKnownResult = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.tryToGenerateKnownResult = true; + return gen; } public static SQLite3Expression getRandomLiteralValue(SQLite3GlobalState globalState) { From f23e825bacf655c416461f422f12e5c715e7b90c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 1 Oct 2020 17:59:10 +0200 Subject: [PATCH 0261/1284] [SQLite] Recompute the table count when needed --- src/sqlancer/GlobalState.java | 4 ++++ src/sqlancer/common/schema/AbstractTable.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 959d14935..760d53882 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -8,6 +8,7 @@ import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; /** * Represents a global state that is valid for a testing session on a given database. @@ -166,6 +167,9 @@ protected void setSchema(S schema) { public void updateSchema() throws SQLException { setSchema(readSchema()); + for (AbstractTable table : schema.getDatabaseTables()) { + table.recomputeCount(); + } } protected abstract S readSchema() throws SQLException; diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 3c204c908..007345121 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -115,4 +115,8 @@ public long getNrRows(GlobalState globalState) { return rowCount; } } + + public void recomputeCount() { + rowCount = NO_ROW_COUNT_AVAILABLE; + } } From 516c68247f110e0649647b3013f68c74757d9437 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 1 Oct 2020 19:44:11 +0200 Subject: [PATCH 0262/1284] [SQLite] Add copy constructors for SELECTs and joins --- configs/pmd-rules.xml | 1 + .../sqlite3/ast/SQLite3Expression.java | 6 ++++++ src/sqlancer/sqlite3/ast/SQLite3Select.java | 20 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/configs/pmd-rules.xml b/configs/pmd-rules.xml index f89de4028..f464ca832 100644 --- a/configs/pmd-rules.xml +++ b/configs/pmd-rules.xml @@ -90,5 +90,6 @@ + diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 7bfbb7ea7..748c70375 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -138,6 +138,12 @@ public enum JoinType { private SQLite3Expression onClause; private JoinType type; + public Join(Join other) { + this.table = other.table; + this.onClause = other.onClause; + this.type = other.type; + } + public Join(SQLite3Table table, SQLite3Expression onClause, JoinType type) { this.table = table; this.onClause = onClause; diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 7c5db143a..340c708fb 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -1,5 +1,6 @@ package sqlancer.sqlite3.ast; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -18,6 +19,25 @@ public class SQLite3Select extends SQLite3Expression { private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; + public SQLite3Select() { + } + + public SQLite3Select(SQLite3Select other) { + fromOptions = other.fromOptions; + fromList = new ArrayList<>(other.fromList); + whereClause = other.whereClause; + groupByClause = other.groupByClause; + limitClause = other.limitClause; + orderByClause = new ArrayList<>(other.orderByClause); + offsetClause = other.offsetClause; + fetchColumns = new ArrayList<>(fetchColumns); + joinStatements = new ArrayList<>(); + for (Join j : other.joinStatements) { + joinStatements.add(new Join(j)); + } + havingClause = other.havingClause; + } + public enum SelectType { DISTINCT, ALL; } From d8264a9234832a7d2e8ffcbb08e73de227a35ae8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2020 06:41:11 +0000 Subject: [PATCH 0263/1284] Bump formatter-maven-plugin from 2.12.2 to 2.13.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.12.2 to 2.13.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.12.2...formatter-maven-plugin-2.13.0) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6722b5d4c..05b921b07 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.12.2 + 2.13.0 eclipseformat From 2c4f14b68cf8f08406e1c9f134dddd3af324ec45 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 24 Sep 2020 10:44:47 +0200 Subject: [PATCH 0264/1284] Re-enable the ClickHouse unit tests --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5546d12bf..13cf6704d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -142,3 +142,5 @@ matrix: - docker pull yandex/clickhouse-server:latest - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest - sleep 5 + script: + - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test From f665e45c35847a28f3c6885ed8eb18857f5e51f2 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 2 Oct 2020 12:33:37 +0200 Subject: [PATCH 0265/1284] [H2] Update the method to read the schema --- src/sqlancer/h2/H2Provider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 15ff389ac..4494d6faf 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -66,8 +66,8 @@ private static int mapActions(H2GlobalState globalState, Action a) { public static class H2GlobalState extends GlobalState { @Override - protected void updateSchema() throws SQLException { - setSchema(H2Schema.fromConnection(getConnection(), getDatabaseName())); + protected H2Schema readSchema() throws SQLException { + return H2Schema.fromConnection(getConnection(), getDatabaseName()); } } From 6ba88b677519b3a54527b89341a34db181c2fdea Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 2 Oct 2020 13:54:49 +0200 Subject: [PATCH 0266/1284] Remove the "conditionally accepted" note for the TLP paper The paper has now been accepted. --- docs/PAPERS.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/PAPERS.md b/docs/PAPERS.md index 704608b61..c84af6b91 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -44,7 +44,6 @@ This paper describes TLP, a metamorphic testing approach that can detect various title={Finding Bugs in Database Systems via Query Partitioning}, journal = {Proc. ACM Program. Lang.}, number = {OOPSLA}, - year={2020}, - note={Conditionally accepted} + year={2020} } ``` From 1c1b5f221ad34da2b07a65ff4c528522b59828ea Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 6 Oct 2020 13:43:00 +0200 Subject: [PATCH 0267/1284] Remove the commercialization paragraph --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index fbd28f7d5..058a498be 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,6 @@ If the execution prints progress information every five seconds, then the tool w If you launch SQLancer without parameters, available options and commands are displayed. Note that general options that are supported by all DBMS-testing implementations (e.g., `--num-threads`) need to precede the name of DBMS to be tested (e.g., `sqlite3`). Options that are supported only for specific DBMS (e.g., `--test-rtree` for SQLite3), or options for which each testing implementation provides different values (e.g. `--oracle NoREC`) need to go after the DBMS name. -# Potential Commercialization - -Due to the significant interest that we have received, we are considering to commercialize our bug-finding efforts. If you represent a company and would be interested in a bug-finding service, please approach us ([Manuel Rigger](mailto:manuel.rigger@inf.ethz.ch) and [Zhendong Su](mailto:zhendong.su@inf.ethz.ch)) with your expectations and requirements for such a service. - # Research Prototype This project should at this stage still be seen as a research prototype. We believe that the tool is not ready to be used. However, we have received many requests by companies, organizations, and individual developers, which is why we decided to prematurely release the tool. Expect errors, incompatibilities, lack of documentation, and insufficient code quality. That being said, we are working hard to address these issues and enhance SQLancer to become a production-quality piece of software. We welcome any issue reports, extension requests, and code contributions. From 0024f977004fc60b0565a131431002e05e3256a7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 8 Oct 2020 12:47:55 +0200 Subject: [PATCH 0268/1284] Update the OOPSLA bib entry with the DOI --- docs/PAPERS.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/PAPERS.md b/docs/PAPERS.md index c84af6b91..916eee52d 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -44,6 +44,9 @@ This paper describes TLP, a metamorphic testing approach that can detect various title={Finding Bugs in Database Systems via Query Partitioning}, journal = {Proc. ACM Program. Lang.}, number = {OOPSLA}, - year={2020} + year={2020}, + doi={10.1145/3428279}, + volume={4}, + articleno={211} } ``` From 13f8a301b7ecb50c354d921a0ae3d7d7677e7273 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2020 06:42:26 +0000 Subject: [PATCH 0269/1284] Bump postgresql from 42.2.16 to 42.2.17 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.16 to 42.2.17. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/REL42.2.17/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.16...REL42.2.17) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b1126458f..4dff3b182 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ org.postgresql postgresql - 42.2.16 + 42.2.17 org.xerial From d14a25d4862dd61426592d1034e338f47b28b924 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 13 Oct 2020 16:18:49 +0200 Subject: [PATCH 0270/1284] Add arity=1 to the boolean options in the main option class --- src/sqlancer/MainOptions.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 45867edb4..144826298 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -73,7 +73,7 @@ public class MainOptions { @Parameter(names = "--test-only-nonempty-tables", description = "Test only databases each of whose tables contain at least a single row", arity = 1) private boolean testOnlyWithMoreThanZeroRows; // NOPMD - @Parameter(names = "--pqs-test-aggregates", description = "Partially test aggregate functions when all tables contain only a single row.") + @Parameter(names = "--pqs-test-aggregates", description = "Partially test aggregate functions when all tables contain only a single row.", arity = 1) private boolean testAggregateFunctions; // NOPMD @Parameter(names = "--random-string-generation", description = "Select the random-string eneration approach") @@ -82,10 +82,10 @@ public class MainOptions { @Parameter(names = "--string-constant-max-length", description = "Specify the maximum-length of generated string constants") private int maxStringConstantLength = 10; // NOPMD - @Parameter(names = "--use-constant-caching", description = "Specifies whether constants should be cached and re-used with a certain probability") + @Parameter(names = "--use-constant-caching", description = "Specifies whether constants should be cached and re-used with a certain probability", arity = 1) private boolean useConstantCaching = true; // NOPMD - @Parameter(names = "--use-connection-test", description = "Test whether the DBMS is accessible before trying to connect using multiple threads") + @Parameter(names = "--use-connection-test", description = "Test whether the DBMS is accessible before trying to connect using multiple threads", arity = 1) private boolean useConnectionTest = true; // NOPMD @Parameter(names = "--constant-cache-size", description = "Specifies the size of the constant cache. This option only takes effect when constant caching is enabled") From e763cb028f96471ceb599b64b1cb61d2eadb2e69 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2020 06:39:10 +0000 Subject: [PATCH 0271/1284] Bump postgresql from 42.2.17 to 42.2.18 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.17 to 42.2.18. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/REL42.2.18/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.17...REL42.2.18) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4dff3b182..3c4bfcd2e 100644 --- a/pom.xml +++ b/pom.xml @@ -191,7 +191,7 @@ org.postgresql postgresql - 42.2.17 + 42.2.18 org.xerial From 5c58da785055b3555729c6bfce4487635b037cd0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2020 06:40:36 +0000 Subject: [PATCH 0272/1284] Bump mysql-connector-java from 8.0.21 to 8.0.22 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.21 to 8.0.22. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/commits) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3c4bfcd2e..145cb203a 100644 --- a/pom.xml +++ b/pom.xml @@ -201,7 +201,7 @@ mysql mysql-connector-java - 8.0.21 + 8.0.22 org.mariadb.jdbc From 8090f18f6771cde1e1ebce9759488d1bdd5e9eb8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2020 06:35:54 +0000 Subject: [PATCH 0273/1284] Bump spotbugs-maven-plugin from 4.1.3 to 4.1.4 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.1.3 to 4.1.4. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.1.3...spotbugs-maven-plugin-4.1.4) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 145cb203a..9aa769c21 100644 --- a/pom.xml +++ b/pom.xml @@ -164,7 +164,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.1.3 + 4.1.4 spotbugs From f408d01d50771616e877b2aa0e1634a7a06095e9 Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 4 Nov 2020 20:14:42 +0300 Subject: [PATCH 0274/1284] Add known Citus error 'cannot modify views over distributed tables' --- src/sqlancer/citus/gen/CitusCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 5e26285c9..2b7bdaea5 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -36,6 +36,7 @@ public static void addCitusErrors(ExpectedErrors errors) { errors.add("cannot create foreign key constraint"); // SET NULL or SET DEFAULT is not supported in ON DELETE // operation when distribution key is included in the // foreign key constraint + errors.add("cannot modify views over distributed tables"); // not supported by Citus (restrictions on SELECT queries) errors.add( From 2a796dae2d5457bbc11246f9f87b36e8a432d411 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 13 Nov 2020 10:50:01 +0100 Subject: [PATCH 0275/1284] Refactor Logging to remove SQL Dependency, keeping same Functionality As of this commit the classes StateLogger and StateToReproduce are no longer dependent on SQL strings. The functionality is now supported by a LoggableFactory, specifically for all current DBMSs the SQLLoggableFactory. Parts of the functionality may be removed later, for example commentStatements is now marked depricated. --- src/sqlancer/DatabaseProvider.java | 4 ++ src/sqlancer/Main.java | 49 +++++++---------- src/sqlancer/ProviderAdapter.java | 2 +- src/sqlancer/SQLProviderAdapter.java | 16 ++++++ src/sqlancer/StateToReproduce.java | 16 +++--- .../clickhouse/ClickHouseProvider.java | 4 +- .../cockroachdb/CockroachDBProvider.java | 4 +- src/sqlancer/common/log/Loggable.java | 5 ++ src/sqlancer/common/log/LoggableFactory.java | 37 +++++++++++++ src/sqlancer/common/log/LoggedString.java | 15 ++++++ .../common/log/SQLLoggableFactory.java | 52 +++++++++++++++++++ src/sqlancer/common/query/Query.java | 3 +- src/sqlancer/common/query/QueryAdapter.java | 4 ++ src/sqlancer/duckdb/DuckDBProvider.java | 4 +- src/sqlancer/h2/H2Provider.java | 4 +- src/sqlancer/mariadb/MariaDBProvider.java | 4 +- src/sqlancer/mysql/MySQLProvider.java | 4 +- src/sqlancer/postgres/PostgresProvider.java | 4 +- src/sqlancer/sqlite3/SQLite3Provider.java | 5 +- src/sqlancer/tidb/TiDBProvider.java | 4 +- 20 files changed, 183 insertions(+), 57 deletions(-) create mode 100644 src/sqlancer/SQLProviderAdapter.java create mode 100644 src/sqlancer/common/log/Loggable.java create mode 100644 src/sqlancer/common/log/LoggableFactory.java create mode 100644 src/sqlancer/common/log/LoggedString.java create mode 100644 src/sqlancer/common/log/SQLLoggableFactory.java diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 81fdbb2e7..d7e19db90 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -3,6 +3,8 @@ import java.sql.Connection; import java.sql.SQLException; +import sqlancer.common.log.LoggableFactory; + public interface DatabaseProvider, O extends DBMSSpecificOptions> { /** @@ -37,6 +39,8 @@ public interface DatabaseProvider, O extends DBMSSpe */ String getDBMSName(); + LoggableFactory getLoggableFactory(); + StateToReproduce getStateToReproduce(String databaseName); } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index be49b9410..bf9b5abc0 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -3,8 +3,6 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; import java.io.Writer; import java.nio.file.Files; import java.sql.Connection; @@ -29,6 +27,7 @@ import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.cockroachdb.CockroachDBProvider; +import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.duckdb.DuckDBProvider; @@ -67,6 +66,7 @@ public static final class StateLogger { public FileWriter currentFileWriter; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; + private final DatabaseProvider databaseProvider; private static final class AlsoWriteToConsoleFileWriter extends FileWriter { @@ -98,6 +98,7 @@ public StateLogger(String databaseName, DatabaseProvider provider, MainOpt if (logEachSelect) { curFile = new File(dir, databaseName + "-cur.log"); } + this.databaseProvider = provider; } private void ensureExistsAndIsEmpty(File dir, DatabaseProvider provider) { @@ -161,37 +162,32 @@ public void writeCurrent(StateToReproduce state) { } } - public void writeCurrent(String queryString) { - write(queryString, "\n"); + public void writeCurrent(String input) { + write(databaseProvider.getLoggableFactory().createLoggable(input)); } - private void write(String queryString, String suffix) { + public void writeCurrentNoLineBreak(String input) { + write(databaseProvider.getLoggableFactory().createLoggableWithNoLinebreak(input)); + } + + private void write(Loggable loggable) { if (!logEachSelect) { throw new UnsupportedOperationException(); } try { - getCurrentFileWriter().write(queryString); - if (!queryString.endsWith(";")) { - getCurrentFileWriter().write(';'); - } - if (suffix != null && suffix.length() != 0) { - getCurrentFileWriter().write(suffix); - } + getCurrentFileWriter().write(loggable.getLogString()); + currentFileWriter.flush(); } catch (IOException e) { throw new AssertionError(); } } - public void writeCurrentNoLineBreak(String queryString) { - write(queryString, ""); - } - public void logException(Throwable reduce, StateToReproduce state) { - String stackTrace = getStackTrace(reduce); + Loggable stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); try { - logFileWriter2.write(stackTrace); + logFileWriter2.write(stackTrace.getLogString()); printState(logFileWriter2, state); } catch (IOException e) { throw new AssertionError(e); @@ -205,21 +201,16 @@ public void logException(Throwable reduce, StateToReproduce state) { } } - private String getStackTrace(Throwable e1) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - e1.printStackTrace(pw); - return "--" + sw.toString().replace("\n", "\n--"); + private Loggable getStackTrace(Throwable e1) { + return databaseProvider.getLoggableFactory().convertStacktraceToLoggable(e1); } private void printState(FileWriter writer, StateToReproduce state) { StringBuilder sb = new StringBuilder(); - DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); - Date date = new Date(); - sb.append("-- Time: " + dateFormat.format(date) + "\n"); - sb.append("-- Database: " + state.getDatabaseName() + "\n"); - sb.append("-- Database version: " + state.getDatabaseVersion() + "\n"); - sb.append("-- seed value: " + state.getSeedValue() + "\n"); + + sb.append(databaseProvider.getLoggableFactory() + .getInfo(state.getDatabaseName(), state.getDatabaseVersion(), state.getSeedValue()).getLogString()); + for (Query s : state.getStatements()) { sb.append(s.getQueryString()); sb.append('\n'); diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 772bd142a..814d9ecb4 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -23,7 +23,7 @@ public ProviderAdapter(Class globalClass, Class optionClass) { @Override public StateToReproduce getStateToReproduce(String databaseName) { - return new StateToReproduce(databaseName); + return new StateToReproduce(databaseName, this); } @Override diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java new file mode 100644 index 000000000..ca4563768 --- /dev/null +++ b/src/sqlancer/SQLProviderAdapter.java @@ -0,0 +1,16 @@ +package sqlancer; + +import sqlancer.common.log.LoggableFactory; +import sqlancer.common.log.SQLLoggableFactory; + +public abstract class SQLProviderAdapter, O extends DBMSSpecificOptions>> + extends ProviderAdapter { + public SQLProviderAdapter(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + @Override + public LoggableFactory getLoggableFactory() { + return new SQLLoggableFactory(); + } +} diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index e77d8a400..071c20bdc 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -6,7 +6,6 @@ import java.util.List; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; public class StateToReproduce { @@ -14,6 +13,8 @@ public class StateToReproduce { private final String databaseName; + private final DatabaseProvider databaseProvider; + public String databaseVersion; protected long seedValue; @@ -22,8 +23,9 @@ public class StateToReproduce { public OracleRunReproductionState localState; - public StateToReproduce(String databaseName) { + public StateToReproduce(String databaseName, DatabaseProvider databaseProvider) { this.databaseName = databaseName; + this.databaseProvider = databaseProvider; } public String getException() { @@ -48,7 +50,7 @@ public void logStatement(String queryString) { if (queryString == null) { throw new IllegalArgumentException(); } - logStatement(new QueryAdapter(queryString)); + logStatement(databaseProvider.getLoggableFactory().getQueryForStateToReproduce(queryString)); } /** @@ -68,12 +70,12 @@ public List getStatements() { return Collections.unmodifiableList(statements); } + @Deprecated public void commentStatements() { for (int i = 0; i < statements.size(); i++) { Query statement = statements.get(i); - String queryString = statement.getQueryString(); - String newQueryString = "-- " + queryString; - statements.set(i, new QueryAdapter(newQueryString)); + Query newQuery = databaseProvider.getLoggableFactory().commentOutQuery(statement); + statements.set(i, newQuery); } } @@ -109,7 +111,7 @@ public void executedWithoutError() { } public void log(String s) { - statements.add(new QueryAdapter(s)); + statements.add(databaseProvider.getLoggableFactory().getQueryForStateToReproduce(s)); } @Override diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 947d8c90a..0f4578494 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -9,8 +9,8 @@ import sqlancer.AbstractAction; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.gen.ClickHouseCommon; @@ -19,7 +19,7 @@ import sqlancer.common.query.Query; import sqlancer.common.query.QueryProvider; -public class ClickHouseProvider extends ProviderAdapter { +public class ClickHouseProvider extends SQLProviderAdapter { public ClickHouseProvider() { super(ClickHouseGlobalState.class, ClickHouseOptions.class); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index b82d5fcdd..0a4351d3b 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -12,8 +12,8 @@ import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; import sqlancer.MainOptions; -import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.SQLProviderAdapter; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.gen.CockroachDBCommentOnGenerator; @@ -34,7 +34,7 @@ import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.QueryProvider; -public class CockroachDBProvider extends ProviderAdapter { +public class CockroachDBProvider extends SQLProviderAdapter { public CockroachDBProvider() { super(CockroachDBGlobalState.class, CockroachDBOptions.class); diff --git a/src/sqlancer/common/log/Loggable.java b/src/sqlancer/common/log/Loggable.java new file mode 100644 index 000000000..0d5fef218 --- /dev/null +++ b/src/sqlancer/common/log/Loggable.java @@ -0,0 +1,5 @@ +package sqlancer.common.log; + +public interface Loggable { + String getLogString(); +} diff --git a/src/sqlancer/common/log/LoggableFactory.java b/src/sqlancer/common/log/LoggableFactory.java new file mode 100644 index 000000000..2386ef10f --- /dev/null +++ b/src/sqlancer/common/log/LoggableFactory.java @@ -0,0 +1,37 @@ +package sqlancer.common.log; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; + +import sqlancer.common.query.Query; + +public abstract class LoggableFactory { + + public Loggable createLoggableWithNoLinebreak(String input) { + return createLoggable(input, ""); + } + + public Loggable createLoggable(String input) { + return createLoggable(input, "\n"); + } + + protected abstract Loggable createLoggable(String input, String suffix); + + public abstract Query getQueryForStateToReproduce(String queryString); + + @Deprecated + public abstract Query commentOutQuery(Query query); + + public Loggable getInfo(String databaseName, String databaseVersion, long seedValue) { + Date date = new Date(); + DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + return infoToLoggable(dateFormat.format(date), databaseName, databaseVersion, seedValue); + } + + protected abstract Loggable infoToLoggable(String time, String databaseName, String databaseVersion, + long seedValue); + + public abstract Loggable convertStacktraceToLoggable(Throwable throwable); + +} diff --git a/src/sqlancer/common/log/LoggedString.java b/src/sqlancer/common/log/LoggedString.java new file mode 100644 index 000000000..4f449034f --- /dev/null +++ b/src/sqlancer/common/log/LoggedString.java @@ -0,0 +1,15 @@ +package sqlancer.common.log; + +public class LoggedString implements Loggable { + + private final String loggedString; + + public LoggedString(String loggedString) { + this.loggedString = loggedString; + } + + @Override + public String getLogString() { + return this.loggedString; + } +} diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java new file mode 100644 index 000000000..78b96e04c --- /dev/null +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -0,0 +1,52 @@ +package sqlancer.common.log; + +import java.io.PrintWriter; +import java.io.StringWriter; + +import sqlancer.common.query.Query; +import sqlancer.common.query.QueryAdapter; + +public class SQLLoggableFactory extends LoggableFactory { + + @Override + protected Loggable createLoggable(String input, String suffix) { + String completeString = input; + if (!input.endsWith(";")) { + completeString += ";"; + } + if (suffix != null && suffix.length() != 0) { + completeString += suffix; + } + return new LoggedString(completeString); + } + + @Override + public Query getQueryForStateToReproduce(String queryString) { + return new QueryAdapter(queryString); + } + + @Override + public Query commentOutQuery(Query query) { + String queryString = query.getLogString(); + String newQueryString = "-- " + queryString; + return new QueryAdapter(newQueryString); + } + + @Override + protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { + StringBuilder sb = new StringBuilder(); + sb.append("-- Time: " + time + "\n"); + sb.append("-- Database: " + databaseName + "\n"); + sb.append("-- Database version: " + databaseVersion + "\n"); + sb.append("-- seed value: " + seedValue + "\n"); + return new LoggedString(sb.toString()); + } + + @Override + public Loggable convertStacktraceToLoggable(Throwable throwable) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + throwable.printStackTrace(pw); + return new LoggedString("--" + sw.toString().replace("\n", "\n--")); + } +} diff --git a/src/sqlancer/common/query/Query.java b/src/sqlancer/common/query/Query.java index 430c93cad..fa827bf65 100644 --- a/src/sqlancer/common/query/Query.java +++ b/src/sqlancer/common/query/Query.java @@ -3,8 +3,9 @@ import java.sql.SQLException; import sqlancer.GlobalState; +import sqlancer.common.log.Loggable; -public abstract class Query { +public abstract class Query implements Loggable { /** * Gets the query string, which is guaranteed to be terminated with a semicolon. diff --git a/src/sqlancer/common/query/QueryAdapter.java b/src/sqlancer/common/query/QueryAdapter.java index 8934bdc14..b0616a518 100644 --- a/src/sqlancer/common/query/QueryAdapter.java +++ b/src/sqlancer/common/query/QueryAdapter.java @@ -140,4 +140,8 @@ public ExpectedErrors getExpectedErrors() { return expectedErrors; } + @Override + public String getLogString() { + return getQueryString(); + } } diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 2b757a7f5..cf8d24183 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -7,8 +7,8 @@ import sqlancer.AbstractAction; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; @@ -23,7 +23,7 @@ import sqlancer.duckdb.gen.DuckDBUpdateGenerator; import sqlancer.duckdb.gen.DuckDBViewGenerator; -public class DuckDBProvider extends ProviderAdapter { +public class DuckDBProvider extends SQLProviderAdapter { public DuckDBProvider() { super(DuckDBGlobalState.class, DuckDBOptions.class); diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 4494d6faf..43102338f 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -7,15 +7,15 @@ import sqlancer.AbstractAction; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.h2.H2Provider.H2GlobalState; -public class H2Provider extends ProviderAdapter { +public class H2Provider extends SQLProviderAdapter { public H2Provider() { super(H2GlobalState.class, H2Options.class); diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 960fd5ae3..c9bbba54b 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -10,8 +10,8 @@ import sqlancer.GlobalState; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; -import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.SQLProviderAdapter; import sqlancer.common.query.Query; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.gen.MariaDBIndexGenerator; @@ -23,7 +23,7 @@ import sqlancer.mariadb.gen.MariaDBUpdateGenerator; import sqlancer.sqlite3.gen.SQLite3Common; -public class MariaDBProvider extends ProviderAdapter { +public class MariaDBProvider extends SQLProviderAdapter { public static final int MAX_EXPRESSION_DEPTH = 3; diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index fcdd31672..b9251c1a5 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -7,8 +7,8 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; @@ -30,7 +30,7 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; import sqlancer.sqlite3.gen.SQLite3Common; -public class MySQLProvider extends ProviderAdapter { +public class MySQLProvider extends SQLProviderAdapter { public MySQLProvider() { super(MySQLGlobalState.class, MySQLOptions.class); diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 3e4f3e793..d8a76cb77 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -10,8 +10,8 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; @@ -43,7 +43,7 @@ // EXISTS // IN -public class PostgresProvider extends ProviderAdapter { +public class PostgresProvider extends SQLProviderAdapter { /** * Generate only data types and expressions that are understood by PQS. diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 7616718d7..868fc2876 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -11,8 +11,8 @@ import sqlancer.AbstractAction; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; @@ -44,7 +44,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3Provider extends ProviderAdapter { +public class SQLite3Provider extends SQLProviderAdapter { public static boolean allowFloatingPointFp = true; public static boolean mustKnowResult; @@ -300,5 +300,4 @@ public Connection createDatabase(SQLite3GlobalState globalState) throws SQLExcep public String getDBMSName() { return "sqlite3"; } - } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 3cf7e7590..23e47d072 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -8,8 +8,8 @@ import sqlancer.AbstractAction; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; @@ -27,7 +27,7 @@ import sqlancer.tidb.gen.TiDBUpdateGenerator; import sqlancer.tidb.gen.TiDBViewGenerator; -public class TiDBProvider extends ProviderAdapter { +public class TiDBProvider extends SQLProviderAdapter { public TiDBProvider() { super(TiDBGlobalState.class, TiDBOptions.class); From 5c3c3c3aa045d31d75f7a13ef873e5eef92052d9 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 16 Nov 2020 12:02:15 +0100 Subject: [PATCH 0276/1284] Refactor Exceptions to remove SQLDependency while keeping the same functionality Multiple high-level classes such as DatabaseProvider carry already the semantics for non-SQL DBMS except for the fact that a lot of methods still throw SQLExceptions from the java.sql package. This commit exchanges SQLException with Exception in signatures to solve this. Also changes the configuration of pmd to exclude the rule "SignatureDeclareThrowsException" --- configs/pmd-rules.xml | 1 + src/sqlancer/AbstractAction.java | 4 +--- src/sqlancer/DatabaseProvider.java | 5 ++--- src/sqlancer/GlobalState.java | 17 ++++++++--------- src/sqlancer/Main.java | 11 +++++------ src/sqlancer/OracleFactory.java | 4 +--- src/sqlancer/ProviderAdapter.java | 9 ++++----- src/sqlancer/StatementExecutor.java | 5 ++--- src/sqlancer/citus/CitusProvider.java | 12 ++++++------ src/sqlancer/clickhouse/ClickHouseProvider.java | 4 ++-- .../cockroachdb/CockroachDBProvider.java | 4 ++-- .../common/oracle/CompositeTestOracle.java | 3 +-- .../oracle/PivotedQuerySynthesisBase.java | 15 +++++++-------- src/sqlancer/common/oracle/TestOracle.java | 4 +--- src/sqlancer/common/query/Query.java | 10 ++++------ src/sqlancer/common/query/QueryProvider.java | 4 +--- src/sqlancer/duckdb/DuckDBProvider.java | 4 ++-- src/sqlancer/h2/H2Provider.java | 4 ++-- src/sqlancer/mariadb/MariaDBProvider.java | 2 +- src/sqlancer/mysql/MySQLProvider.java | 4 ++-- src/sqlancer/postgres/PostgresProvider.java | 8 ++++---- src/sqlancer/sqlite3/SQLite3Provider.java | 8 ++++---- .../sqlite3/gen/SQLite3ExplainGenerator.java | 4 +--- src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java | 4 +--- src/sqlancer/tidb/TiDBProvider.java | 4 ++-- 25 files changed, 67 insertions(+), 87 deletions(-) diff --git a/configs/pmd-rules.xml b/configs/pmd-rules.xml index f464ca832..fd2ad9fc4 100644 --- a/configs/pmd-rules.xml +++ b/configs/pmd-rules.xml @@ -71,6 +71,7 @@ + 2 diff --git a/src/sqlancer/AbstractAction.java b/src/sqlancer/AbstractAction.java index 2ed0a7ed2..92996a7fa 100644 --- a/src/sqlancer/AbstractAction.java +++ b/src/sqlancer/AbstractAction.java @@ -1,12 +1,10 @@ package sqlancer; -import java.sql.SQLException; - import sqlancer.common.query.Query; public interface AbstractAction { - Query getQuery(G globalState) throws SQLException; + Query getQuery(G globalState) throws Exception; /** * Specifies whether it makes sense to request a {@link Query}, when the previous call to {@link #getQuery(Object)} diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index d7e19db90..9351bbe6d 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -1,7 +1,6 @@ package sqlancer; import java.sql.Connection; -import java.sql.SQLException; import sqlancer.common.log.LoggableFactory; @@ -28,9 +27,9 @@ public interface DatabaseProvider, O extends DBMSSpe * the state created and is valid for this method call. * */ - void generateAndTestDatabase(G globalState) throws SQLException; + void generateAndTestDatabase(G globalState) throws Exception; - Connection createDatabase(G globalState) throws SQLException; + Connection createDatabase(G globalState) throws Exception; /** * The DBMS name is used to name the log directory and command to test the respective DBMS. diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 760d53882..3d7ddcecc 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,7 +1,6 @@ package sqlancer; import java.sql.Connection; -import java.sql.SQLException; import sqlancer.Main.QueryManager; import sqlancer.Main.StateLogger; @@ -95,7 +94,7 @@ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } - private ExecutionTimer executePrologue(Query q) throws SQLException { + private ExecutionTimer executePrologue(Query q) throws Exception { boolean logExecutionTime = getOptions().logExecutionTime(); ExecutionTimer timer = null; if (logExecutionTime) { @@ -114,7 +113,7 @@ private ExecutionTimer executePrologue(Query q) throws SQLException { return timer; } - private void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws SQLException { + private void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { boolean logExecutionTime = getOptions().logExecutionTime(); if (success && getOptions().printSucceedingStatements()) { System.out.println(q.getQueryString()); @@ -127,14 +126,14 @@ private void executeEpilogue(Query q, boolean success, ExecutionTimer timer) thr } } - public boolean executeStatement(Query q, String... fills) throws SQLException { + public boolean executeStatement(Query q, String... fills) throws Exception { ExecutionTimer timer = executePrologue(q); boolean success = manager.execute(q, fills); executeEpilogue(q, success, timer); return success; } - public SQLancerResultSet executeStatementAndGet(Query q, String... fills) throws SQLException { + public SQLancerResultSet executeStatementAndGet(Query q, String... fills) throws Exception { ExecutionTimer timer = executePrologue(q); SQLancerResultSet result = manager.executeAndGet(q, fills); boolean success = result != null; @@ -142,7 +141,7 @@ public SQLancerResultSet executeStatementAndGet(Query q, String... fills) throws result.registerEpilogue(() -> { try { executeEpilogue(q, success, timer); - } catch (SQLException e) { + } catch (Exception e) { throw new AssertionError(e); } }); @@ -154,7 +153,7 @@ public S getSchema() { if (schema == null) { try { updateSchema(); - } catch (SQLException e) { + } catch (Exception e) { throw new AssertionError(); } } @@ -165,13 +164,13 @@ protected void setSchema(S schema) { this.schema = schema; } - public void updateSchema() throws SQLException { + public void updateSchema() throws Exception { setSchema(readSchema()); for (AbstractTable table : schema.getDatabaseTables()) { table.recomputeCount(); } } - protected abstract S readSchema() throws SQLException; + protected abstract S readSchema() throws Exception; } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index bf9b5abc0..cf83bf76e 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -6,7 +6,6 @@ import java.io.Writer; import java.nio.file.Files; import java.sql.Connection; -import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -232,7 +231,7 @@ public static class QueryManager { this.globalState = globalState; } - public boolean execute(Query q, String... fills) throws SQLException { + public boolean execute(Query q, String... fills) throws Exception { globalState.getState().logStatement(q); boolean success; success = q.execute(globalState, fills); @@ -240,7 +239,7 @@ public boolean execute(Query q, String... fills) throws SQLException { return success; } - public SQLancerResultSet executeAndGet(Query q, String... fills) throws SQLException { + public SQLancerResultSet executeAndGet(Query q, String... fills) throws Exception { globalState.getState().logStatement(q); SQLancerResultSet result; result = q.executeAndGet(globalState, fills); @@ -293,14 +292,14 @@ public O getCommand() { return command; } - public void testConnection() throws SQLException { + public void testConnection() throws Exception { G state = getInitializedGlobalState(options.getRandomSeed()); try (Connection con = provider.createDatabase(state)) { return; } } - public void run() throws SQLException { + public void run() throws Exception { G state = createGlobalState(); stateToRepro = provider.getStateToReproduce(databaseName); stateToRepro.seedValue = r.getSeed(); @@ -455,7 +454,7 @@ private String formatInteger(long intValue) { try { executorFactory.getDBMSExecutor(options.getDatabasePrefix() + "connectiontest", new Randomly()) .testConnection(); - } catch (SQLException e) { + } catch (Exception e) { System.err.println( "SQLancer failed creating a test database, indicating that SQLancer might have failed connecting to the DBMS. In order to change the username and password, you can use the --username and --password options. Currently, SQLancer does not yet support passing a host and port (see https://github.com/sqlancer/sqlancer/issues/95).\n\n"); e.printStackTrace(); diff --git a/src/sqlancer/OracleFactory.java b/src/sqlancer/OracleFactory.java index 5b64cc4c2..8e096eeec 100644 --- a/src/sqlancer/OracleFactory.java +++ b/src/sqlancer/OracleFactory.java @@ -1,12 +1,10 @@ package sqlancer; -import java.sql.SQLException; - import sqlancer.common.oracle.TestOracle; public interface OracleFactory> { - TestOracle create(G globalState) throws SQLException; + TestOracle create(G globalState) throws Exception; /** * Indicates whether the test oracle requires that all tables (including views) contain at least one row. diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 814d9ecb4..104398959 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -1,6 +1,5 @@ package sqlancer; -import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; @@ -37,7 +36,7 @@ public Class getOptionClass() { } @Override - public void generateAndTestDatabase(G globalState) throws SQLException { + public void generateAndTestDatabase(G globalState) throws Exception { try { generateDatabase(globalState); checkViewsAreValid(globalState); @@ -74,7 +73,7 @@ private void checkViewsAreValid(G globalState) { } } - protected TestOracle getTestOracle(G globalState) throws SQLException { + protected TestOracle getTestOracle(G globalState) throws Exception { List> testOracleFactory = globalState.getDmbsSpecificOptions() .getTestOracleFactory(); boolean testOracleRequiresMoreThanZeroRows = testOracleFactory.stream() @@ -90,13 +89,13 @@ protected TestOracle getTestOracle(G globalState) throws SQLException { return new CompositeTestOracle(testOracleFactory.stream().map(o -> { try { return o.create(globalState); - } catch (SQLException e1) { + } catch (Exception e1) { throw new AssertionError(e1); } }).collect(Collectors.toList()), globalState); } } - public abstract void generateDatabase(G globalState) throws SQLException; + public abstract void generateDatabase(G globalState) throws Exception; } diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index 88fe3c164..6034c5ad2 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -1,6 +1,5 @@ package sqlancer; -import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -15,7 +14,7 @@ public class StatementExecutor, A extends AbstractAc @FunctionalInterface public interface AfterQueryAction { - void notify(Query q) throws SQLException; + void notify(Query q) throws Exception; } @FunctionalInterface @@ -30,7 +29,7 @@ public StatementExecutor(G globalState, A[] actions, ActionMapper mapping, this.queryConsumer = queryConsumer; } - public void executeStatements() throws SQLException { + public void executeStatements() throws Exception { Randomly r = globalState.getRandomly(); int[] nrRemaining = new int[actions.length]; List availableActions = new ArrayList<>(); diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 907a37e2f..a2de975f7 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -113,7 +113,7 @@ public enum Action implements AbstractAction { } @Override - public Query getQuery(PostgresGlobalState state) throws SQLException { + public Query getQuery(PostgresGlobalState state) throws Exception { return queryProvider.getQuery(state); } } @@ -200,7 +200,7 @@ public int getPort() { } private static void distributeTable(List columns, String tableName, CitusGlobalState globalState) - throws SQLException { + throws Exception { if (!columns.isEmpty()) { PostgresColumn columnToDistribute = Randomly.fromList(columns); String queryString = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() @@ -227,7 +227,7 @@ private static List getTableConstraints(String tableName, CitusGlobalSta return constraints; } - private static void createDistributedTable(String tableName, CitusGlobalState globalState) throws SQLException { + private static void createDistributedTable(String tableName, CitusGlobalState globalState) throws Exception { List columns = new ArrayList<>(); List tableConstraints = getTableConstraints(tableName, globalState); if (tableConstraints.isEmpty()) { @@ -276,7 +276,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl } @Override - public void generateDatabase(PostgresGlobalState globalState) throws SQLException { + public void generateDatabase(PostgresGlobalState globalState) throws Exception { readFunctions(globalState); createTables(globalState, Randomly.fromOptions(4, 5, 6)); for (PostgresTable table : globalState.getSchema().getDatabaseTables()) { @@ -307,7 +307,7 @@ protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLEx List oracles = ((CitusOptions) globalState.getDmbsSpecificOptions()).citusOracle.stream().map(o -> { try { return o.create(globalState); - } catch (SQLException e1) { + } catch (Exception e1) { throw new AssertionError(e1); } }).collect(Collectors.toList()); @@ -424,7 +424,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce } @Override - protected void prepareTables(PostgresGlobalState globalState) throws SQLException { + protected void prepareTables(PostgresGlobalState globalState) throws Exception { StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), CitusProvider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 0f4578494..aa403ae01 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -36,7 +36,7 @@ public enum Action implements AbstractAction { } @Override - public Query getQuery(ClickHouseGlobalState state) throws SQLException { + public Query getQuery(ClickHouseGlobalState state) throws Exception { return queryProvider.getQuery(state); } } @@ -80,7 +80,7 @@ protected ClickHouseSchema readSchema() throws SQLException { } @Override - public void generateDatabase(ClickHouseGlobalState globalState) throws SQLException { + public void generateDatabase(ClickHouseGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(1); i++) { boolean success; do { diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 0a4351d3b..e58a13244 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -105,7 +105,7 @@ public enum Action { this.queryProvider = queryProvider; } - public Query getQuery(CockroachDBGlobalState state) throws SQLException { + public Query getQuery(CockroachDBGlobalState state) throws Exception { return queryProvider.getQuery(state); } } @@ -120,7 +120,7 @@ protected CockroachDBSchema readSchema() throws SQLException { } @Override - public void generateDatabase(CockroachDBGlobalState globalState) throws SQLException { + public void generateDatabase(CockroachDBGlobalState globalState) throws Exception { QueryManager manager = globalState.getManager(); MainOptions options = globalState.getOptions(); List standardSettings = new ArrayList<>(); diff --git a/src/sqlancer/common/oracle/CompositeTestOracle.java b/src/sqlancer/common/oracle/CompositeTestOracle.java index 4f89815ac..9eb59e345 100644 --- a/src/sqlancer/common/oracle/CompositeTestOracle.java +++ b/src/sqlancer/common/oracle/CompositeTestOracle.java @@ -1,6 +1,5 @@ package sqlancer.common.oracle; -import java.sql.SQLException; import java.util.List; import sqlancer.GlobalState; @@ -17,7 +16,7 @@ public CompositeTestOracle(List oracles, GlobalState globalSta } @Override - public void check() throws SQLException { + public void check() throws Exception { try { oracles[i].check(); boolean lastOracleIndex = i == oracles.length - 1; diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index f4ea9a191..aa7cf5059 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -1,6 +1,5 @@ package sqlancer.common.oracle; -import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -34,7 +33,7 @@ public PivotedQuerySynthesisBase(S globalState) { } @Override - public final void check() throws SQLException { + public final void check() throws Exception { rectifiedPredicates.clear(); Query pivotRowQuery = getRectifiedQuery(); if (globalState.getOptions().logEachSelect()) { @@ -60,9 +59,9 @@ public final void check() throws SQLException { * * @return true if at least one row is contained, false otherwise * - * @throws SQLException + * @throws Exception */ - private boolean containsRows(Query query) throws SQLException { + private boolean containsRows(Query query) throws Exception { try (SQLancerResultSet result = query.executeAndGet(globalState)) { if (result == null) { throw new IgnoreMeException(); @@ -107,9 +106,9 @@ protected void reportMissingPivotRow(Query query) { * * @return a query that checks whether the pivot row is contained in pivotRowQuery * - * @throws SQLException + * @throws Exception */ - protected abstract Query getContainmentCheckQuery(Query pivotRowQuery) throws SQLException; + protected abstract Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception; /** * Obtains a rectified query (i.e., a query that is guaranteed to fetch the pivot row. This corresponds to steps 2-5 @@ -117,9 +116,9 @@ protected void reportMissingPivotRow(Query query) { * * @return the rectified query * - * @throws SQLException + * @throws Exception */ - protected abstract Query getRectifiedQuery() throws SQLException; + protected abstract Query getRectifiedQuery() throws Exception; /** * Prints the value to which the expression is expected to evaluate, and then recursively prints the subexpressions' diff --git a/src/sqlancer/common/oracle/TestOracle.java b/src/sqlancer/common/oracle/TestOracle.java index 04101b950..737adb692 100644 --- a/src/sqlancer/common/oracle/TestOracle.java +++ b/src/sqlancer/common/oracle/TestOracle.java @@ -1,9 +1,7 @@ package sqlancer.common.oracle; -import java.sql.SQLException; - public interface TestOracle { - void check() throws SQLException; + void check() throws Exception; } diff --git a/src/sqlancer/common/query/Query.java b/src/sqlancer/common/query/Query.java index fa827bf65..3f75a7986 100644 --- a/src/sqlancer/common/query/Query.java +++ b/src/sqlancer/common/query/Query.java @@ -1,7 +1,5 @@ package sqlancer.common.query; -import java.sql.SQLException; - import sqlancer.GlobalState; import sqlancer.common.log.Loggable; @@ -28,7 +26,7 @@ public abstract class Query implements Loggable { */ public abstract boolean couldAffectSchema(); - public abstract boolean execute(GlobalState globalState, String... fills) throws SQLException; + public abstract boolean execute(GlobalState globalState, String... fills) throws Exception; public abstract ExpectedErrors getExpectedErrors(); @@ -37,16 +35,16 @@ public String toString() { return getQueryString(); } - public SQLancerResultSet executeAndGet(GlobalState globalState, String... fills) throws SQLException { + public SQLancerResultSet executeAndGet(GlobalState globalState, String... fills) throws Exception { throw new AssertionError(); } - public boolean executeLogged(GlobalState globalState) throws SQLException { + public boolean executeLogged(GlobalState globalState) throws Exception { logQueryString(globalState); return execute(globalState); } - public SQLancerResultSet executeAndGetLogged(GlobalState globalState) throws SQLException { + public SQLancerResultSet executeAndGetLogged(GlobalState globalState) throws Exception { logQueryString(globalState); return executeAndGet(globalState); } diff --git a/src/sqlancer/common/query/QueryProvider.java b/src/sqlancer/common/query/QueryProvider.java index 628aaf323..a3a60bbbe 100644 --- a/src/sqlancer/common/query/QueryProvider.java +++ b/src/sqlancer/common/query/QueryProvider.java @@ -1,8 +1,6 @@ package sqlancer.common.query; -import java.sql.SQLException; - @FunctionalInterface public interface QueryProvider { - Query getQuery(S globalState) throws SQLException; + Query getQuery(S globalState) throws Exception; } diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index cf8d24183..b64d1063f 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -55,7 +55,7 @@ public enum Action implements AbstractAction { } @Override - public Query getQuery(DuckDBGlobalState state) throws SQLException { + public Query getQuery(DuckDBGlobalState state) throws Exception { return queryProvider.getQuery(state); } } @@ -95,7 +95,7 @@ protected DuckDBSchema readSchema() throws SQLException { } @Override - public void generateDatabase(DuckDBGlobalState globalState) throws SQLException { + public void generateDatabase(DuckDBGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { boolean success = false; do { diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 43102338f..5171c6304 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -38,7 +38,7 @@ public enum Action implements AbstractAction { } @Override - public Query getQuery(H2GlobalState state) throws SQLException { + public Query getQuery(H2GlobalState state) throws Exception { return queryProvider.getQuery(state); } } @@ -73,7 +73,7 @@ protected H2Schema readSchema() throws SQLException { } @Override - public void generateDatabase(H2GlobalState globalState) throws SQLException { + public void generateDatabase(H2GlobalState globalState) throws Exception { if (Randomly.getBoolean()) { H2SetGenerator.getQuery(globalState).execute(globalState); } diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index c9bbba54b..96f03a634 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -45,7 +45,7 @@ enum Action { } @Override - public void generateDatabase(MariaDBGlobalState globalState) throws SQLException { + public void generateDatabase(MariaDBGlobalState globalState) throws Exception { MainOptions options = globalState.getOptions(); while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index b9251c1a5..0913a0dd7 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -66,7 +66,7 @@ enum Action implements AbstractAction { } @Override - public Query getQuery(MySQLGlobalState globalState) throws SQLException { + public Query getQuery(MySQLGlobalState globalState) throws Exception { return queryProvider.getQuery(globalState); } } @@ -131,7 +131,7 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { } @Override - public void generateDatabase(MySQLGlobalState globalState) throws SQLException { + public void generateDatabase(MySQLGlobalState globalState) throws Exception { while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); Query createTable = MySQLTableGenerator.generate(globalState, tableName); diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index d8a76cb77..4014a0030 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -121,7 +121,7 @@ public enum Action implements AbstractAction { } @Override - public Query getQuery(PostgresGlobalState state) throws SQLException { + public Query getQuery(PostgresGlobalState state) throws Exception { return queryProvider.getQuery(state); } } @@ -188,7 +188,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { } @Override - public void generateDatabase(PostgresGlobalState globalState) throws SQLException { + public void generateDatabase(PostgresGlobalState globalState) throws Exception { readFunctions(globalState); createTables(globalState, Randomly.fromOptions(4, 5, 6)); prepareTables(globalState); @@ -268,7 +268,7 @@ protected void readFunctions(PostgresGlobalState globalState) throws SQLExceptio } } - protected void createTables(PostgresGlobalState globalState, int numTables) throws SQLException { + protected void createTables(PostgresGlobalState globalState, int numTables) throws Exception { while (globalState.getSchema().getDatabaseTables().size() < numTables) { try { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); @@ -281,7 +281,7 @@ protected void createTables(PostgresGlobalState globalState, int numTables) thro } } - protected void prepareTables(PostgresGlobalState globalState) throws SQLException { + protected void prepareTables(PostgresGlobalState globalState) throws Exception { StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), PostgresProvider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 868fc2876..ba410aba0 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -107,7 +107,7 @@ public boolean canBeRetried() { } @Override - public Query getQuery(SQLite3GlobalState state) throws SQLException { + public Query getQuery(SQLite3GlobalState state) throws Exception { return queryProvider.getQuery(state); } } @@ -178,7 +178,7 @@ private static int mapActions(SQLite3GlobalState globalState, Action a) { } @Override - public void generateDatabase(SQLite3GlobalState globalState) throws SQLException { + public void generateDatabase(SQLite3GlobalState globalState) throws Exception { Randomly r = new Randomly(SQLite3SpecialStringGenerator::generate); globalState.setRandomly(r); if (globalState.getDmbsSpecificOptions().generateDatabase) { @@ -221,7 +221,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws SQLException } } - private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws SQLException { + private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws Exception { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { Query q = new QueryAdapter("SELECT * FROM " + table.getName(), ExpectedErrors.from("needs an odd number of arguments", " requires an even number of arguments", @@ -263,7 +263,7 @@ private Query getTableQuery(SQLite3GlobalState globalState, int i) throws Assert return tableQuery; } - private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws SQLException { + private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws Exception { List pragmasToExecute = new ArrayList<>(); if (!Randomly.getBooleanWithSmallProbability()) { pragmasToExecute.addAll(DEFAULT_PRAGMAS); diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java index f67c8cd03..aca168919 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java @@ -1,7 +1,5 @@ package sqlancer.sqlite3.gen; -import java.sql.SQLException; - import sqlancer.Randomly; import sqlancer.common.query.Query; import sqlancer.common.query.QueryAdapter; @@ -14,7 +12,7 @@ public final class SQLite3ExplainGenerator { private SQLite3ExplainGenerator() { } - public static Query explain(SQLite3GlobalState globalState) throws SQLException { + public static Query explain(SQLite3GlobalState globalState) throws Exception { StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN "); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index fad867944..939b78833 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -1,7 +1,5 @@ package sqlancer.sqlite3.oracle; -import java.sql.SQLException; - import sqlancer.Randomly; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.QueryAdapter; @@ -18,7 +16,7 @@ public SQLite3Fuzzer(SQLite3GlobalState globalState) { } @Override - public void check() throws SQLException { + public void check() throws Exception { String s = SQLite3Visitor .asString(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + ";"; try { diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 23e47d072..882618aac 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -61,7 +61,7 @@ public enum Action implements AbstractAction { } @Override - public Query getQuery(TiDBGlobalState state) throws SQLException { + public Query getQuery(TiDBGlobalState state) throws Exception { return queryProvider.getQuery(state); } } @@ -103,7 +103,7 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { } @Override - public void generateDatabase(TiDBGlobalState globalState) throws SQLException { + public void generateDatabase(TiDBGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { boolean success = false; do { From ed579f965f63253b18681b30f23b2478816696ed Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 20 Nov 2020 17:33:18 +0100 Subject: [PATCH 0277/1284] Provide some general comparison guidelines --- docs/PAPERS.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/PAPERS.md b/docs/PAPERS.md index 916eee52d..ca9d40de7 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -50,3 +50,11 @@ This paper describes TLP, a metamorphic testing approach that can detect various articleno={211} } ``` + +# Comparing SQLancer With Other Tools that Find Logic Bugs + +If you want to fairly compare other tools with SQLancer, we would be glad to provide feedback (e.g., feel free to send an email to manuel.rigger@inf.ethz.ch). We have the following general recommendations and comments: +* PostgreSQL and SQLite are DBMSs that we comprehensively tested, and where all or most of the bugs that SQLancer could find were fixed. We believe these two systems to be the most challenging test targets. Finding bugs that the approaches implemented in SQLancer overlooked in these systems might thus best demonstrate a new approach's effectiveness. For some other DBMSs like MySQL and MariaDB, SQLancer could still detect unreported bugs; we stopped testing these DBMSs and reporting bugs due to the large number of unfixed bugs. +* We programmatically disabled the generation of features that are likely to trigger known bugs (e.g., see [TiDB](https://github.com/sqlancer/sqlancer/blob/master/src/sqlancer/tidb/TiDBBugs.java)). If a comparison investigates metrics such as code coverage that is achieved when fuzzing a DBMS, it might be desirable to enable the generation of such features. +* For the default SQLite JDBC driver, a number of extensions (e.g., the [soundex function](https://sqlite.org/lang_corefunc.html#soundex)) are disabled by default, which is why they are also disabled by default in the DBMS' options (e.g., see [SQLite3Options](https://github.com/sqlancer/sqlancer/blob/c71b9741f680f4877fc5047445787ed184a5a5e0/src/sqlancer/sqlite3/SQLite3Options.java#L67)). To investigate metrics such as code coverage, it might again be desirable to enable such options. +* The maximum expression depth (see the `--max-expression-depth` option), the number of queries issued per database (see the `--num-queries` option), and the number of tables and views that are created (currently, SQLancer does not have an option to set these) significantly influence the tool's effectiveness and performance characteristics. It might be desirable to experiment with different values for the expression depth (e.g., values between 2 and 4), the number of queries (1000-100,000), as well as the number of tables and views. From 97b8850ed697359d91f77950741e5bd4a0681449 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 23 Nov 2020 01:28:06 +0100 Subject: [PATCH 0278/1284] Rework Connection to DB to support non-SQL connections In order to achieve this goal, multiple core classes had to change the generics to add a new one C. The connection object of the sql package is now packed into the class SQLConnection. --- src/sqlancer/ComparatorHelper.java | 14 ++--- src/sqlancer/DBMSSpecificOptions.java | 2 +- src/sqlancer/DatabaseProvider.java | 7 +-- src/sqlancer/GlobalState.java | 43 +++++--------- src/sqlancer/Main.java | 59 +++++++++---------- src/sqlancer/OracleFactory.java | 2 +- src/sqlancer/ProviderAdapter.java | 18 +----- src/sqlancer/SQLConnection.java | 34 +++++++++++ src/sqlancer/SQLGlobalState.java | 15 +++++ src/sqlancer/SQLProviderAdapter.java | 21 ++++++- src/sqlancer/SQLancerDBConnection.java | 6 ++ src/sqlancer/StateToReproduce.java | 4 +- src/sqlancer/StatementExecutor.java | 2 +- src/sqlancer/citus/CitusProvider.java | 49 +++++++-------- src/sqlancer/citus/CitusSchema.java | 3 +- .../clickhouse/ClickHouseProvider.java | 9 +-- src/sqlancer/clickhouse/ClickHouseSchema.java | 15 ++--- .../gen/ClickHouseInsertGenerator.java | 4 +- .../gen/ClickHouseTableGenerator.java | 4 +- .../cockroachdb/CockroachDBProvider.java | 23 ++++---- .../cockroachdb/CockroachDBSchema.java | 17 +++--- .../gen/CockroachDBCommentOnGenerator.java | 4 +- .../CockroachDBCreateStatisticsGenerator.java | 4 +- .../gen/CockroachDBDeleteGenerator.java | 4 +- .../gen/CockroachDBInsertGenerator.java | 4 +- .../CockroachDBRandomQuerySynthesizer.java | 4 +- ...CockroachDBSetClusterSettingGenerator.java | 4 +- .../gen/CockroachDBSetSessionGenerator.java | 4 +- .../gen/CockroachDBShowGenerator.java | 4 +- .../gen/CockroachDBTruncateGenerator.java | 4 +- .../gen/CockroachDBUpdateGenerator.java | 4 +- .../gen/CockroachDBViewGenerator.java | 4 +- .../oracle/CockroachDBNoRECOracle.java | 10 ++-- .../tlp/CockroachDBTLPAggregateOracle.java | 4 +- .../common/ast/newast/TableReferenceNode.java | 2 +- .../common/gen/AbstractGenerator.java | 4 +- .../common/log/SQLLoggableFactory.java | 6 +- .../common/oracle/CompositeTestOracle.java | 8 +-- src/sqlancer/common/oracle/NoRECBase.java | 8 +-- .../oracle/PivotedQuerySynthesisBase.java | 4 +- .../TernaryLogicPartitioningOracleBase.java | 4 +- src/sqlancer/common/query/Query.java | 13 ++-- ...QueryAdapter.java => SQLQueryAdapter.java} | 17 +++--- ...r.java => SQLQueryResultCheckAdapter.java} | 9 +-- .../schema/AbstractRelationalTable.java | 35 +++++++++++ .../common/schema/AbstractRowValue.java | 4 +- .../common/schema/AbstractSchema.java | 4 +- src/sqlancer/common/schema/AbstractTable.java | 42 ++++--------- .../common/schema/AbstractTableColumn.java | 2 +- .../common/schema/AbstractTables.java | 2 +- src/sqlancer/duckdb/DuckDBProvider.java | 19 +++--- src/sqlancer/duckdb/DuckDBSchema.java | 15 ++--- .../duckdb/gen/DuckDBAlterTableGenerator.java | 4 +- .../duckdb/gen/DuckDBDeleteGenerator.java | 4 +- .../duckdb/gen/DuckDBIndexGenerator.java | 4 +- .../duckdb/gen/DuckDBInsertGenerator.java | 4 +- .../duckdb/gen/DuckDBTableGenerator.java | 4 +- .../duckdb/gen/DuckDBUpdateGenerator.java | 4 +- .../duckdb/gen/DuckDBViewGenerator.java | 4 +- .../duckdb/test/DuckDBNoRECOracle.java | 9 +-- ...uckDBQueryPartitioningAggregateTester.java | 4 +- src/sqlancer/h2/H2DeleteGenerator.java | 4 +- src/sqlancer/h2/H2IndexGenerator.java | 4 +- src/sqlancer/h2/H2InsertGenerator.java | 4 +- src/sqlancer/h2/H2Provider.java | 13 ++-- src/sqlancer/h2/H2Schema.java | 15 ++--- src/sqlancer/h2/H2SetGenerator.java | 4 +- src/sqlancer/h2/H2TableGenerator.java | 4 +- src/sqlancer/h2/H2UpdateGenerator.java | 4 +- src/sqlancer/h2/H2ViewGenerator.java | 4 +- src/sqlancer/mariadb/MariaDBProvider.java | 9 +-- src/sqlancer/mariadb/MariaDBSchema.java | 16 ++--- .../gen/MariaDBExpressionGenerator.java | 3 +- .../mariadb/gen/MariaDBIndexGenerator.java | 4 +- .../mariadb/gen/MariaDBInsertGenerator.java | 4 +- .../mariadb/gen/MariaDBSetGenerator.java | 4 +- .../MariaDBTableAdminCommandGenerator.java | 8 +-- .../mariadb/gen/MariaDBTableGenerator.java | 4 +- .../mariadb/gen/MariaDBTruncateGenerator.java | 4 +- .../mariadb/gen/MariaDBUpdateGenerator.java | 4 +- .../mariadb/oracle/MariaDBNoRECOracle.java | 6 +- src/sqlancer/mysql/MySQLGlobalState.java | 5 +- src/sqlancer/mysql/MySQLProvider.java | 11 ++-- src/sqlancer/mysql/MySQLSchema.java | 16 ++--- src/sqlancer/mysql/gen/MySQLAlterTable.java | 4 +- .../mysql/gen/MySQLDeleteGenerator.java | 4 +- src/sqlancer/mysql/gen/MySQLDropIndex.java | 4 +- .../mysql/gen/MySQLInsertGenerator.java | 4 +- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 4 +- .../mysql/gen/MySQLTableGenerator.java | 8 +-- .../gen/MySQLTruncateTableGenerator.java | 4 +- src/sqlancer/mysql/gen/admin/MySQLFlush.java | 4 +- src/sqlancer/mysql/gen/admin/MySQLReset.java | 4 +- .../gen/datadef/MySQLIndexGenerator.java | 4 +- .../gen/tblmaintenance/MySQLAnalyzeTable.java | 4 +- .../gen/tblmaintenance/MySQLCheckTable.java | 4 +- .../gen/tblmaintenance/MySQLChecksum.java | 4 +- .../gen/tblmaintenance/MySQLOptimize.java | 4 +- .../mysql/gen/tblmaintenance/MySQLRepair.java | 6 +- .../MySQLPivotedQuerySynthesisOracle.java | 6 +- .../postgres/PostgresGlobalState.java | 14 ++--- src/sqlancer/postgres/PostgresProvider.java | 23 ++++---- src/sqlancer/postgres/PostgresSchema.java | 18 +++--- .../gen/PostgresAlterTableGenerator.java | 4 +- .../gen/PostgresAnalyzeGenerator.java | 4 +- .../gen/PostgresClusterGenerator.java | 4 +- .../gen/PostgresCommentGenerator.java | 4 +- .../postgres/gen/PostgresDeleteGenerator.java | 4 +- .../gen/PostgresDiscardGenerator.java | 4 +- .../gen/PostgresDropIndexGenerator.java | 4 +- .../postgres/gen/PostgresIndexGenerator.java | 4 +- .../postgres/gen/PostgresInsertGenerator.java | 4 +- .../postgres/gen/PostgresNotifyGenerator.java | 8 +-- .../gen/PostgresQueryCatalogGenerator.java | 9 +-- .../gen/PostgresReindexGenerator.java | 4 +- .../gen/PostgresSequenceGenerator.java | 4 +- .../postgres/gen/PostgresSetGenerator.java | 4 +- .../gen/PostgresStatisticsGenerator.java | 6 +- .../postgres/gen/PostgresTableGenerator.java | 4 +- .../gen/PostgresTransactionGenerator.java | 4 +- .../gen/PostgresTruncateGenerator.java | 4 +- .../postgres/gen/PostgresUpdateGenerator.java | 4 +- .../postgres/gen/PostgresVacuumGenerator.java | 4 +- .../postgres/gen/PostgresViewGenerator.java | 4 +- .../postgres/oracle/PostgresNoRECOracle.java | 4 +- .../PostgresPivotedQuerySynthesisOracle.java | 6 +- .../tlp/PostgresTLPAggregateOracle.java | 4 +- src/sqlancer/sqlite3/SQLite3Provider.java | 19 +++--- .../sqlite3/gen/SQLite3AnalyzeGenerator.java | 4 +- ...Lite3CreateVirtualRtreeTabelGenerator.java | 4 +- .../sqlite3/gen/SQLite3ExplainGenerator.java | 4 +- .../sqlite3/gen/SQLite3PragmaGenerator.java | 8 +-- .../sqlite3/gen/SQLite3ReindexGenerator.java | 4 +- .../gen/SQLite3TransactionGenerator.java | 8 +-- .../sqlite3/gen/SQLite3VacuumGenerator.java | 4 +- ...QLite3VirtualFTSTableCommandGenerator.java | 4 +- .../sqlite3/gen/ddl/SQLite3AlterTable.java | 4 +- .../ddl/SQLite3CreateTriggerGenerator.java | 4 +- ...SQLite3CreateVirtualFTSTableGenerator.java | 4 +- .../gen/ddl/SQLite3DropIndexGenerator.java | 4 +- .../gen/ddl/SQLite3DropTableGenerator.java | 4 +- .../gen/ddl/SQLite3IndexGenerator.java | 4 +- .../gen/ddl/SQLite3TableGenerator.java | 4 +- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 6 +- .../gen/dml/SQLite3DeleteGenerator.java | 4 +- .../gen/dml/SQLite3InsertGenerator.java | 4 +- .../gen/dml/SQLite3StatTableGenerator.java | 4 +- .../gen/dml/SQLite3UpdateGenerator.java | 4 +- .../sqlite3/oracle/SQLite3Fuzzer.java | 4 +- .../sqlite3/oracle/SQLite3NoRECOracle.java | 10 ++-- .../SQLite3PivotedQuerySynthesisOracle.java | 6 +- .../oracle/tlp/SQLite3TLPAggregateOracle.java | 6 +- .../sqlite3/schema/SQLite3Schema.java | 14 ++--- src/sqlancer/tidb/TiDBProvider.java | 17 +++--- src/sqlancer/tidb/TiDBSchema.java | 17 +++--- .../tidb/gen/TiDBAlterTableGenerator.java | 4 +- .../tidb/gen/TiDBAnalyzeTableGenerator.java | 4 +- .../tidb/gen/TiDBDeleteGenerator.java | 4 +- src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 4 +- .../tidb/gen/TiDBInsertGenerator.java | 4 +- .../tidb/gen/TiDBRandomQuerySynthesizer.java | 4 +- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 4 +- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 4 +- .../tidb/gen/TiDBUpdateGenerator.java | 4 +- src/sqlancer/tidb/gen/TiDBViewGenerator.java | 4 +- test/sqlancer/TestParameterFormat.java | 2 +- 166 files changed, 675 insertions(+), 588 deletions(-) create mode 100644 src/sqlancer/SQLConnection.java create mode 100644 src/sqlancer/SQLGlobalState.java create mode 100644 src/sqlancer/SQLancerDBConnection.java rename src/sqlancer/common/query/{QueryAdapter.java => SQLQueryAdapter.java} (86%) rename src/sqlancer/common/query/{QueryResultCheckAdapter.java => SQLQueryResultCheckAdapter.java} (64%) create mode 100644 src/sqlancer/common/schema/AbstractRelationalTable.java diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 3f595a721..3233ab243 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -9,7 +9,7 @@ import java.util.stream.Collectors; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; public final class ComparatorHelper { @@ -35,8 +35,8 @@ static boolean equals(double a, double b) { return Math.abs(a - b) < 0.001 * Math.max(Math.abs(a), Math.abs(b)) + 0.001; } - public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, - GlobalState state) throws SQLException { + public static List getResultSetFirstColumnAsString(String queryString, + ExpectedErrors errors, SQLGlobalState state) throws SQLException { if (state.getOptions().logEachSelect()) { // TODO: refactor me state.getLogger().writeCurrent(queryString); @@ -47,7 +47,7 @@ public static List getResultSetFirstColumnAsString(String queryString, E e.printStackTrace(); } } - QueryAdapter q = new QueryAdapter(queryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); List resultSet = new ArrayList<>(); SQLancerResultSet result = null; try { @@ -82,7 +82,7 @@ public static List getResultSetFirstColumnAsString(String queryString, E } public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - String originalQueryString, List combinedString, GlobalState state) { + String originalQueryString, List combinedString, SQLGlobalState state) { if (resultSet.size() != secondResultSet.size()) { String queryFormatString = "-- %s;\n-- cardinality: %d"; String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); @@ -115,7 +115,7 @@ public static void assumeResultSetsAreEqual(List resultSet, List } public static List getCombinedResultSet(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, + String thirdQueryString, List combinedString, boolean asUnion, SQLGlobalState state, ExpectedErrors errors) throws SQLException { List secondResultSet; if (asUnion) { @@ -136,7 +136,7 @@ public static List getCombinedResultSet(String firstQueryString, String } public static List getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, + String thirdQueryString, List combinedString, boolean asUnion, SQLGlobalState state, ExpectedErrors errors) throws SQLException { String unionString; if (asUnion) { diff --git a/src/sqlancer/DBMSSpecificOptions.java b/src/sqlancer/DBMSSpecificOptions.java index 0cb7daf92..4607557d1 100644 --- a/src/sqlancer/DBMSSpecificOptions.java +++ b/src/sqlancer/DBMSSpecificOptions.java @@ -2,7 +2,7 @@ import java.util.List; -public interface DBMSSpecificOptions>> { +public interface DBMSSpecificOptions>> { List getTestOracleFactory(); diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 9351bbe6d..6530eddcc 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -1,10 +1,9 @@ package sqlancer; -import java.sql.Connection; - import sqlancer.common.log.LoggableFactory; -public interface DatabaseProvider, O extends DBMSSpecificOptions> { +public interface DatabaseProvider, O extends DBMSSpecificOptions, + C extends SQLancerDBConnection> { /** * Gets the the {@link GlobalState} class. @@ -29,7 +28,7 @@ public interface DatabaseProvider, O extends DBMSSpe */ void generateAndTestDatabase(G globalState) throws Exception; - Connection createDatabase(G globalState) throws Exception; + C createDatabase(G globalState) throws Exception; /** * The DBMS name is used to name the log directory and command to test the respective DBMS. diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 3d7ddcecc..7476171c9 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,36 +1,28 @@ package sqlancer; -import java.sql.Connection; - -import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; -/** - * Represents a global state that is valid for a testing session on a given database. - * - * @param - * the option parameter - * @param - * the schema parameter - */ -public abstract class GlobalState, S extends AbstractSchema> { - - private Connection con; +public abstract class GlobalState, S extends AbstractSchema, C extends SQLancerDBConnection> { + + protected C databaseConnection; private Randomly r; private MainOptions options; private O dmbsSpecificOptions; private S schema; - private StateLogger logger; + private Main.StateLogger logger; private StateToReproduce state; - private QueryManager manager; + private Main.QueryManager manager; private String databaseName; - public void setConnection(Connection con) { - this.con = con; + public void setConnection(C con) { + this.databaseConnection = con; + } + + public C getConnection() { + return databaseConnection; } @SuppressWarnings("unchecked") @@ -42,9 +34,6 @@ public O getDmbsSpecificOptions() { return dmbsSpecificOptions; } - public Connection getConnection() { - return con; - } public void setRandomly(Randomly r) { this.r = r; @@ -62,11 +51,11 @@ public void setMainOptions(MainOptions options) { this.options = options; } - public void setStateLogger(StateLogger logger) { + public void setStateLogger(Main.StateLogger logger) { this.logger = logger; } - public StateLogger getLogger() { + public Main.StateLogger getLogger() { return logger; } @@ -78,11 +67,11 @@ public StateToReproduce getState() { return state; } - public QueryManager getManager() { + public Main.QueryManager getManager() { return manager; } - public void setManager(QueryManager manager) { + public void setManager(Main.QueryManager manager) { this.manager = manager; } @@ -166,7 +155,7 @@ protected void setSchema(S schema) { public void updateSchema() throws Exception { setSchema(readSchema()); - for (AbstractTable table : schema.getDatabaseTables()) { + for (AbstractTable table : schema.getDatabaseTables()) { table.recomputeCount(); } } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index cf83bf76e..1a8e2ec93 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -5,8 +5,6 @@ import java.io.IOException; import java.io.Writer; import java.nio.file.Files; -import java.sql.Connection; -import java.sql.SQLFeatureNotSupportedException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -65,7 +63,7 @@ public static final class StateLogger { public FileWriter currentFileWriter; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; - private final DatabaseProvider databaseProvider; + private final DatabaseProvider databaseProvider; private static final class AlsoWriteToConsoleFileWriter extends FileWriter { @@ -86,7 +84,7 @@ public void write(String str) throws IOException { } } - public StateLogger(String databaseName, DatabaseProvider provider, MainOptions options) { + public StateLogger(String databaseName, DatabaseProvider provider, MainOptions options) { File dir = new File(LOG_DIRECTORY, provider.getDBMSName()); if (dir.exists() && !dir.isDirectory()) { throw new AssertionError(dir); @@ -100,7 +98,7 @@ public StateLogger(String databaseName, DatabaseProvider provider, MainOpt this.databaseProvider = provider; } - private void ensureExistsAndIsEmpty(File dir, DatabaseProvider provider) { + private void ensureExistsAndIsEmpty(File dir, DatabaseProvider provider) { if (INITIALIZED_PROVIDER_NAMES.contains(provider.getDBMSName())) { return; } @@ -225,9 +223,9 @@ private void printState(FileWriter writer, StateToReproduce state) { public static class QueryManager { - private final GlobalState globalState; + private final GlobalState globalState; - QueryManager(GlobalState globalState) { + QueryManager(GlobalState globalState) { this.globalState = globalState; } @@ -261,9 +259,10 @@ public static void main(String[] args) { System.exit(executeMain(args)); } - public static class DBMSExecutor, O extends DBMSSpecificOptions> { + public static class DBMSExecutor, O extends DBMSSpecificOptions, C + extends SQLancerDBConnection> { - private final DatabaseProvider provider; + private final DatabaseProvider provider; private final MainOptions options; private final O command; private final String databaseName; @@ -271,7 +270,7 @@ public static class DBMSExecutor, O extends DBMSSpec private StateToReproduce stateToRepro; private final Randomly r; - public DBMSExecutor(DatabaseProvider provider, MainOptions options, O dbmsSpecificOptions, + public DBMSExecutor(DatabaseProvider provider, MainOptions options, O dbmsSpecificOptions, String databaseName, Randomly r) { this.provider = provider; this.options = options; @@ -294,7 +293,7 @@ public O getCommand() { public void testConnection() throws Exception { G state = getInitializedGlobalState(options.getRandomSeed()); - try (Connection con = provider.createDatabase(state)) { + try (SQLancerDBConnection con = provider.createDatabase(state)) { return; } } @@ -309,12 +308,11 @@ public void run() throws Exception { state.setDatabaseName(databaseName); state.setMainOptions(options); state.setDmbsSpecificOptions(command); - try (Connection con = provider.createDatabase(state)) { + try (C con = provider.createDatabase(state)) { QueryManager manager = new QueryManager(state); try { - java.sql.DatabaseMetaData meta = con.getMetaData(); - stateToRepro.databaseVersion = meta.getDatabaseProductVersion(); - } catch (SQLFeatureNotSupportedException e) { + stateToRepro.databaseVersion = con.getDatabaseVersion(); + } catch (Exception e) { // ignore } state.setConnection(con); @@ -356,13 +354,14 @@ public StateToReproduce getStateToReproduce() { } } - public static class DBMSExecutorFactory, O extends DBMSSpecificOptions> { + public static class DBMSExecutorFactory, O extends DBMSSpecificOptions, + C extends SQLancerDBConnection> { - private final DatabaseProvider provider; + private final DatabaseProvider provider; private final MainOptions options; private final O command; - public DBMSExecutorFactory(DatabaseProvider provider, MainOptions options) { + public DBMSExecutorFactory(DatabaseProvider provider, MainOptions options) { this.provider = provider; this.options = options; this.command = createCommand(); @@ -381,32 +380,32 @@ public O getCommand() { } @SuppressWarnings("unchecked") - public DBMSExecutor getDBMSExecutor(String databaseName, Randomly r) { + public DBMSExecutor getDBMSExecutor(String databaseName, Randomly r) { try { - return new DBMSExecutor(provider.getClass().getDeclaredConstructor().newInstance(), options, + return new DBMSExecutor(provider.getClass().getDeclaredConstructor().newInstance(), options, command, databaseName, r); } catch (Exception e) { throw new AssertionError(e); } } - public DatabaseProvider getProvider() { + public DatabaseProvider getProvider() { return provider; } } public static int executeMain(String... args) throws AssertionError { - List> providers = getDBMSProviders(); - Map> nameToProvider = new HashMap<>(); + List> providers = getDBMSProviders(); + Map> nameToProvider = new HashMap<>(); MainOptions options = new MainOptions(); Builder commandBuilder = JCommander.newBuilder().addObject(options); - for (DatabaseProvider provider : providers) { + for (DatabaseProvider provider : providers) { String name = provider.getDBMSName(); if (!name.toLowerCase().equals(name)) { throw new AssertionError(name + " should be in lowercase!"); } - DBMSExecutorFactory executorFactory = new DBMSExecutorFactory<>(provider, options); + DBMSExecutorFactory executorFactory = new DBMSExecutorFactory<>(provider, options); commandBuilder = commandBuilder.addCommand(name, executorFactory.getCommand()); nameToProvider.put(name, executorFactory); } @@ -448,7 +447,7 @@ private String formatInteger(long intValue) { } ExecutorService execService = Executors.newFixedThreadPool(options.getNumberConcurrentThreads()); - DBMSExecutorFactory executorFactory = nameToProvider.get(jc.getParsedCommand()); + DBMSExecutorFactory executorFactory = nameToProvider.get(jc.getParsedCommand()); if (options.performConnectionTest()) { try { @@ -504,8 +503,8 @@ private void runThread(final String databaseName) { } private boolean run(MainOptions options, ExecutorService execService, - DBMSExecutorFactory executorFactory, Randomly r, final String databaseName) { - DBMSExecutor executor = executorFactory.getDBMSExecutor(databaseName, r); + DBMSExecutorFactory executorFactory, Randomly r, final String databaseName) { + DBMSExecutor executor = executorFactory.getDBMSExecutor(databaseName, r); try { executor.run(); return true; @@ -545,8 +544,8 @@ private boolean run(MainOptions options, ExecutorService execService, return threadsShutdown == 0 ? 0 : options.getErrorExitCode(); } - static List> getDBMSProviders() { - List> providers = new ArrayList<>(); + static List> getDBMSProviders() { + List> providers = new ArrayList<>(); providers.add(new SQLite3Provider()); providers.add(new CockroachDBProvider()); providers.add(new MySQLProvider()); diff --git a/src/sqlancer/OracleFactory.java b/src/sqlancer/OracleFactory.java index 8e096eeec..897293c6b 100644 --- a/src/sqlancer/OracleFactory.java +++ b/src/sqlancer/OracleFactory.java @@ -2,7 +2,7 @@ import sqlancer.common.oracle.TestOracle; -public interface OracleFactory> { +public interface OracleFactory> { TestOracle create(G globalState) throws Exception; diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 104398959..caeb20314 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -6,11 +6,9 @@ import sqlancer.StateToReproduce.OracleRunReproductionState; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.QueryAdapter; -import sqlancer.common.schema.AbstractTable; -public abstract class ProviderAdapter, O extends DBMSSpecificOptions>> - implements DatabaseProvider { +public abstract class ProviderAdapter, O extends DBMSSpecificOptions>, + C extends SQLancerDBConnection> implements DatabaseProvider { private final Class globalClass; private final Class optionClass; @@ -61,17 +59,7 @@ public void generateAndTestDatabase(G globalState) throws Exception { } } - private void checkViewsAreValid(G globalState) { - List> views = globalState.getSchema().getViews(); - for (AbstractTable view : views) { - QueryAdapter q = new QueryAdapter("SELECT 1 FROM " + view.getName() + " LIMIT 1"); - try { - q.execute(globalState); - } catch (Throwable t) { - throw new IgnoreMeException(); - } - } - } + protected abstract void checkViewsAreValid(G globalState); protected TestOracle getTestOracle(G globalState) throws Exception { List> testOracleFactory = globalState.getDmbsSpecificOptions() diff --git a/src/sqlancer/SQLConnection.java b/src/sqlancer/SQLConnection.java new file mode 100644 index 000000000..cb72b40d0 --- /dev/null +++ b/src/sqlancer/SQLConnection.java @@ -0,0 +1,34 @@ +package sqlancer; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.sql.Statement; + +public class SQLConnection implements SQLancerDBConnection { + + private final Connection connection; + + public SQLConnection(Connection connection) { + this.connection = connection; + } + + @Override + public String getDatabaseVersion() throws SQLException{ + DatabaseMetaData meta = connection.getMetaData(); + return meta.getDatabaseProductVersion(); + } + + @Override + public void close() throws SQLException { + connection.close(); + } + + public Statement prepareStatement(String arg) throws SQLException { + return connection.prepareStatement(arg); + } + + public Statement createStatement() throws SQLException{ + return connection.createStatement(); + } +} diff --git a/src/sqlancer/SQLGlobalState.java b/src/sqlancer/SQLGlobalState.java new file mode 100644 index 000000000..0101fb577 --- /dev/null +++ b/src/sqlancer/SQLGlobalState.java @@ -0,0 +1,15 @@ +package sqlancer; + +import sqlancer.common.schema.AbstractSchema; + +/** + * Represents a global state that is valid for a testing session on a given database. + * + * @param + * the option parameter + * @param + * the schema parameter + */ +public abstract class SQLGlobalState, S extends AbstractSchema> + extends GlobalState { +} diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java index ca4563768..91468f27e 100644 --- a/src/sqlancer/SQLProviderAdapter.java +++ b/src/sqlancer/SQLProviderAdapter.java @@ -2,9 +2,13 @@ import sqlancer.common.log.LoggableFactory; import sqlancer.common.log.SQLLoggableFactory; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractRelationalTable; -public abstract class SQLProviderAdapter, O extends DBMSSpecificOptions>> - extends ProviderAdapter { +import java.util.List; + +public abstract class SQLProviderAdapter, O extends DBMSSpecificOptions>> + extends ProviderAdapter { public SQLProviderAdapter(Class globalClass, Class optionClass) { super(globalClass, optionClass); } @@ -13,4 +17,17 @@ public SQLProviderAdapter(Class globalClass, Class optionClass) { public LoggableFactory getLoggableFactory() { return new SQLLoggableFactory(); } + + @Override + protected void checkViewsAreValid(G globalState) { + List> views = globalState.getSchema().getViews(); + for (AbstractRelationalTable view : views) { + SQLQueryAdapter q = new SQLQueryAdapter("SELECT 1 FROM " + view.getName() + " LIMIT 1"); + try { + q.execute(globalState); + } catch (Throwable t) { + throw new IgnoreMeException(); + } + } + } } diff --git a/src/sqlancer/SQLancerDBConnection.java b/src/sqlancer/SQLancerDBConnection.java new file mode 100644 index 000000000..1724dda6c --- /dev/null +++ b/src/sqlancer/SQLancerDBConnection.java @@ -0,0 +1,6 @@ +package sqlancer; + +public interface SQLancerDBConnection extends AutoCloseable { + + String getDatabaseVersion() throws Exception; +} diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 071c20bdc..4477c25cb 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -13,7 +13,7 @@ public class StateToReproduce { private final String databaseName; - private final DatabaseProvider databaseProvider; + private final DatabaseProvider databaseProvider; public String databaseVersion; @@ -23,7 +23,7 @@ public class StateToReproduce { public OracleRunReproductionState localState; - public StateToReproduce(String databaseName, DatabaseProvider databaseProvider) { + public StateToReproduce(String databaseName, DatabaseProvider databaseProvider) { this.databaseName = databaseName; this.databaseProvider = databaseProvider; } diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index 6034c5ad2..8b09e2a6f 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -5,7 +5,7 @@ import sqlancer.common.query.Query; -public class StatementExecutor, A extends AbstractAction> { +public class StatementExecutor, A extends AbstractAction> { private final G globalState; private final A[] actions; diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index a2de975f7..2e69a6b1d 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -14,6 +14,7 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.StatementExecutor; import sqlancer.citus.gen.CitusAlterTableGenerator; import sqlancer.citus.gen.CitusCommon; @@ -27,7 +28,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresGlobalState; @@ -67,11 +68,11 @@ public enum Action implements AbstractAction { COMMIT(g -> { Query query; if (Randomly.getBoolean()) { - query = new QueryAdapter("COMMIT", true); + query = new SQLQueryAdapter("COMMIT", true); } else if (Randomly.getBoolean()) { query = PostgresTransactionGenerator.executeBegin(); } else { - query = new QueryAdapter("ROLLBACK", true); + query = new SQLQueryAdapter("ROLLBACK", true); } return query; }), // @@ -91,11 +92,11 @@ public enum Action implements AbstractAction { StringBuilder sb = new StringBuilder(); sb.append("SET CONSTRAINTS ALL "); sb.append(Randomly.fromOptions("DEFERRED", "IMMEDIATE")); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); }), // - RESET_ROLE((g) -> new QueryAdapter("RESET ROLE")), // + RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // COMMENT_ON(PostgresCommentGenerator::generate), // - RESET((g) -> new QueryAdapter("RESET ALL") /* + RESET((g) -> new SQLQueryAdapter("RESET ALL") /* * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also * configuration parameter */), // @@ -205,7 +206,7 @@ private static void distributeTable(List columns, String tableNa PostgresColumn columnToDistribute = Randomly.fromList(columns); String queryString = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + "');"; - QueryAdapter query = new QueryAdapter(queryString, getCitusErrors()); + SQLQueryAdapter query = new SQLQueryAdapter(queryString, getCitusErrors()); globalState.executeStatement(query, "SELECT create_distributed_table(?, ?);", tableName, columnToDistribute.getName()); } @@ -217,7 +218,7 @@ private static List getTableConstraints(String tableName, CitusGlobalSta String queryString = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + tableName + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; - QueryAdapter query = new QueryAdapter(queryString); + SQLQueryAdapter query = new SQLQueryAdapter(queryString); SQLancerResultSet rs = query.executeAndGet(globalState, "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');", tableName); @@ -233,7 +234,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl if (tableConstraints.isEmpty()) { String queryString = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + tableName + "';"; - QueryAdapter query = new QueryAdapter(queryString); + SQLQueryAdapter query = new SQLQueryAdapter(queryString); SQLancerResultSet rs = query.executeAndGet(globalState, "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;", tableName); while (rs.next()) { @@ -248,7 +249,7 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl HashMap> columnConstraints = new HashMap<>(); String queryString = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + tableName + "';"; - QueryAdapter query = new QueryAdapter(queryString); + SQLQueryAdapter query = new SQLQueryAdapter(queryString); SQLancerResultSet rs = query.executeAndGet(globalState, "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;", tableName); @@ -284,7 +285,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws Exception { if (Randomly.getBooleanWithRatherLowProbability()) { // create reference table String queryString = "SELECT create_reference_table('" + table.getName() + "');"; - QueryAdapter query = new QueryAdapter(queryString, getCitusErrors()); + SQLQueryAdapter query = new SQLQueryAdapter(queryString, getCitusErrors()); globalState.executeStatement(query, "SELECT create_reference_table(?);", table.getName()); } else { // create distributed table @@ -298,7 +299,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws Exception { if (((CitusGlobalState) globalState).getRepartition()) { // allow repartition joins globalState.executeStatement( - new QueryAdapter("SET citus.enable_repartition_joins to ON;\n", getCitusErrors())); + new SQLQueryAdapter("SET citus.enable_repartition_joins to ON;\n", getCitusErrors())); } } @@ -314,7 +315,7 @@ protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLEx return new CompositeTestOracle(oracles, globalState); } - private List readCitusWorkerNodes(PostgresGlobalState globalState, Connection con) + private List readCitusWorkerNodes(PostgresGlobalState globalState, SQLConnection con) throws SQLException { globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); List citusWorkerNodes = new ArrayList<>(); @@ -330,7 +331,7 @@ private List readCitusWorkerNodes(PostgresGlobalState globalSta return citusWorkerNodes; } - private void addCitusExtension(PostgresGlobalState globalState, Connection con) throws SQLException { + private void addCitusExtension(PostgresGlobalState globalState, SQLConnection con) throws SQLException { globalState.getState().logStatement("CREATE EXTENSION citus;"); try (Statement s = con.createStatement()) { s.execute("CREATE EXTENSION citus;"); @@ -347,7 +348,7 @@ private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes) throws SQLException { for (CitusWorkerNode w : citusWorkerNodes) { String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; @@ -386,10 +387,10 @@ private void addCitusWorkerNodes(PostgresGlobalState globalState, Connection con } @Override - public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLException { synchronized (CitusProvider.class) { // returns connection to coordinator node, test database - Connection con = super.createDatabase(globalState); + SQLConnection con = super.createDatabase(globalState); String entryDatabaseName = entryPath.substring(1); int databaseIndex = entryURL.indexOf(entryPath) + 1; // add citus extension to coordinator node, test database @@ -398,7 +399,7 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce // reconnect to coordinator node, entry database globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - con = DriverManager.getConnection("jdbc:" + entryURL, username, password); + con = new SQLConnection(DriverManager.getConnection("jdbc:" + entryURL, username, password)); // read info about worker nodes List citusWorkerNodes = readCitusWorkerNodes(globalState, con); con.close(); @@ -409,13 +410,13 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce // reconnect to coordinator node, test database globalState.getState().logStatement("\\q"); globalState.getState().logStatement(testURL); - con = DriverManager.getConnection("jdbc:" + testURL, username, password); + con = new SQLConnection(DriverManager.getConnection("jdbc:" + testURL, username, password)); // add worker nodes to coordinator node for test database addCitusWorkerNodes(globalState, con, citusWorkerNodes); con.close(); // reconnect to coordinator node, test database - con = DriverManager.getConnection("jdbc:" + testURL, username, password); + con = new SQLConnection(DriverManager.getConnection("jdbc:" + testURL, username, password)); ((CitusGlobalState) globalState) .setRepartition(((CitusOptions) globalState.getDmbsSpecificOptions()).repartition); globalState.getState().commentStatements(); @@ -432,8 +433,8 @@ protected void prepareTables(PostgresGlobalState globalState) throws Exception { } }); se.executeStatements(); - globalState.executeStatement(new QueryAdapter("COMMIT", true)); - globalState.executeStatement(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 5000;\n")); } @Override diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index 03d7669a6..87f0f6b37 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.SQLConnection; import sqlancer.postgres.PostgresSchema; public class CitusSchema extends PostgresSchema { @@ -56,7 +57,7 @@ public Integer getColocationId() { } - public static CitusSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static CitusSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); try { List databaseTables = new ArrayList<>(); diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index aa403ae01..b30c3e1ff 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -7,7 +7,8 @@ import java.util.stream.Collectors; import sqlancer.AbstractAction; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLProviderAdapter; @@ -51,7 +52,7 @@ private static int mapActions(ClickHouseGlobalState globalState, Action a) { } } - public static class ClickHouseGlobalState extends GlobalState { + public static class ClickHouseGlobalState extends SQLGlobalState { private ClickHouseOptions clickHouseOptions; @@ -100,7 +101,7 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws Exception } @Override - public Connection createDatabase(ClickHouseGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQLException { ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); String url = "jdbc:clickhouse://localhost:8123/default"; @@ -126,7 +127,7 @@ public Connection createDatabase(ClickHouseGlobalState globalState) throws SQLEx con.close(); con = DriverManager.getConnection("jdbc:clickhouse://localhost:8123/" + databaseName, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - return con; + return new SQLConnection(con); } @Override diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index c9e77f30a..86011b4da 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -1,6 +1,5 @@ package sqlancer.clickhouse; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -11,16 +10,18 @@ import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ast.ClickHouseConstant; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; -public class ClickHouseSchema extends AbstractSchema { +public class ClickHouseSchema extends AbstractSchema { public static class ClickHouseLancerDataType { @@ -167,7 +168,7 @@ private static ClickHouseLancerDataType getColumnType(String typeString) { return new ClickHouseLancerDataType(typeString); } - public static class ClickHouseTable extends AbstractTable { + public static class ClickHouseTable extends AbstractRelationalTable { public ClickHouseTable(String tableName, List columns, List indexes, boolean isView) { @@ -175,7 +176,7 @@ public ClickHouseTable(String tableName, List columns, List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { @@ -192,7 +193,7 @@ public static ClickHouseSchema fromConnection(Connection con, String databaseNam return new ClickHouseSchema(databaseTables); } - private static List getTableNames(Connection con) throws SQLException { + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { ResultSet tableRs = s.executeQuery("SHOW TABLES"); @@ -204,7 +205,7 @@ private static List getTableNames(Connection con) throws SQLException { return tableNames; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("DESCRIBE " + tableName)) { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index fb4cfb475..21c8cc870 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -12,7 +12,7 @@ import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public class ClickHouseInsertGenerator extends AbstractInsertGenerator { @@ -46,7 +46,7 @@ private Query get() { sb.append(")"); sb.append(" VALUES "); insertColumns(columns); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index b31175c8d..13f7a152f 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -12,7 +12,7 @@ import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public class ClickHouseTableGenerator { @@ -38,7 +38,7 @@ public static Query createTableStatement(String tableName, ClickHouseProvider.Cl chTableGenerator.start(); ExpectedErrors errors = new ExpectedErrors(); ClickHouseErrors.addTableManipulationErrors(errors); - return new QueryAdapter(chTableGenerator.sb.toString(), errors, true); + return new SQLQueryAdapter(chTableGenerator.sb.toString(), errors, true); } public void start() { diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index e58a13244..d0cee36f3 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -8,7 +8,8 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; import sqlancer.MainOptions; @@ -31,7 +32,7 @@ import sqlancer.cockroachdb.gen.CockroachDBViewGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.QueryProvider; public class CockroachDBProvider extends SQLProviderAdapter { @@ -54,7 +55,7 @@ public enum Action { SHOW(CockroachDBShowGenerator::show), // TRANSACTION((g) -> { String s = Randomly.fromOptions("BEGIN", "ROLLBACK", "COMMIT"); - return new QueryAdapter(s, + return new SQLQueryAdapter(s, ExpectedErrors.from("there is no transaction in progress", "there is already a transaction in progress", "current transaction is aborted", "does not exist" /* interleaved indexes */)); @@ -74,9 +75,9 @@ public enum Action { } sb.append(CockroachDBRandomQuerySynthesizer.generate(g, Randomly.smallNumber() + 1)); CockroachDBErrors.addExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); }), // - SCRUB((g) -> new QueryAdapter( + SCRUB((g) -> new SQLQueryAdapter( "EXPERIMENTAL SCRUB table " + g.getSchema().getRandomTable(t -> !t.isView()).getName(), // https://github.com/cockroachdb/cockroach/issues/46401 ExpectedErrors.from("scrub-fk: column \"t.rowid\" does not exist", @@ -96,7 +97,7 @@ public enum Action { } else { sb.append(" SPLIT AT VALUES (NULL);"); } - return new QueryAdapter(sb.toString(), ExpectedErrors.from("must be of type")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("must be of type")); }); private final QueryProvider queryProvider; @@ -110,7 +111,7 @@ public Query getQuery(CockroachDBGlobalState state) throws Exception { } } - public static class CockroachDBGlobalState extends GlobalState { + public static class CockroachDBGlobalState extends SQLGlobalState { @Override protected CockroachDBSchema readSchema() throws SQLException { @@ -142,7 +143,7 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio standardSettings.add("SET experimental_enable_temp_tables = 'on'"); } for (String s : standardSettings) { - manager.execute(new QueryAdapter(s)); + manager.execute(new SQLQueryAdapter(s)); } for (int i = 0; i < Randomly.fromOptions(2, 3); i++) { @@ -242,12 +243,12 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio total--; } if (globalState.getDmbsSpecificOptions().makeVectorizationMoreLikely && Randomly.getBoolean()) { - manager.execute(new QueryAdapter("SET vectorize=on;")); + manager.execute(new SQLQueryAdapter("SET vectorize=on;")); } } @Override - public Connection createDatabase(CockroachDBGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws SQLException { String databaseName = globalState.getDatabaseName(); String url = "jdbc:postgresql://localhost:26257/test"; Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), @@ -272,7 +273,7 @@ public Connection createDatabase(CockroachDBGlobalState globalState) throws SQLE con.close(); con = DriverManager.getConnection("jdbc:postgresql://localhost:26257/" + databaseName, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - return con; + return new SQLConnection(con); } @Override diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index bb1b18664..f51fdce88 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -1,6 +1,5 @@ package sqlancer.cockroachdb; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -8,14 +7,16 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; -public class CockroachDBSchema extends AbstractSchema { +public class CockroachDBSchema extends AbstractSchema { public enum CockroachDBDataType { @@ -263,7 +264,7 @@ private static CockroachDBCompositeDataType getColumnType(String typeString) { } } - public static class CockroachDBTable extends AbstractTable { + public static class CockroachDBTable extends AbstractRelationalTable { public CockroachDBTable(String tableName, List columns, List indexes, boolean isView) { @@ -272,7 +273,7 @@ public CockroachDBTable(String tableName, List columns, List< } - public static CockroachDBSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static CockroachDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { @@ -289,7 +290,7 @@ public static CockroachDBSchema fromConnection(Connection con, String databaseNa return new CockroachDBSchema(databaseTables); } - private static List getTableNames(Connection con) throws SQLException { + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { ResultSet tableRs = s.executeQuery( @@ -302,7 +303,7 @@ private static List getTableNames(Connection con) throws SQLException { return tableNames; } - private static List getIndexes(Connection con, String tableName) throws SQLException { + private static List getIndexes(SQLConnection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format("SHOW INDEX FROM %s", tableName))) { @@ -315,7 +316,7 @@ private static List getIndexes(Connection con, String tableName) thr return indexes; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SHOW COLUMNS FROM " + tableName)) { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java index 953414eed..3a052edaf 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java @@ -10,7 +10,7 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.TableIndex; public final class CockroachDBCommentOnGenerator { @@ -56,7 +56,7 @@ public static Query comment(CockroachDBGlobalState globalState) { sb.append("'"); ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java index cb670f420..b98639651 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java @@ -5,7 +5,7 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBCreateStatisticsGenerator { @@ -23,7 +23,7 @@ public static Query create(CockroachDBGlobalState globalState) { sb.append(" FROM "); sb.append(randomTable.getName()); - return new QueryAdapter(sb.toString(), + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("current transaction is aborted, commands ignored until end of transaction block", "ERROR: unable to encode table key: *tree.DArray" /* * https://github.com/cockroachdb/cockroach/ diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index 69c918c72..7df102338 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -8,7 +8,7 @@ import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBDeleteGenerator { @@ -31,7 +31,7 @@ public static Query delete(CockroachDBGlobalState globalState) { } errors.add("foreign key violation"); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java index 2ecf179df..9a1496152 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java @@ -11,7 +11,7 @@ import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBInsertGenerator { @@ -96,7 +96,7 @@ public static Query insert(CockroachDBGlobalState globalState) { errors.add("there is no unique or exclusion constraint matching the ON CONFLICT specification"); } CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java index 945777148..88d2ee906 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java @@ -15,7 +15,7 @@ import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBRandomQuerySynthesizer { @@ -24,7 +24,7 @@ private CockroachDBRandomQuerySynthesizer() { public static Query generate(CockroachDBGlobalState globalState, int nrColumns) { CockroachDBSelect select = generateSelect(globalState, nrColumns); - return new QueryAdapter(CockroachDBVisitor.asString(select)); + return new SQLQueryAdapter(CockroachDBVisitor.asString(select)); } public static CockroachDBSelect generateSelect(CockroachDBGlobalState globalState, int nrColumns) { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 2712a533d..4690143de 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -7,7 +7,7 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBSetClusterSettingGenerator { @@ -55,7 +55,7 @@ public static Query create(CockroachDBGlobalState globalState) { errors.add("setting updated but timed out waiting to read new value"); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index c5d508725..2b3f549a7 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -7,7 +7,7 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBSetSessionGenerator { @@ -53,7 +53,7 @@ public static Query create(CockroachDBGlobalState globalState) { sb.append(s.f.apply(globalState)); ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java index ddd944711..d299b7e30 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java @@ -5,7 +5,7 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBShowGenerator { @@ -57,7 +57,7 @@ public static Query show(CockroachDBGlobalState globalState) { throw new AssertionError(); } CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index 6433f7225..9230ccd75 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -4,7 +4,7 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBTruncateGenerator { @@ -39,7 +39,7 @@ public static Query truncate(CockroachDBGlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index 86c6777dd..6b18f9700 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -11,7 +11,7 @@ import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBUpdateGenerator { @@ -53,7 +53,7 @@ public static Query gen(CockroachDBGlobalState globalState) { errors.add("cannot write directly to computed column"); CockroachDBErrors.addExpressionErrors(errors); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java index ef3c3dad9..f8b112e6b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java @@ -5,7 +5,7 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBViewGenerator { @@ -32,7 +32,7 @@ public static Query generate(CockroachDBGlobalState globalState) { CockroachDBErrors.addTransactionErrors(errors); errors.add("value type unknown cannot be used for table columns"); errors.add("already exists"); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 966a37ab3..f0f741194 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -6,7 +6,7 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.GlobalState; +import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBCommon; @@ -27,7 +27,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; public class CockroachDBNoRECOracle extends NoRECBase implements TestOracle { @@ -113,7 +113,7 @@ private int getOptimizedResult(CockroachDBExpression whereCondition, List globalState, Query q) throws AssertionError { + private int getCount(SQLGlobalState globalState, Query q) throws AssertionError { int count = 0; try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs == null) { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index 41f9c4ea7..8d1a53719 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -32,7 +32,7 @@ import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; public class CockroachDBTLPAggregateOracle implements TestOracle { @@ -116,7 +116,7 @@ private String createMetamorphicUnionQuery(CockroachDBSelect select, CockroachDB private String getAggregateResult(String queryString) throws SQLException { String resultString; - QueryAdapter q = new QueryAdapter(queryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); diff --git a/src/sqlancer/common/ast/newast/TableReferenceNode.java b/src/sqlancer/common/ast/newast/TableReferenceNode.java index b350a9af2..981da70ad 100644 --- a/src/sqlancer/common/ast/newast/TableReferenceNode.java +++ b/src/sqlancer/common/ast/newast/TableReferenceNode.java @@ -2,7 +2,7 @@ import sqlancer.common.schema.AbstractTable; -public class TableReferenceNode> implements Node { +public class TableReferenceNode> implements Node { private final T t; diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java index c9acef68b..b5b94e8c0 100644 --- a/src/sqlancer/common/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -2,7 +2,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public abstract class AbstractGenerator { @@ -12,7 +12,7 @@ public abstract class AbstractGenerator { public Query getQuery() { buildStatement(); - return new QueryAdapter(sb.toString(), errors, canAffectSchema); + return new SQLQueryAdapter(sb.toString(), errors, canAffectSchema); } public abstract void buildStatement(); diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java index 78b96e04c..ce9170ed7 100644 --- a/src/sqlancer/common/log/SQLLoggableFactory.java +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -4,7 +4,7 @@ import java.io.StringWriter; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public class SQLLoggableFactory extends LoggableFactory { @@ -22,14 +22,14 @@ protected Loggable createLoggable(String input, String suffix) { @Override public Query getQueryForStateToReproduce(String queryString) { - return new QueryAdapter(queryString); + return new SQLQueryAdapter(queryString); } @Override public Query commentOutQuery(Query query) { String queryString = query.getLogString(); String newQueryString = "-- " + queryString; - return new QueryAdapter(newQueryString); + return new SQLQueryAdapter(newQueryString); } @Override diff --git a/src/sqlancer/common/oracle/CompositeTestOracle.java b/src/sqlancer/common/oracle/CompositeTestOracle.java index 9eb59e345..478e67702 100644 --- a/src/sqlancer/common/oracle/CompositeTestOracle.java +++ b/src/sqlancer/common/oracle/CompositeTestOracle.java @@ -1,16 +1,16 @@ package sqlancer.common.oracle; -import java.util.List; - import sqlancer.GlobalState; +import java.util.List; + public class CompositeTestOracle implements TestOracle { private final TestOracle[] oracles; - private final GlobalState globalState; + private final GlobalState globalState; private int i; - public CompositeTestOracle(List oracles, GlobalState globalState) { + public CompositeTestOracle(List oracles, GlobalState globalState) { this.globalState = globalState; this.oracles = oracles.toArray(new TestOracle[oracles.size()]); } diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java index ad15eeed7..b6c83f524 100644 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ b/src/sqlancer/common/oracle/NoRECBase.java @@ -1,19 +1,19 @@ package sqlancer.common.oracle; -import java.sql.Connection; - import sqlancer.GlobalState; import sqlancer.Main.StateLogger; import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.common.query.ExpectedErrors; -public abstract class NoRECBase> implements TestOracle { +public abstract class NoRECBase> implements TestOracle { protected final S state; protected final ExpectedErrors errors = new ExpectedErrors(); protected final StateLogger logger; protected final MainOptions options; - protected final Connection con; + protected final SQLConnection con; protected String optimizedQueryString; protected String unoptimizedQueryString; diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index aa7cf5059..d541872ae 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -3,14 +3,14 @@ import java.util.ArrayList; import java.util.List; -import sqlancer.GlobalState; +import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractRowValue; -public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E> +public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E> implements TestOracle { protected final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java index 6b5025806..991824628 100644 --- a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java @@ -1,6 +1,6 @@ package sqlancer.common.oracle; -import sqlancer.GlobalState; +import sqlancer.SQLGlobalState; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.query.ExpectedErrors; @@ -14,7 +14,7 @@ * @param * the global state type */ -public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { +public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { protected E predicate; protected E negatedPredicate; diff --git a/src/sqlancer/common/query/Query.java b/src/sqlancer/common/query/Query.java index 3f75a7986..ec2a3b67b 100644 --- a/src/sqlancer/common/query/Query.java +++ b/src/sqlancer/common/query/Query.java @@ -1,9 +1,10 @@ package sqlancer.common.query; import sqlancer.GlobalState; +import sqlancer.SQLancerDBConnection; import sqlancer.common.log.Loggable; -public abstract class Query implements Loggable { +public abstract class Query, C extends SQLancerDBConnection> implements Loggable { /** * Gets the query string, which is guaranteed to be terminated with a semicolon. @@ -26,7 +27,7 @@ public abstract class Query implements Loggable { */ public abstract boolean couldAffectSchema(); - public abstract boolean execute(GlobalState globalState, String... fills) throws Exception; + public abstract boolean execute(G globalState, String... fills) throws Exception; public abstract ExpectedErrors getExpectedErrors(); @@ -35,21 +36,21 @@ public String toString() { return getQueryString(); } - public SQLancerResultSet executeAndGet(GlobalState globalState, String... fills) throws Exception { + public SQLancerResultSet executeAndGet(G globalState, String... fills) throws Exception { throw new AssertionError(); } - public boolean executeLogged(GlobalState globalState) throws Exception { + public boolean executeLogged(G globalState) throws Exception { logQueryString(globalState); return execute(globalState); } - public SQLancerResultSet executeAndGetLogged(GlobalState globalState) throws Exception { + public SQLancerResultSet executeAndGetLogged(G globalState) throws Exception { logQueryString(globalState); return executeAndGet(globalState); } - private void logQueryString(GlobalState globalState) { + private void logQueryString(G globalState) { if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(getQueryString()); } diff --git a/src/sqlancer/common/query/QueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java similarity index 86% rename from src/sqlancer/common/query/QueryAdapter.java rename to src/sqlancer/common/query/SQLQueryAdapter.java index b0616a518..c80a9eb69 100644 --- a/src/sqlancer/common/query/QueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -5,28 +5,29 @@ import java.sql.SQLException; import java.sql.Statement; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.Main; -public class QueryAdapter extends Query { +public class SQLQueryAdapter> extends Query { private final String query; private final ExpectedErrors expectedErrors; private final boolean couldAffectSchema; - public QueryAdapter(String query) { + public SQLQueryAdapter(String query) { this(query, new ExpectedErrors()); } - public QueryAdapter(String query, boolean couldAffectSchema) { + public SQLQueryAdapter(String query, boolean couldAffectSchema) { this(query, new ExpectedErrors(), couldAffectSchema); } - public QueryAdapter(String query, ExpectedErrors expectedErrors) { + public SQLQueryAdapter(String query, ExpectedErrors expectedErrors) { this(query, expectedErrors, false); } - public QueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema) { + public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema) { this.query = canonicalizeString(query); this.expectedErrors = expectedErrors; this.couldAffectSchema = couldAffectSchema; @@ -68,7 +69,7 @@ public String getUnterminatedQueryString() { } @Override - public boolean execute(GlobalState globalState, String... fills) throws SQLException { + public boolean execute(G globalState, String... fills) throws SQLException { Statement s; if (fills.length > 0) { s = globalState.getConnection().prepareStatement(fills[0]); @@ -100,7 +101,7 @@ public void checkException(Exception e) throws AssertionError { } @Override - public SQLancerResultSet executeAndGet(GlobalState globalState, String... fills) throws SQLException { + public SQLancerResultSet executeAndGet(G globalState, String... fills) throws SQLException { Statement s; if (fills.length > 0) { s = globalState.getConnection().prepareStatement(fills[0]); diff --git a/src/sqlancer/common/query/QueryResultCheckAdapter.java b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java similarity index 64% rename from src/sqlancer/common/query/QueryResultCheckAdapter.java rename to src/sqlancer/common/query/SQLQueryResultCheckAdapter.java index a1626fda6..b2692fecf 100644 --- a/src/sqlancer/common/query/QueryResultCheckAdapter.java +++ b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java @@ -5,19 +5,20 @@ import java.sql.Statement; import java.util.function.Consumer; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; -public class QueryResultCheckAdapter extends QueryAdapter { +public class SQLQueryResultCheckAdapter> extends SQLQueryAdapter { private final Consumer rsChecker; - public QueryResultCheckAdapter(String query, Consumer rsChecker) { + public SQLQueryResultCheckAdapter(String query, Consumer rsChecker) { super(query); this.rsChecker = rsChecker; } @Override - public boolean execute(GlobalState globalState, String... fills) throws SQLException { + public boolean execute(G globalState, String... fills) throws SQLException { try (Statement s = globalState.getConnection().createStatement()) { ResultSet rs = s.executeQuery(getQueryString()); rsChecker.accept(rs); diff --git a/src/sqlancer/common/schema/AbstractRelationalTable.java b/src/sqlancer/common/schema/AbstractRelationalTable.java new file mode 100644 index 000000000..cd4d2287f --- /dev/null +++ b/src/sqlancer/common/schema/AbstractRelationalTable.java @@ -0,0 +1,35 @@ +package sqlancer.common.schema; + +import sqlancer.IgnoreMeException; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; + +import java.util.List; + +public class AbstractRelationalTable, I extends TableIndex, G extends SQLGlobalState> + extends AbstractTable { + + public AbstractRelationalTable(String name, List columns, List indexes, boolean isView) { + super(name, columns, indexes, isView); + } + + public long getNrRows(G globalState) { + if (rowCount == NO_ROW_COUNT_AVAILABLE) { + SQLQueryAdapter q = new SQLQueryAdapter("SELECT COUNT(*) FROM " + name); + try (SQLancerResultSet query = q.executeAndGet(globalState)) { + if (query == null) { + throw new IgnoreMeException(); + } + query.next(); + rowCount = query.getLong(1); + return rowCount; + } catch (Throwable t) { + // an exception might be expected, for example, when invalid view is created + throw new IgnoreMeException(); + } + } else { + return rowCount; + } + } +} diff --git a/src/sqlancer/common/schema/AbstractRowValue.java b/src/sqlancer/common/schema/AbstractRowValue.java index 7714b860b..c7f933c9b 100644 --- a/src/sqlancer/common/schema/AbstractRowValue.java +++ b/src/sqlancer/common/schema/AbstractRowValue.java @@ -56,13 +56,13 @@ public String getRowValuesAsString(List columnsToCheck) { public String asStringGroupedByTables() { StringBuilder sb = new StringBuilder(); List columnList = getValues().keySet().stream().collect(Collectors.toList()); - List> tableList = columnList.stream().map(c -> c.getTable()).distinct().sorted() + List> tableList = columnList.stream().map(c -> c.getTable()).distinct().sorted() .collect(Collectors.toList()); for (int j = 0; j < tableList.size(); j++) { if (j != 0) { sb.append("\n"); } - AbstractTable t = tableList.get(j); + AbstractTable t = tableList.get(j); sb.append("-- " + t.getName() + "\n"); List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) .collect(Collectors.toList()); diff --git a/src/sqlancer/common/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java index e93a1106b..84a854f72 100644 --- a/src/sqlancer/common/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -10,7 +10,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -public class AbstractSchema> { +public class AbstractSchema, A extends AbstractTable> { private final List databaseTables; @@ -139,7 +139,7 @@ public String getFreeViewName() { } while (true); } - public boolean containsTableWithZeroRows(GlobalState globalState) { + public boolean containsTableWithZeroRows(G globalState) { return databaseTables.stream().anyMatch(t -> t.getNrRows(globalState) == 0); } diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 007345121..4755164df 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -1,24 +1,21 @@ package sqlancer.common.schema; +import sqlancer.GlobalState; +import sqlancer.Randomly; + import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import sqlancer.GlobalState; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.query.QueryAdapter; -import sqlancer.common.query.SQLancerResultSet; - -public class AbstractTable, I extends TableIndex> - implements Comparable> { +public abstract class AbstractTable, I extends TableIndex, G extends GlobalState> + implements Comparable>{ - private static final int NO_ROW_COUNT_AVAILABLE = -1; - private final String name; + protected static final int NO_ROW_COUNT_AVAILABLE = -1; + protected final String name; private final List columns; private final List indexes; private final boolean isView; - private long rowCount = NO_ROW_COUNT_AVAILABLE; + protected long rowCount = NO_ROW_COUNT_AVAILABLE; public AbstractTable(String name, List columns, List indexes, boolean isView) { this.name = name; @@ -32,7 +29,7 @@ public String getName() { } @Override - public int compareTo(AbstractTable o) { + public int compareTo(AbstractTable o) { return o.getName().compareTo(getName()); } @@ -97,26 +94,9 @@ public String getFreeColumnName() { } - public long getNrRows(GlobalState globalState) { - if (rowCount == NO_ROW_COUNT_AVAILABLE) { - QueryAdapter q = new QueryAdapter("SELECT COUNT(*) FROM " + name); - try (SQLancerResultSet query = q.executeAndGet(globalState)) { - if (query == null) { - throw new IgnoreMeException(); - } - query.next(); - rowCount = query.getLong(1); - return rowCount; - } catch (Throwable t) { - // an exception might be expected, for example, when invalid view is created - throw new IgnoreMeException(); - } - } else { - return rowCount; - } - } - public void recomputeCount() { rowCount = NO_ROW_COUNT_AVAILABLE; } + + public abstract long getNrRows(G globalState); } diff --git a/src/sqlancer/common/schema/AbstractTableColumn.java b/src/sqlancer/common/schema/AbstractTableColumn.java index 17ddace8f..e519bea97 100644 --- a/src/sqlancer/common/schema/AbstractTableColumn.java +++ b/src/sqlancer/common/schema/AbstractTableColumn.java @@ -1,6 +1,6 @@ package sqlancer.common.schema; -public class AbstractTableColumn, U> implements Comparable> { +public class AbstractTableColumn, U> implements Comparable> { private final String name; private final U type; diff --git a/src/sqlancer/common/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java index ad80011b0..2afff82ab 100644 --- a/src/sqlancer/common/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -5,7 +5,7 @@ import java.util.function.Function; import java.util.stream.Collectors; -public class AbstractTables, C extends AbstractTableColumn> { +public class AbstractTables, C extends AbstractTableColumn> { private final List tables; private final List columns; diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index b64d1063f..a20f41d50 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -5,14 +5,15 @@ import java.sql.SQLException; import sqlancer.AbstractAction; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.gen.DuckDBDeleteGenerator; @@ -33,8 +34,8 @@ public enum Action implements AbstractAction { INSERT(DuckDBInsertGenerator::getQuery), // CREATE_INDEX(DuckDBIndexGenerator::getQuery), // - VACUUM((g) -> new QueryAdapter("VACUUM;")), // - ANALYZE((g) -> new QueryAdapter("ANALYZE;")), // + VACUUM((g) -> new SQLQueryAdapter("VACUUM;")), // + ANALYZE((g) -> new SQLQueryAdapter("ANALYZE;")), // DELETE(DuckDBDeleteGenerator::generate), // UPDATE(DuckDBUpdateGenerator::getQuery), // CREATE_VIEW(DuckDBViewGenerator::generate), // @@ -42,7 +43,7 @@ public enum Action implements AbstractAction { ExpectedErrors errors = new ExpectedErrors(); DuckDBErrors.addExpressionErrors(errors); DuckDBErrors.addGroupByErrors(errors); - return new QueryAdapter( + return new SQLQueryAdapter( "EXPLAIN " + DuckDBToStringVisitor .asString(DuckDBRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), errors); @@ -85,7 +86,7 @@ private static int mapActions(DuckDBGlobalState globalState, Action a) { } } - public static class DuckDBGlobalState extends GlobalState { + public static class DuckDBGlobalState extends SQLGlobalState { @Override protected DuckDBSchema readSchema() throws SQLException { @@ -116,10 +117,10 @@ public void generateDatabase(DuckDBGlobalState globalState) throws Exception { } @Override - public Connection createDatabase(DuckDBGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLException { String url = "jdbc:duckdb:"; - return DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); + return new SQLConnection(DriverManager.getConnection(url, globalState.getOptions().getUserName(), + globalState.getOptions().getPassword())); } @Override diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index b0c153625..7f8a51fb0 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -1,6 +1,5 @@ package sqlancer.duckdb; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -9,14 +8,16 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; +import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -public class DuckDBSchema extends AbstractSchema { +public class DuckDBSchema extends AbstractSchema { public enum DuckDBDataType { @@ -199,7 +200,7 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { return new DuckDBCompositeDataType(primitiveType, size); } - public static class DuckDBTable extends AbstractTable { + public static class DuckDBTable extends AbstractRelationalTable { public DuckDBTable(String tableName, List columns, boolean isView) { super(tableName, columns, Collections.emptyList(), isView); @@ -207,7 +208,7 @@ public DuckDBTable(String tableName, List columns, boolean isView) } - public static DuckDBSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static DuckDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { @@ -223,7 +224,7 @@ public static DuckDBSchema fromConnection(Connection con, String databaseName) t return new DuckDBSchema(databaseTables); } - private static List getTableNames(Connection con) throws SQLException { + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master()")) { @@ -235,7 +236,7 @@ private static List getTableNames(Connection con) throws SQLException { return tableNames; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format("SELECT * FROM pragma_table_info('%s');", tableName))) { diff --git a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java index fb59ff178..e1c1088f6 100644 --- a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; @@ -65,7 +65,7 @@ public static Query getQuery(DuckDBGlobalState globalState) { default: throw new AssertionError(action); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index ffe9af5a2..da7c16c4f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; @@ -25,7 +25,7 @@ public static Query generate(DuckDBGlobalState globalState) { new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } DuckDBErrors.addExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 51bba3e64..f9820797c 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -6,7 +6,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; @@ -54,7 +54,7 @@ public static Query getQuery(DuckDBGlobalState globalState) { if (globalState.getDmbsSpecificOptions().testRowid) { errors.add("Cannot create an index on the rowid!"); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index f8c287522..2193109c0 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -7,7 +7,7 @@ import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; @@ -38,7 +38,7 @@ private Query generate() { sb.append(" VALUES "); insertColumns(columns); DuckDBErrors.addInsertErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index 2ed1c22c0..3e6f12a61 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -9,7 +9,7 @@ import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; @@ -70,7 +70,7 @@ public Query getQuery(DuckDBGlobalState globalState) { sb.append(")"); } sb.append(")"); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } public static String getRandomCollate() { diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 7e3fb6282..2969bd21d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -6,7 +6,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; @@ -43,7 +43,7 @@ public static Query getQuery(DuckDBGlobalState globalState) { sb.append(DuckDBToStringVisitor.asString(expr)); } DuckDBErrors.addInsertErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java index 1cf5ba023..495e000a3 100644 --- a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBToStringVisitor; @@ -31,7 +31,7 @@ public static Query generate(DuckDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); DuckDBErrors.addExpressionErrors(errors); DuckDBErrors.addGroupByErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 494eb2afb..a701a9fc3 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -10,6 +10,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewPostfixTextNode; import sqlancer.common.ast.newast.Node; @@ -17,7 +18,7 @@ import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -83,7 +84,7 @@ private int getSecondQuery(List> tableList, Node> tableList, Node> tableList, List columns, - Node randomWhereCondition, List> joins) throws SQLException { + private int getFirstQueryCount(SQLConnection con, List> tableList, List columns, + Node randomWhereCondition, List> joins) throws SQLException { DuckDBSelect select = new DuckDBSelect(); // select.setGroupByClause(groupBys); // DuckDBAggregate aggr = new DuckDBAggregate( diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index d8ed7a928..21eedf0be 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -15,7 +15,7 @@ import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -99,7 +99,7 @@ private String createMetamorphicUnionQuery(DuckDBSelect select, private String getAggregateResult(String queryString) throws SQLException { String resultString; - QueryAdapter q = new QueryAdapter(queryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java index 973399eb2..49aff01d6 100644 --- a/src/sqlancer/h2/H2DeleteGenerator.java +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; @@ -28,7 +28,7 @@ public static Query getQuery(H2GlobalState globalState) { } H2Errors.addExpressionErrors(errors); H2Errors.addDeleteErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/h2/H2IndexGenerator.java b/src/sqlancer/h2/H2IndexGenerator.java index 0f68a7aff..18d23fb6d 100644 --- a/src/sqlancer/h2/H2IndexGenerator.java +++ b/src/sqlancer/h2/H2IndexGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; @@ -51,7 +51,7 @@ private Query generate() { } } sb.append(')'); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index cc36d3ac4..cc59b9af6 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -7,7 +7,7 @@ import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2Table; @@ -53,7 +53,7 @@ private Query generate() { insertColumns(columns); H2Errors.addInsertErrors(errors); H2Errors.addExpressionErrors(errors); // generated columns - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 5171c6304..ce36eca2c 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -5,13 +5,14 @@ import java.sql.SQLException; import sqlancer.AbstractAction; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.h2.H2Provider.H2GlobalState; @@ -25,7 +26,7 @@ public enum Action implements AbstractAction { INSERT(H2InsertGenerator::getQuery), // INDEX(H2IndexGenerator::getQuery), // - ANALYZE((g) -> new QueryAdapter("ANALYZE")), // + ANALYZE((g) -> new SQLQueryAdapter("ANALYZE")), // CREATE_VIEW(H2ViewGenerator::getQuery), // UPDATE(H2UpdateGenerator::getQuery), // DELETE(H2DeleteGenerator::getQuery), // @@ -63,7 +64,7 @@ private static int mapActions(H2GlobalState globalState, Action a) { } } - public static class H2GlobalState extends GlobalState { + public static class H2GlobalState extends SQLGlobalState { @Override protected H2Schema readSchema() throws SQLException { @@ -94,13 +95,13 @@ public void generateDatabase(H2GlobalState globalState) throws Exception { } @Override - public Connection createDatabase(H2GlobalState globalState) throws SQLException { + public SQLConnection createDatabase(H2GlobalState globalState) throws SQLException { String connectionString = "jdbc:h2:~/" + globalState.getDatabaseName() + ";DB_CLOSE_ON_EXIT=FALSE"; Connection connection = DriverManager.getConnection(connectionString, "sa", ""); connection.createStatement().execute("DROP ALL OBJECTS DELETE FILES"); connection.close(); connection = DriverManager.getConnection(connectionString, "sa", ""); - return connection; + return new SQLConnection(connection); } @Override diff --git a/src/sqlancer/h2/H2Schema.java b/src/sqlancer/h2/H2Schema.java index 3f88abcba..506a67864 100644 --- a/src/sqlancer/h2/H2Schema.java +++ b/src/sqlancer/h2/H2Schema.java @@ -1,6 +1,5 @@ package sqlancer.h2; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -9,14 +8,16 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; +import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; -public class H2Schema extends AbstractSchema { +public class H2Schema extends AbstractSchema { public enum H2DataType { @@ -152,7 +153,7 @@ public H2Tables getRandomTableNonEmptyTables() { return new H2Tables(Randomly.nonEmptySubset(getDatabaseTables())); } - public static class H2Table extends AbstractTable { + public static class H2Table extends AbstractRelationalTable { public H2Table(String tableName, List columns) { super(tableName, columns, Collections.emptyList(), tableName.startsWith("V")); @@ -160,7 +161,7 @@ public H2Table(String tableName, List columns) { } - public static H2Schema fromConnection(Connection con, String databaseName) throws SQLException { + public static H2Schema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { @@ -175,7 +176,7 @@ public static H2Schema fromConnection(Connection con, String databaseName) throw return new H2Schema(databaseTables); } - private static List getTableNames(Connection con) throws SQLException { + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SHOW TABLES")) { @@ -187,7 +188,7 @@ private static List getTableNames(Connection con) throws SQLException { return tableNames; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format("SHOW COLUMNS FROM %s;", tableName))) { diff --git a/src/sqlancer/h2/H2SetGenerator.java b/src/sqlancer/h2/H2SetGenerator.java index 34065d544..af14f016a 100644 --- a/src/sqlancer/h2/H2SetGenerator.java +++ b/src/sqlancer/h2/H2SetGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; public final class H2SetGenerator { @@ -58,7 +58,7 @@ public static Query getQuery(H2GlobalState globalState) { sb.append(o.prod.apply(globalState.getRandomly())); ExpectedErrors errors = new ExpectedErrors(); errors.add("Collation cannot be changed because there is a data table"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java index 3ba631ca6..d28bdad66 100644 --- a/src/sqlancer/h2/H2TableGenerator.java +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2CompositeDataType; @@ -103,7 +103,7 @@ public Query getQuery(H2GlobalState globalState) { errors.add(" cannot be updatable by a referential constraint with"); // generated columns } sb.append(")"); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } private void addReferentialAction(StringBuilder sb) { diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index 111704eb0..965483e28 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2Table; @@ -38,7 +38,7 @@ public static Query getQuery(H2GlobalState globalState) { sb.append(H2ToStringVisitor.asString(gen.generateExpression())); } H2Errors.addExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/h2/H2ViewGenerator.java b/src/sqlancer/h2/H2ViewGenerator.java index 0ab9fb0f1..2c56f5549 100644 --- a/src/sqlancer/h2/H2ViewGenerator.java +++ b/src/sqlancer/h2/H2ViewGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; public final class H2ViewGenerator { @@ -28,7 +28,7 @@ public static Query getQuery(H2GlobalState globalState) { sb.append(H2ToStringVisitor.asString(H2RandomQuerySynthesizer.generateSelect(globalState, nrColumns))); ExpectedErrors errors = new ExpectedErrors(); H2Errors.addExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 96f03a634..bc70f266a 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -7,7 +7,8 @@ import java.util.ArrayList; import java.util.List; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -152,7 +153,7 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { } } - public static class MariaDBGlobalState extends GlobalState { + public static class MariaDBGlobalState extends SQLGlobalState { @Override protected MariaDBSchema readSchema() throws SQLException { @@ -162,7 +163,7 @@ protected MariaDBSchema readSchema() throws SQLException { } @Override - public Connection createDatabase(MariaDBGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(MariaDBGlobalState globalState) throws SQLException { globalState.getState().logStatement("DROP DATABASE IF EXISTS " + globalState.getDatabaseName()); globalState.getState().logStatement("CREATE DATABASE " + globalState.getDatabaseName()); globalState.getState().logStatement("USE " + globalState.getDatabaseName()); @@ -179,7 +180,7 @@ public Connection createDatabase(MariaDBGlobalState globalState) throws SQLExcep try (Statement s = con.createStatement()) { s.execute("USE " + globalState.getDatabaseName()); } - return con; + return new SQLConnection(con); } @Override diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index dfefdcd6f..2acf3d79e 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -1,6 +1,5 @@ package sqlancer.mariadb; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; @@ -12,14 +11,17 @@ import java.util.stream.Stream; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.TableIndex; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.MariaDBSchema.MariaDBTable.MariaDBEngine; -public class MariaDBSchema extends AbstractSchema { +public class MariaDBSchema extends AbstractSchema { private static final int NR_SCHEMA_READ_TRIES = 10; @@ -113,7 +115,7 @@ private static MariaDBDataType getColumnType(String typeString) { } } - public static class MariaDBTable extends AbstractTable { + public static class MariaDBTable extends AbstractRelationalTable { public enum MariaDBEngine { @@ -170,7 +172,7 @@ public String getIndexName() { } - public static MariaDBSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static MariaDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { Exception ex = null; /* the loop is a workaround for https://bugs.MariaDB.com/bug.php?id=95929 */ for (int i = 0; i < NR_SCHEMA_READ_TRIES; i++) { @@ -202,7 +204,7 @@ public static MariaDBSchema fromConnection(Connection con, String databaseName) throw new AssertionError(ex); } - private static List getIndexes(Connection con, String tableName, String databaseName) + private static List getIndexes(SQLConnection con, String tableName, String databaseName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -218,7 +220,7 @@ private static List getIndexes(Connection con, String tableName, S return indexes; } - private static List getTableColumns(Connection con, String tableName, String databaseName) + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { diff --git a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java index c15c21b06..a39ec3270 100644 --- a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java @@ -6,6 +6,7 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.StateToReproduce; import sqlancer.mariadb.MariaDBProvider; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; @@ -66,7 +67,7 @@ public MariaDBExpressionGenerator setColumns(List columns) { return this; } - public MariaDBExpressionGenerator setCon(Connection con) { + public MariaDBExpressionGenerator setCon(SQLConnection con) { return this; } diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 1dc100c37..4dc7ac4d1 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; @@ -55,7 +55,7 @@ public static Query generate(MariaDBSchema s) { // errors.add("is not supported for this operation"); // } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java index b45b43afc..c6346c569 100644 --- a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; @@ -34,7 +34,7 @@ public static Query insert(MariaDBSchema s, Randomly r) { sb.append(")"); ExpectedErrors errors = new ExpectedErrors(); MariaDBErrors.addInsertErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index f950eb474..4ecc3e02d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -9,7 +9,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public class MariaDBSetGenerator { @@ -191,7 +191,7 @@ private Query get() { sb.append(a.name); sb.append(" = "); sb.append(a.prod.apply(r)); - return new QueryAdapter(sb.toString(), ExpectedErrors + return new SQLQueryAdapter(sb.toString(), ExpectedErrors .from("At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'")); } diff --git a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java index 934fb1c7a..5a2f4f5c4 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java @@ -7,8 +7,8 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; -import sqlancer.common.query.QueryResultCheckAdapter; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryResultCheckAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; @@ -23,7 +23,7 @@ public static Query checksumTable(MariaDBSchema newSchema) { sb.append(" "); sb.append(Randomly.fromOptions("QUICK", "EXTENDED")); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } public static Query repairTable(MariaDBSchema newSchema) { @@ -62,7 +62,7 @@ public static Query optimizeTable(MariaDBSchema newSchema) { } private static Query checkForMsgText(StringBuilder sb, Function checker) { - return new QueryResultCheckAdapter(sb.toString(), rs -> { + return new SQLQueryResultCheckAdapter<>(sb.toString(), rs -> { try { while (rs.next()) { String s = rs.getString("Msg_text"); diff --git a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java index e4cecfa28..128eb61c9 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java @@ -7,7 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBBugs; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; @@ -41,7 +41,7 @@ private Query gen() { } else { likeOtherTable(); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } private enum PrimaryKeyState { diff --git a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java index a29178a85..4913b48b3 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java @@ -1,7 +1,7 @@ package sqlancer.mariadb.gen; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBSchema; public final class MariaDBTruncateGenerator { @@ -14,7 +14,7 @@ public static Query truncate(MariaDBSchema s) { sb.append(s.getRandomTable().getName()); sb.append(" "); MariaDBCommon.addWaitClause(sb); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java index 171878b18..db01383b5 100644 --- a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; @@ -40,7 +40,7 @@ public static Query update(MariaDBSchema s, Randomly r) { } ExpectedErrors errors = new ExpectedErrors(); MariaDBErrors.addInsertErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index 3fd8d997a..836983134 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -8,7 +8,7 @@ import sqlancer.IgnoreMeException; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema; @@ -81,7 +81,7 @@ private int getUnoptimizedQuery(MariaDBTable randomTable, MariaDBExpression rand int secondCount = 0; unoptimizedQueryString = "SELECT SUM(count) FROM (" + MariaDBVisitor.asString(select) + ") as asdf"; - QueryAdapter q = new QueryAdapter(unoptimizedQueryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { return NOT_FOUND; @@ -108,7 +108,7 @@ private int getOptimizedQuery(MariaDBTable randomTable, MariaDBExpression random select.setSelectType(MariaDBSelectType.ALL); int firstCount = 0; optimizedQueryString = MariaDBVisitor.asString(select); - QueryAdapter q = new QueryAdapter(optimizedQueryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { firstCount = NOT_FOUND; diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 9b79bf3b4..0bbd1f915 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -3,10 +3,11 @@ import java.sql.SQLException; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; -public class MySQLGlobalState extends GlobalState { +public class MySQLGlobalState extends SQLGlobalState { @Override protected MySQLSchema readSchema() throws SQLException { diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 0913a0dd7..a1d86a55f 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -8,10 +8,11 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.mysql.gen.MySQLAlterTable; import sqlancer.mysql.gen.MySQLDeleteGenerator; @@ -37,7 +38,7 @@ public MySQLProvider() { } enum Action implements AbstractAction { - SHOW_TABLES((g) -> new QueryAdapter("SHOW TABLES")), // + SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), // INSERT(MySQLInsertGenerator::insertRow), // SET_VARIABLE(MySQLSetGenerator::set), // REPAIR(MySQLRepair::repair), // @@ -48,7 +49,7 @@ enum Action implements AbstractAction { FLUSH(MySQLFlush::create), RESET(MySQLReset::create), CREATE_INDEX(MySQLIndexGenerator::create), // ALTER_TABLE(MySQLAlterTable::create), // TRUNCATE_TABLE(MySQLTruncateTableGenerator::generate), // - SELECT_INFO((g) -> new QueryAdapter( + SELECT_INFO((g) -> new SQLQueryAdapter( "select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + g.getDatabaseName() + "'")), // CREATE_TABLE((g) -> { @@ -148,7 +149,7 @@ public void generateDatabase(MySQLGlobalState globalState) throws Exception { } @Override - public Connection createDatabase(MySQLGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(MySQLGlobalState globalState) throws SQLException { String databaseName = globalState.getDatabaseName(); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); globalState.getState().logStatement("CREATE DATABASE " + databaseName); @@ -165,7 +166,7 @@ public Connection createDatabase(MySQLGlobalState globalState) throws SQLExcepti try (Statement s = con.createStatement()) { s.execute("USE " + databaseName); } - return con; + return new SQLConnection(con); } @Override diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 112cc72b0..aef4c61c9 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -1,6 +1,5 @@ package sqlancer.mysql; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; @@ -12,9 +11,10 @@ import java.util.stream.Stream; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; @@ -22,7 +22,7 @@ import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; import sqlancer.mysql.ast.MySQLConstant; -public class MySQLSchema extends AbstractSchema { +public class MySQLSchema extends AbstractSchema { private static final int NR_SCHEMA_READ_TRIES = 10; @@ -88,7 +88,7 @@ public MySQLTables(List tables) { super(tables); } - public MySQLRowValue getRandomRowValue(Connection con) throws SQLException { + public MySQLRowValue getRandomRowValue(SQLConnection con) throws SQLException { String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", columnNamesAsString( c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), // columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + @@ -165,7 +165,7 @@ public static class MySQLRowValue extends AbstractRowValue { + public static class MySQLTable extends AbstractRelationalTable { public enum MySQLEngine { INNO_DB("InnoDB"), MY_ISAM("MyISAM"), MEMORY("MEMORY"), HEAP("HEAP"), CSV("CSV"), MERGE("MERGE"), @@ -221,7 +221,7 @@ public String getIndexName() { } - public static MySQLSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static MySQLSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { Exception ex = null; /* the loop is a workaround for https://bugs.mysql.com/bug.php?id=95929 */ for (int i = 0; i < NR_SCHEMA_READ_TRIES; i++) { @@ -253,7 +253,7 @@ public static MySQLSchema fromConnection(Connection con, String databaseName) th throw new AssertionError(ex); } - private static List getIndexes(Connection con, String tableName, String databaseName) + private static List getIndexes(SQLConnection con, String tableName, String databaseName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -269,7 +269,7 @@ private static List getIndexes(Connection con, String tableName, Str return indexes; } - private static List getTableColumns(Connection con, String tableName, String databaseName) + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { diff --git a/src/sqlancer/mysql/gen/MySQLAlterTable.java b/src/sqlancer/mysql/gen/MySQLAlterTable.java index ae322c4ba..5425d26b9 100644 --- a/src/sqlancer/mysql/gen/MySQLAlterTable.java +++ b/src/sqlancer/mysql/gen/MySQLAlterTable.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -174,7 +174,7 @@ private Query create() { errors.add(error); } } - return new QueryAdapter(sb.toString(), errors, couldAffectSchema); + return new SQLQueryAdapter(sb.toString(), errors, couldAffectSchema); } } diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index c2506aa08..49fbe7b87 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -52,7 +52,7 @@ private Query generate() { */, "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index")); // TODO: support ORDER BY - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mysql/gen/MySQLDropIndex.java b/src/sqlancer/mysql/gen/MySQLDropIndex.java index 800508b4e..ec1ee7cb5 100644 --- a/src/sqlancer/mysql/gen/MySQLDropIndex.java +++ b/src/sqlancer/mysql/gen/MySQLDropIndex.java @@ -4,7 +4,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -43,7 +43,7 @@ public static Query generate(MySQLGlobalState globalState) { sb.append(" LOCK="); sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); } - return new QueryAdapter(sb.toString(), + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("LOCK=NONE is not supported", "ALGORITHM=INPLACE is not supported", "Data truncation", "Data truncated for functional index", "A primary key index cannot be invisible")); diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 788baa5ca..c3112c9ff 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -7,7 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -92,7 +92,7 @@ private Query generateInto() { errors.add("Data truncated for column"); errors.add("cannot be null"); errors.add("Incorrect decimal value"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 1d3b5623a..f14a361d4 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -7,7 +7,7 @@ import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; public class MySQLSetGenerator { @@ -188,7 +188,7 @@ private Query get() { sb.append(a.name); sb.append(" = "); sb.append(a.prod.apply(r)); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index eddf39b40..5b0f85a39 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -10,7 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -59,7 +59,7 @@ private Query create() { if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { sb.append(" LIKE "); sb.append(schema.getRandomTable().getName()); - return new QueryAdapter(sb.toString(), true); + return new SQLQueryAdapter(sb.toString(), true); } else { sb.append("("); for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { @@ -81,10 +81,10 @@ private Query create() { errors.add("Too many keys specified; max 1 keys allowed"); errors.add("Table handler doesn't support NULL in given index"); addCommonErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } addCommonErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java index b57354779..a06428f97 100644 --- a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java @@ -2,7 +2,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; public final class MySQLTruncateTableGenerator { @@ -13,7 +13,7 @@ private MySQLTruncateTableGenerator() { public static Query generate(MySQLGlobalState globalState) { StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); sb.append(globalState.getSchema().getRandomTable().getName()); - return new QueryAdapter(sb.toString(), ExpectedErrors.from("doesn't have this option")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("doesn't have this option")); } } diff --git a/src/sqlancer/mysql/gen/admin/MySQLFlush.java b/src/sqlancer/mysql/gen/admin/MySQLFlush.java index d826fa6f2..ad9be8601 100644 --- a/src/sqlancer/mysql/gen/admin/MySQLFlush.java +++ b/src/sqlancer/mysql/gen/admin/MySQLFlush.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -45,7 +45,7 @@ private Query generate() { // TODO implement READ LOCK and other variants } } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/admin/MySQLReset.java b/src/sqlancer/mysql/gen/admin/MySQLReset.java index 70abef70c..c0fdab986 100644 --- a/src/sqlancer/mysql/gen/admin/MySQLReset.java +++ b/src/sqlancer/mysql/gen/admin/MySQLReset.java @@ -4,7 +4,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; public final class MySQLReset { @@ -16,7 +16,7 @@ public static Query create(MySQLGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("RESET "); sb.append(Randomly.nonEmptySubset("MASTER", "SLAVE").stream().collect(Collectors.joining(", "))); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index 1fa0ba898..6772ce95d 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -115,7 +115,7 @@ public Query create() { errors.add("out of range"); errors.add("Data truncated for functional index"); errors.add("used in key specification without a key length"); - return new QueryAdapter(string, errors, true); + return new SQLQueryAdapter(string, errors, true); } private void algorithmOption() { diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java index 0cdaf120f..3d4cae59a 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -44,7 +44,7 @@ private Query generate() { updateHistogram(); } } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } // ANALYZE [NO_WRITE_TO_BINLOG | LOCAL] diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java index 82a28cfbb..e1e4d0f34 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -41,7 +41,7 @@ private Query generate() { sb.append(" "); List options = Randomly.subset("FOR UPGRADE", "QUICK", "FAST", "MEDIUM", "EXTENDED", "CHANGED"); sb.append(options.stream().collect(Collectors.joining(" "))); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java index 1c2034aa9..c3e96adcd 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -33,7 +33,7 @@ private Query checksum() { sb.append(" "); sb.append(Randomly.fromOptions("QUICK", "EXTENDED")); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java index febdf6968..f3b3895a7 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -35,7 +35,7 @@ private Query optimize() { } sb.append(" TABLE "); sb.append(tables.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java index bc67477c3..a546301af 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; @@ -27,7 +27,7 @@ public static Query repair(MySQLGlobalState globalState) { for (MySQLTable table : tables) { // see https://bugs.mysql.com/bug.php?id=95820 if (table.getEngine() == MySQLEngine.MY_ISAM) { - return new QueryAdapter("SELECT 1"); + return new SQLQueryAdapter("SELECT 1"); } } return new MySQLRepair(tables).repair(); @@ -53,7 +53,7 @@ private Query repair() { if (Randomly.getBoolean()) { sb.append(" USE_FRM"); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 68cb39b2e..0c122d41e 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; @@ -69,7 +69,7 @@ public Query getRectifiedQuery() throws SQLException { .generateOrderBys(); selectStatement.setOrderByExpressions(orderBy); - return new QueryAdapter(MySQLVisitor.asString(selectStatement), errors); + return new SQLQueryAdapter(MySQLVisitor.asString(selectStatement), errors); } private List generateGroupByClause(List columns, MySQLRowValue rw) { @@ -136,7 +136,7 @@ protected Query getContainmentCheckQuery(Query query) throws SQLException { } String resultingQueryString = sb.toString(); - return new QueryAdapter(resultingQueryString, query.getExpectedErrors()); + return new SQLQueryAdapter(resultingQueryString, query.getExpectedErrors()); } @Override diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 7d7e56d49..5b5d60d0e 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -1,6 +1,5 @@ package sqlancer.postgres; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -11,10 +10,11 @@ import java.util.List; import java.util.Map; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.Randomly; -public class PostgresGlobalState extends GlobalState { +public class PostgresGlobalState extends SQLGlobalState { public static final char IMMUTABLE = 'i'; public static final char STABLE = 's'; @@ -28,7 +28,7 @@ public class PostgresGlobalState extends GlobalState allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); @Override - public void setConnection(Connection con) { + public void setConnection(SQLConnection con) { super.setConnection(con); try { this.opClasses = getOpclasses(getConnection()); @@ -39,7 +39,7 @@ public void setConnection(Connection con) { } } - private List getCollnames(Connection con) throws SQLException { + private List getCollnames(SQLConnection con) throws SQLException { List opClasses = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s @@ -52,7 +52,7 @@ private List getCollnames(Connection con) throws SQLException { return opClasses; } - private List getOpclasses(Connection con) throws SQLException { + private List getOpclasses(SQLConnection con) throws SQLException { List opClasses = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("select opcname FROM pg_opclass;")) { @@ -64,7 +64,7 @@ private List getOpclasses(Connection con) throws SQLException { return opClasses; } - private List getOperators(Connection con) throws SQLException { + private List getOperators(SQLConnection con) throws SQLException { List opClasses = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 4014a0030..786988eba 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -11,10 +11,11 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; @@ -75,11 +76,11 @@ public enum Action implements AbstractAction { COMMIT(g -> { Query query; if (Randomly.getBoolean()) { - query = new QueryAdapter("COMMIT", true); + query = new SQLQueryAdapter("COMMIT", true); } else if (Randomly.getBoolean()) { query = PostgresTransactionGenerator.executeBegin(); } else { - query = new QueryAdapter("ROLLBACK", true); + query = new SQLQueryAdapter("ROLLBACK", true); } return query; }), // @@ -99,11 +100,11 @@ public enum Action implements AbstractAction { StringBuilder sb = new StringBuilder(); sb.append("SET CONSTRAINTS ALL "); sb.append(Randomly.fromOptions("DEFERRED", "IMMEDIATE")); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); }), // - RESET_ROLE((g) -> new QueryAdapter("RESET ROLE")), // + RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // COMMENT_ON(PostgresCommentGenerator::generate), // - RESET((g) -> new QueryAdapter("RESET ALL") /* + RESET((g) -> new SQLQueryAdapter("RESET ALL") /* * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also * configuration parameter */), // @@ -195,7 +196,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws Exception { } @Override - public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLException { if (globalState.getDmbsSpecificOptions().getTestOracleFactory().stream() .anyMatch((o) -> o == PostgresOracleFactory.PQS)) { generateOnlyKnown = true; @@ -255,11 +256,11 @@ public Connection createDatabase(PostgresGlobalState globalState) throws SQLExce testURL = preDatabaseName + databaseName + postDatabaseName; globalState.getState().logStatement(String.format("\\c %s;", databaseName)); con = DriverManager.getConnection("jdbc:" + testURL, username, password); - return con; + return new SQLConnection(con); } protected void readFunctions(PostgresGlobalState globalState) throws SQLException { - QueryAdapter query = new QueryAdapter("SELECT proname, provolatile FROM pg_proc;"); + SQLQueryAdapter query = new SQLQueryAdapter("SELECT proname, provolatile FROM pg_proc;"); SQLancerResultSet rs = query.executeAndGet(globalState); while (rs.next()) { String functionName = rs.getString(1); @@ -289,8 +290,8 @@ protected void prepareTables(PostgresGlobalState globalState) throws Exception { } }); se.executeStatements(); - globalState.executeStatement(new QueryAdapter("COMMIT", true)); - globalState.executeStatement(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 5000;\n")); } private String getCreateDatabaseCommand(PostgresGlobalState state) { diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 947e286f9..2a52fb645 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -1,6 +1,5 @@ package sqlancer.postgres; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; @@ -15,9 +14,10 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; @@ -25,7 +25,7 @@ import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; import sqlancer.postgres.ast.PostgresConstant; -public class PostgresSchema extends AbstractSchema { +public class PostgresSchema extends AbstractSchema { private final String databaseName; @@ -65,7 +65,7 @@ public PostgresTables(List tables) { super(tables); } - public PostgresRowValue getRandomRowValue(Connection con) throws SQLException { + public PostgresRowValue getRandomRowValue(SQLConnection con) throws SQLException { String randomRow = String.format("SELECT %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), // columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + @@ -152,7 +152,7 @@ protected PostgresRowValue(PostgresTables tables, Map { + public static class PostgresTable extends AbstractRelationalTable { public enum TableType { STANDARD, TEMPORARY @@ -217,7 +217,7 @@ public String getIndexName() { } - public static PostgresSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static PostgresSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { try { List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -251,7 +251,7 @@ public static PostgresSchema fromConnection(Connection con, String databaseName) } } - protected static List getStatistics(Connection con) throws SQLException { + protected static List getStatistics(SQLConnection con) throws SQLException { List statistics = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SELECT stxname FROM pg_statistic_ext ORDER BY stxname;")) { @@ -275,7 +275,7 @@ protected static PostgresTable.TableType getTableType(String tableTypeStr) throw return tableType; } - protected static List getIndexes(Connection con, String tableName) throws SQLException { + protected static List getIndexes(SQLConnection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String @@ -293,7 +293,7 @@ protected static List getIndexes(Connection con, String tableName return indexes; } - protected static List getTableColumns(Connection con, String tableName) throws SQLException { + protected static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 3dfd8c707..3859a7f5c 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -368,7 +368,7 @@ public Query generate() { } } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } private static void alterColumn(PostgresTable randomTable, StringBuilder sb) { diff --git a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java index 5baa8695e..46408c960 100644 --- a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -37,7 +37,7 @@ public static Query create(PostgresGlobalState globalState) { } } // FIXME: bug in postgres? - return new QueryAdapter(sb.toString(), ExpectedErrors.from("deadlock")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("deadlock")); } } diff --git a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java index 4adf47906..1b1d6d456 100644 --- a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -27,7 +27,7 @@ public static Query create(PostgresGlobalState globalState) { errors.add("cannot cluster on partial index"); } } - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java index 15a7fbfa0..f21826760 100644 --- a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java @@ -3,7 +3,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -63,7 +63,7 @@ public static Query generate(PostgresGlobalState globalState) { sb.append(globalState.getRandomly().getString().replace("'", "''")); sb.append("'"); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 131972345..132e5ae5e 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -41,7 +41,7 @@ public static Query create(PostgresGlobalState globalState) { errors.add("cannot cast"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java index 75187220d..1793387e0 100644 --- a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; @@ -25,7 +25,7 @@ public static Query create(PostgresGlobalState globalState) { what = Randomly.fromOptions("PLANS", "SEQUENCES"); } sb.append(what); - return new QueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { @Override public boolean couldAffectSchema() { diff --git a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java index 5a9233b60..eaf43dd1b 100644 --- a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresIndex; import sqlancer.sqlite3.gen.SQLite3Common; @@ -43,7 +43,7 @@ public static Query create(PostgresGlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } - return new QueryAdapter(sb.toString(), + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot drop desired object(s) because other objects depend on them", "cannot drop index", "does not exist"), true); diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 11018cab1..ba6ed17cf 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -136,7 +136,7 @@ public static Query generate(PostgresGlobalState globalState) { errors.add("functions in index expression must be marked IMMUTABLE"); errors.add("result of range difference would not be contiguous"); PostgresCommon.addCommonExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } private static String getNewIndexName(PostgresTable randomTable) { diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index fd9328484..8c6ceca46 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -97,7 +97,7 @@ public static Query insert(PostgresGlobalState globalState) { errors.add("division by zero"); errors.add("violates foreign key constraint"); errors.add("data type unknown"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } private static void insertRow(PostgresGlobalState globalState, StringBuilder sb, List columns, diff --git a/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java b/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java index cbb4f8f30..c7ecfad2b 100644 --- a/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresNotifyGenerator { @@ -24,14 +24,14 @@ public static Query createNotify(PostgresGlobalState globalState) { sb.append(globalState.getRandomly().getString().replace("'", "''")); sb.append("'"); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } public static Query createListen() { StringBuilder sb = new StringBuilder(); sb.append("LISTEN "); sb.append(getChannel()); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } public static Query createUnlisten() { @@ -42,7 +42,7 @@ public static Query createUnlisten() { } else { sb.append("*"); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java index d68e4319f..0e44b0884 100644 --- a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java @@ -4,10 +4,11 @@ import java.sql.SQLException; import java.sql.Statement; -import sqlancer.GlobalState; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class PostgresQueryCatalogGenerator { @@ -31,9 +32,9 @@ public static Query query() { "pg_statistic_ext_data", "pg_subscription", "pg_subscription_rel", "pg_tablespace", "pg_transform", "pg_trigger", "pg_ts_config", "pg_ts_config_map", "pg_ts_dict", "pg_ts_parser", "pg_ts_template", "pg_type", "pg_user_mapping")); - return new QueryAdapter(sb.toString()) { + return new SQLQueryAdapter(sb.toString()) { @Override - public boolean execute(GlobalState globalState, String... fills) throws SQLException { + public boolean execute(SQLGlobalState globalState, String... fills) throws SQLException { try (Statement s = globalState.getConnection().createStatement()) { try (ResultSet rs = s.executeQuery(getQueryString())) { // CHECKSTYLE:OFF diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index c55b324ef..34081e140 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -7,7 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresIndex; @@ -62,7 +62,7 @@ public static Query create(PostgresGlobalState globalState) { errors.add("already contains data"); // FIXME bug report errors.add("does not exist"); // internal index errors.add("REINDEX is not yet implemented for partitioned indexes"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java index 61c401c14..e5f5b342f 100644 --- a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresSequenceGenerator { @@ -85,7 +85,7 @@ public static Query createSequence(PostgresGlobalState globalState) { // sb.append(s.getRandomTable().getRandomColumn().getFullQualifiedName()); // } } - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 56bc61bc5..27034ed1a 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresSetGenerator { @@ -145,7 +145,7 @@ public static Query create(PostgresGlobalState globalState) { } else { sb.append(option.op.apply(globalState.getRandomly())); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java index 0ebb764ed..9b7f88453 100644 --- a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java @@ -7,7 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresStatisticsObject; @@ -44,7 +44,7 @@ public static Query insert(PostgresGlobalState globalState) { sb.append(randomColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); sb.append(" FROM "); sb.append(randomTable.getName()); - return new QueryAdapter(sb.toString(), ExpectedErrors.from("cannot have more than 8 columns in statistics"), + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot have more than 8 columns in statistics"), true); } @@ -56,7 +56,7 @@ public static Query remove(PostgresGlobalState globalState) { throw new IgnoreMeException(); } sb.append(Randomly.fromList(statistics).getName()); - return new QueryAdapter(sb.toString(), true); + return new SQLQueryAdapter(sb.toString(), true); } private static String getNewStatisticsName(PostgresTable randomTable) { diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 874893a7e..861e8ac11 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -7,7 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -85,7 +85,7 @@ private Query generate() { } else { createStandard(); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } private void createStandard() throws AssertionError { diff --git a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java index aa582ea73..4fa67a3b1 100644 --- a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; public final class PostgresTransactionGenerator { @@ -22,7 +22,7 @@ public static Query executeBegin() { // sb.append(Randomly.fromOptions("READ WRITE", "READ ONLY")); // } } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java index 374900232..774e87916 100644 --- a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresTruncateGenerator { @@ -34,7 +34,7 @@ public static Query create(PostgresGlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } - return new QueryAdapter(sb.toString(), ExpectedErrors + return new SQLQueryAdapter(sb.toString(), ExpectedErrors .from("cannot truncate a table referenced in a foreign key constraint", "is not a table")); } diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 253a86c03..8b0d331c2 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -71,7 +71,7 @@ public static Query create(PostgresGlobalState globalState) { sb.append(PostgresVisitor.asString(where)); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java index 3f07e92b6..da0ef9930 100644 --- a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java @@ -7,7 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -61,7 +61,7 @@ public static Query create(PostgresGlobalState globalState) { */ errors.add("ERROR: ANALYZE option must be specified when a column list is provided"); errors.add("VACUUM option DISABLE_PAGE_SKIPPING cannot be used with FULL"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } private static void addTableAndColumns(PostgresTable table, StringBuilder sb) { diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index a1b100bc2..1366c7f41 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresSelect; @@ -87,7 +87,7 @@ public static Query create(PostgresGlobalState globalState) { errors.add("non-integer constant in DISTINCT ON"); errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); PostgresCommon.addCommonExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 58996bee6..61cf44312 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -13,7 +13,7 @@ import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; @@ -120,7 +120,7 @@ private int getUnoptimizedQueryCount(List fromTables, Postgr logger.writeCurrent(unoptimizedQueryString); } errors.add("canceling statement due to statement timeout"); - Query q = new QueryAdapter(unoptimizedQueryString, errors); + Query q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { rs = q.executeAndGet(state); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 77b86eb9c..2dda07258 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -64,7 +64,7 @@ public Query getRectifiedQuery() throws SQLException { List orderBy = new PostgresExpressionGenerator(globalState).setColumns(columns) .generateOrderBy(); selectStatement.setOrderByExpressions(orderBy); - return new QueryAdapter(PostgresVisitor.asString(selectStatement)); + return new SQLQueryAdapter(PostgresVisitor.asString(selectStatement)); } /* @@ -139,7 +139,7 @@ protected Query getContainmentCheckQuery(Query query) throws SQLException { } } String resultingQueryString = sb.toString(); - return new QueryAdapter(resultingQueryString, errors); + return new SQLQueryAdapter(resultingQueryString, errors); } @Override diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index 72a3a4cd3..d87a37ce3 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -12,7 +12,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -114,7 +114,7 @@ private String getAggregateResult(String queryString) throws SQLException { } } String resultString; - QueryAdapter q = new QueryAdapter(queryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index ba410aba0..46c05c9a7 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -9,14 +9,15 @@ import java.util.List; import sqlancer.AbstractAction; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -93,7 +94,7 @@ public boolean canBeRetried() { CHECK_RTREE_TABLE((g) -> { SQLite3Table table = g.getSchema().getRandomTableOrBailout(t -> t.getName().startsWith("r")); String format = String.format("SELECT rtreecheck('%s');", table.getName()); - return new QueryAdapter(format, ExpectedErrors.from("The database file is locked")); + return new SQLQueryAdapter(format, ExpectedErrors.from("The database file is locked")); }), // VIRTUAL_TABLE_ACTION(SQLite3VirtualFTSTableCommandGenerator::create), // CREATE_VIEW(SQLite3ViewGenerator::generate), // @@ -112,7 +113,7 @@ public Query getQuery(SQLite3GlobalState state) throws Exception { } } - public static class SQLite3GlobalState extends GlobalState { + public static class SQLite3GlobalState extends SQLGlobalState { @Override protected SQLite3Schema readSchema() throws SQLException { @@ -200,7 +201,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws Exception { assert globalState.getSchema().getTables().getTables().size() == nrTablesToCreate; checkTablesForGeneratedColumnLoops(globalState); if (globalState.getDmbsSpecificOptions().testDBStats && Randomly.getBooleanWithSmallProbability()) { - QueryAdapter tableQuery = new QueryAdapter( + SQLQueryAdapter tableQuery = new SQLQueryAdapter( "CREATE VIRTUAL TABLE IF NOT EXISTS stat USING dbstat(main)"); globalState.executeStatement(tableQuery); } @@ -223,7 +224,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws Exception { private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws Exception { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { - Query q = new QueryAdapter("SELECT * FROM " + table.getName(), + Query q = new SQLQueryAdapter("SELECT * FROM " + table.getName(), ExpectedErrors.from("needs an odd number of arguments", " requires an even number of arguments", "generated column loop", "integer overflow", "malformed JSON", "JSON cannot hold BLOB values", "JSON path error", "labels must be TEXT", @@ -278,12 +279,12 @@ private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws Ex Randomly.fromOptions("UTF-8", "UTF-16", "UTF-16le", "UTF-16be"))); } for (String s : pragmasToExecute) { - globalState.executeStatement(new QueryAdapter(s)); + globalState.executeStatement(new SQLQueryAdapter(s)); } } @Override - public Connection createDatabase(SQLite3GlobalState globalState) throws SQLException { + public SQLConnection createDatabase(SQLite3GlobalState globalState) throws SQLException { File dir = new File("." + File.separator + "databases"); if (!dir.exists()) { dir.mkdir(); @@ -293,7 +294,7 @@ public Connection createDatabase(SQLite3GlobalState globalState) throws SQLExcep dataBase.delete(); } String url = "jdbc:sqlite:" + dataBase.getAbsolutePath(); - return DriverManager.getConnection(url); + return new SQLConnection(DriverManager.getConnection(url)); } @Override diff --git a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java index 94e3340d8..4c0ac67cb 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public final class SQLite3AnalyzeGenerator { @@ -36,7 +36,7 @@ public static Query generateAnalyze(SQLite3GlobalState globalState) { throw new AssertionError(); } } - return new QueryAdapter(sb.toString(), ExpectedErrors.from("The database file is locked")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("The database file is locked")); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index 1234789d7..702746e15 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; @@ -47,7 +47,7 @@ public static Query createTableStatement(String rTreeTableName, SQLite3GlobalSta errors.add("Wrong number of columns for an rtree table"); errors.add("Too many columns for an rtree table"); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java index aca168919..239ecfc7a 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.sqlite3.SQLite3Provider.Action; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -24,7 +24,7 @@ public static Query explain(SQLite3GlobalState globalState) throws Exception { } while (action == Action.EXPLAIN); Query query = action.getQuery(globalState); sb.append(query); - return new QueryAdapter(sb.toString(), query.getExpectedErrors()); + return new SQLQueryAdapter(sb.toString(), query.getExpectedErrors()); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java index fffeb63ae..65bed34e5 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public class SQLite3PragmaGenerator { @@ -68,7 +68,7 @@ public void createPragma(String pragmaName, Supplier supplier) { } } - public QueryAdapter insert(SQLite3GlobalState globalState) { + public SQLQueryAdapter insert(SQLite3GlobalState globalState) { Randomly r = globalState.getRandomly(); Pragma p = Randomly.fromOptions(Pragma.values()); switch (p) { @@ -232,10 +232,10 @@ public QueryAdapter insert(SQLite3GlobalState globalState) { sb.append(";"); String pragmaString = sb.toString(); errors.add("The database file is locked"); - return new QueryAdapter(pragmaString, errors); + return new SQLQueryAdapter(pragmaString, errors); } - public static QueryAdapter insertPragma(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter insertPragma(SQLite3GlobalState globalState) throws SQLException { return new SQLite3PragmaGenerator().insert(globalState); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java index 9f4e319dd..33105bc7c 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -44,6 +44,6 @@ public static Query executeReindex(SQLite3GlobalState globalState) { throw new AssertionError(t); } } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java index d2e6412a6..38cd1f033 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public final class SQLite3TransactionGenerator { @@ -17,7 +17,7 @@ public static Query generateCommit(SQLite3GlobalState globalState) { if (Randomly.getBoolean()) { sb.append(" TRANSACTION"); } - return new QueryAdapter(sb.toString(), ExpectedErrors.from("no transaction is active", + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("no transaction is active", "The database file is locked", "FOREIGN KEY constraint failed"), true); } @@ -28,13 +28,13 @@ public static Query generateBeginTransaction(SQLite3GlobalState globalState) { sb.append(Randomly.fromOptions("DEFERRED", "IMMEDIATE", "EXCLUSIVE")); } sb.append(" TRANSACTION;"); - return new QueryAdapter(sb.toString(), + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot start a transaction within a transaction", "The database file is locked")); } public static Query generateRollbackTransaction(SQLite3GlobalState globalState) { // TODO: could be extended by savepoint - return new QueryAdapter("ROLLBACK TRANSACTION;", + return new SQLQueryAdapter("ROLLBACK TRANSACTION;", ExpectedErrors.from("no transaction is active", "The database file is locked"), true); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java index 32b462abd..2389adb94 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; /** @@ -20,7 +20,7 @@ public static Query executeVacuum(SQLite3GlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("temp", "main")); } - return new QueryAdapter(sb.toString(), ExpectedErrors.from("cannot VACUUM from within a transaction", + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot VACUUM from within a transaction", "cannot VACUUM - SQL statements in progress", "The database file is locked")); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java index 56616410c..6693ad41c 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; @@ -92,7 +92,7 @@ private Query generate() { throw new AssertionError(); } errors.add("The database file is locked"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java index 7cf0fb00d..d0dc74f53 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; import sqlancer.sqlite3.gen.SQLite3Common; @@ -81,7 +81,7 @@ private Query getQuery(SQLite3Schema s, SQLite3AlterTable alterTable) throws Ass default: throw new AssertionError(); } - return new QueryAdapter(alterTable.sb.toString(), errors, true); + return new SQLQueryAdapter(alterTable.sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java index bea71997b..9933a1459 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; @@ -109,7 +109,7 @@ public static Query create(SQLite3GlobalState globalState) throws SQLException { } sb.append("END"); - return new QueryAdapter(sb.toString(), + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("parser stack overflow", "unsupported frame specification")); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java index 5956aa63a..07c81fba7 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java @@ -7,7 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.gen.SQLite3Common; public class SQLite3CreateVirtualFTSTableGenerator { @@ -46,7 +46,7 @@ public Query create() { } else { createFts5Table(); } - return new QueryAdapter(sb.toString(), + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("unrecognized parameter", "unknown tokenizer: ascii"), true); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java index cc96d1d49..2c39f6771 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; // see https://www.sqlite.org/lang_dropindex.html @@ -22,7 +22,7 @@ public static Query dropIndex(SQLite3GlobalState globalState) { sb.append('"'); sb.append(indexName); sb.append('"'); - return new QueryAdapter(sb.toString(), ExpectedErrors.from( + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from( "[SQLITE_ERROR] SQL error or missing database (index associated with UNIQUE or PRIMARY KEY constraint cannot be dropped)"), true); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java index 4cac79a87..c538da076 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java @@ -4,7 +4,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public final class SQLite3DropTableGenerator { @@ -21,7 +21,7 @@ public static Query dropTable(SQLite3GlobalState globalState) { sb.append("IF EXISTS "); } sb.append(globalState.getSchema().getRandomTableOrBailout(t -> !t.isView()).getName()); - return new QueryAdapter(sb.toString(), + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", "Abort due to constraint violation (FOREIGN KEY constraint failed)", "SQL error or missing database"), diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index f6e487e30..0b806038e 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -58,7 +58,7 @@ private Query create() throws SQLException { * https://www.mail-archive.com/sqlite-users@mailinglists.sqlite.org/msg115014.html). */ errors.add("[SQLITE_ERROR] SQL error or missing database (no such column:"); - return new QueryAdapter(q, errors, true); + return new SQLQueryAdapter(q, errors, true); } private String createIndex(SQLite3Table t, List columns) { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index da04a219e..b439dceda 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -57,7 +57,7 @@ public static Query createTableStatement(String tableName, SQLite3GlobalState gl errors.add("parser stack overflow"); errors.add("malformed JSON"); errors.add("JSON cannot hold BLOB values"); - return new QueryAdapter(sqLite3TableGenerator.sb.toString(), errors, true); + return new SQLQueryAdapter(sqLite3TableGenerator.sb.toString(), errors, true); } public void start() { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index bb5a8b1d2..480908f3d 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -25,7 +25,7 @@ public static Query dropView(SQLite3GlobalState globalState) { SQLite3Schema s = globalState.getSchema(); StringBuilder sb = new StringBuilder("DROP VIEW "); sb.append(s.getRandomViewOrBailout().getName()); - return new QueryAdapter(sb.toString(), true); + return new SQLQueryAdapter(sb.toString(), true); } public static Query generate(SQLite3GlobalState globalState) throws SQLException { @@ -53,7 +53,7 @@ public static Query generate(SQLite3GlobalState globalState) throws SQLException } while (globalState.getDmbsSpecificOptions().oracles == SQLite3OracleFactory.PQS && !checkAffinity(randomQuery)); sb.append(SQLite3Visitor.asString(randomQuery)); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 0453535a6..c4d678297 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; @@ -41,7 +41,7 @@ public static Query deleteContent(SQLite3GlobalState globalState, SQLite3Table t "cannot INSERT into generated column", "A table in the database is locked", "load_extension() prohibited in triggers and views", "The database file is locked")); SQLite3Errors.addDeleteErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java index 2e0d425e8..7a8d6d2ab 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java @@ -7,7 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; @@ -38,7 +38,7 @@ public static Query insertRow(SQLite3GlobalState globalState) throws SQLExceptio public static Query insertRow(SQLite3GlobalState globalState, SQLite3Table randomTable) { SQLite3InsertGenerator generator = new SQLite3InsertGenerator(globalState, globalState.getRandomly()); String query = generator.insertRow(randomTable); - return new QueryAdapter(query, generator.errors, true); + return new SQLQueryAdapter(query, generator.errors, true); } private String insertRow(SQLite3Table table) { diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java index d953afb32..c5a1d8be7 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java @@ -10,7 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; @@ -81,7 +81,7 @@ private Query getQuery() { sb.append(" noskipscan"); } sb.append("')"); - return new QueryAdapter(sb.toString(), ExpectedErrors.from("no such table", "The database file is locked")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("no such table", "The database file is locked")); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 5c7fa9bf6..0ecb8ee11 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; @@ -117,7 +117,7 @@ private Query update(SQLite3Table table) { SQLite3Errors.addInsertNowErrors(errors); SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addDeleteErrors(errors); - return new QueryAdapter(sb.toString(), errors, true /* column could have an ON UPDATE clause */); + return new SQLQueryAdapter(sb.toString(), errors, true /* column could have an ON UPDATE clause */); } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index 939b78833..500522ffb 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; @@ -21,7 +21,7 @@ public void check() throws Exception { .asString(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + ";"; try { if (globalState.getDmbsSpecificOptions().executeQuery) { - globalState.executeStatement(new QueryAdapter(s)); + globalState.executeStatement(new SQLQueryAdapter(s)); globalState.getManager().incrementSelectQueryCount(); } } catch (Error e) { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 1edc97aad..609c76f70 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -9,7 +9,7 @@ import sqlancer.Randomly; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -84,7 +84,7 @@ private int getUnoptimizedQuery(SQLite3Select select, SQLite3Expression randomWh if (options.logEachSelect()) { logger.writeCurrent(unoptimizedQueryString); } - QueryAdapter q = new QueryAdapter(unoptimizedQueryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); return extractCounts(q); } @@ -105,11 +105,11 @@ private int getOptimizedQuery(SQLite3Select select, SQLite3Expression randomWher if (options.logEachSelect()) { logger.writeCurrent(optimizedQueryString); } - QueryAdapter q = new QueryAdapter(optimizedQueryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); return useAggregate ? extractCounts(q) : countRows(q); } - private int countRows(QueryAdapter q) { + private int countRows(SQLQueryAdapter q) { int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { @@ -132,7 +132,7 @@ private int countRows(QueryAdapter q) { return count; } - private int extractCounts(QueryAdapter q) { + private int extractCounts(SQLQueryAdapter q) { int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 353491985..3dcd6824c 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -12,7 +12,7 @@ import sqlancer.StateToReproduce.OracleRunReproductionState; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; @@ -54,7 +54,7 @@ public SQLite3PivotedQuerySynthesisOracle(SQLite3GlobalState globalState) { public Query getRectifiedQuery() throws SQLException { SQLite3Select selectStatement = getQuery(); SQLite3Errors.addExpectedExpressionErrors(errors); - return new QueryAdapter(SQLite3Visitor.asString(selectStatement), errors); + return new SQLQueryAdapter(SQLite3Visitor.asString(selectStatement), errors); } public SQLite3Select getQuery() throws SQLException { @@ -186,7 +186,7 @@ protected Query getContainmentCheckQuery(Query query) throws SQLException { sb.append(query.getUnterminatedQueryString()); sb.append(")"); String resultingQueryString = sb.toString(); - return new QueryAdapter(resultingQueryString, query.getExpectedErrors()); + return new SQLQueryAdapter(resultingQueryString, query.getExpectedErrors()); } private String getGeneralizedPivotRowValues() { diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 3b53cfbbe..79d5bc09b 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -9,7 +9,7 @@ import sqlancer.Randomly; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -74,7 +74,7 @@ public void check() throws SQLException { // state.getState().queryString = "--" + finalText; String firstResult; String secondResult; - QueryAdapter q = new QueryAdapter(originalQuery, errors); + SQLQueryAdapter q = new SQLQueryAdapter(originalQuery, errors); try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); @@ -85,7 +85,7 @@ public void check() throws SQLException { throw new IgnoreMeException(); } - QueryAdapter q2 = new QueryAdapter(metamorphicText, errors); + SQLQueryAdapter q2 = new SQLQueryAdapter(metamorphicText, errors); try (SQLancerResultSet result = q2.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 4870ae109..d0916cfa9 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -1,6 +1,5 @@ package sqlancer.sqlite3.schema; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -14,9 +13,10 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; @@ -26,7 +26,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table.TableKind; -public class SQLite3Schema extends AbstractSchema { +public class SQLite3Schema extends AbstractSchema { /** * All possible aliases for the rowid column. @@ -155,7 +155,7 @@ public SQLite3Tables(List tables) { super(tables); } - public SQLite3RowValue getRandomRowValue(Connection con) throws SQLException { + public SQLite3RowValue getRandomRowValue(SQLConnection con) throws SQLException { String randomRow = String.format("SELECT %s, %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + c.getName() + ")"), @@ -189,7 +189,7 @@ public SQLite3RowValue getRandomRowValue(Connection con) throws SQLException { } - public static class SQLite3Table extends AbstractTable { + public static class SQLite3Table extends AbstractRelationalTable { // TODO: why does the SQLite implementation have no table indexes? public enum TableKind { @@ -271,7 +271,7 @@ public String toString() { public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throws SQLException { List databaseTables = new ArrayList<>(); List indexNames = new ArrayList<>(); - Connection con = globalState.getConnection(); + SQLConnection con = globalState.getConnection(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SELECT name, type as category, sql FROM sqlite_master UNION " @@ -346,7 +346,7 @@ private static boolean isRowIdTable(boolean withoutRowid, boolean isView, boolea return !isView && !isVirtual && !withoutRowid; } - private static List getTableColumns(Connection con, String tableName, String sql, boolean isView, + private static List getTableColumns(SQLConnection con, String tableName, String sql, boolean isView, boolean isDbStatsTable) throws SQLException { List databaseColumns = new ArrayList<>(); try (Statement s2 = con.createStatement()) { diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 882618aac..86f4e63de 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -6,14 +6,15 @@ import java.sql.Statement; import sqlancer.AbstractAction; -import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.QueryProvider; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.gen.TiDBAlterTableGenerator; @@ -36,20 +37,20 @@ public TiDBProvider() { public enum Action implements AbstractAction { INSERT(TiDBInsertGenerator::getQuery), // ANALYZE_TABLE(TiDBAnalyzeTableGenerator::getQuery), // - TRUNCATE((g) -> new QueryAdapter("TRUNCATE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), // + TRUNCATE((g) -> new SQLQueryAdapter("TRUNCATE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), // CREATE_INDEX(TiDBIndexGenerator::getQuery), // DELETE(TiDBDeleteGenerator::getQuery), // SET(TiDBSetGenerator::getQuery), // UPDATE(TiDBUpdateGenerator::getQuery), // ADMIN_CHECKSUM_TABLE( - (g) -> new QueryAdapter("ADMIN CHECKSUM TABLE " + g.getSchema().getRandomTable().getName())), // + (g) -> new SQLQueryAdapter("ADMIN CHECKSUM TABLE " + g.getSchema().getRandomTable().getName())), // VIEW_GENERATOR(TiDBViewGenerator::getQuery), // ALTER_TABLE(TiDBAlterTableGenerator::getQuery), // EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); TiDBErrors.addExpressionErrors(errors); TiDBErrors.addExpressionHavingErrors(errors); - return new QueryAdapter( + return new SQLQueryAdapter( "EXPLAIN " + TiDBRandomQuerySynthesizer.generate(g, Randomly.smallNumber() + 1).getQueryString(), errors); }); @@ -66,7 +67,7 @@ public Query getQuery(TiDBGlobalState state) throws Exception { } } - public static class TiDBGlobalState extends GlobalState { + public static class TiDBGlobalState extends SQLGlobalState { @Override protected TiDBSchema readSchema() throws SQLException { @@ -131,7 +132,7 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { } @Override - public Connection createDatabase(TiDBGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLException { String databaseName = globalState.getDatabaseName(); String url = "jdbc:mysql://127.0.0.1:4000/"; Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), @@ -150,7 +151,7 @@ public Connection createDatabase(TiDBGlobalState globalState) throws SQLExceptio con.close(); con = DriverManager.getConnection("jdbc:mysql://127.0.0.1:4000/" + databaseName, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - return con; + return new SQLConnection(con); } @Override diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index a67930315..be78dab38 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -1,6 +1,5 @@ package sqlancer.tidb; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -8,14 +7,16 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; -public class TiDBSchema extends AbstractSchema { +public class TiDBSchema extends AbstractSchema { public enum TiDBDataType { @@ -230,7 +231,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { return new TiDBCompositeDataType(primitiveType, size); } - public static class TiDBTable extends AbstractTable { + public static class TiDBTable extends AbstractRelationalTable { public TiDBTable(String tableName, List columns, List indexes, boolean isView) { super(tableName, columns, indexes, isView); @@ -242,7 +243,7 @@ public boolean hasPrimaryKey() { } - public static TiDBSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static TiDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { @@ -259,7 +260,7 @@ public static TiDBSchema fromConnection(Connection con, String databaseName) thr return new TiDBSchema(databaseTables); } - private static List getTableNames(Connection con) throws SQLException { + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { ResultSet tableRs = s.executeQuery("SHOW TABLES"); @@ -271,7 +272,7 @@ private static List getTableNames(Connection con) throws SQLException { return tableNames; } - private static List getIndexes(Connection con, String tableName) throws SQLException { + private static List getIndexes(SQLConnection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format("SHOW INDEX FROM %s", tableName))) { @@ -284,7 +285,7 @@ private static List getIndexes(Connection con, String tableName) thr return indexes; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SHOW COLUMNS FROM " + tableName)) { diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 515cd5955..1e35c4640 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; @@ -101,7 +101,7 @@ public static Query getQuery(TiDBGlobalState globalState) { throw new AssertionError(a); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 126a064d3..93d6ebc1d 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -32,7 +32,7 @@ public static Query getQuery(TiDBGlobalState globalState) throws SQLException { sb.append(Randomly.getNotCachedInteger(1, 1024)); sb.append(" BUCKETS"); } - return new QueryAdapter(sb.toString(), ExpectedErrors.from("https://github.com/pingcap/tidb/issues/15993", + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("https://github.com/pingcap/tidb/issues/15993", /* https://github.com/pingcap/tidb/issues/15993 */ "doesn't have a default value" /* * https://github. * com/pingcap/tidb/ diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index c2b40bc2c..f4bf5d6bc 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -54,7 +54,7 @@ public static Query getQuery(TiDBGlobalState globalState) throws SQLException { errors.add("Bad Number"); errors.add("Division by 0"); errors.add("error parsing regexp"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index db7e796a5..00fbd568a 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -58,7 +58,7 @@ public static Query getQuery(TiDBGlobalState globalState) throws SQLException { } errors.add("Cannot decode index value, because"); // invalid value for generated column errors.add("index already exist"); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java index 253fb47a2..cb9aed477 100644 --- a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java @@ -7,7 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -67,7 +67,7 @@ private Query get() { sb.append(TiDBVisitor.asString(gen.generateExpression())); } errors.add("Illegal mix of collations"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } private void insertColumns(StringBuilder sb, List columns) { diff --git a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java index 730c78172..6450ffc6d 100644 --- a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java +++ b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTables; @@ -22,7 +22,7 @@ private TiDBRandomQuerySynthesizer() { public static Query generate(TiDBGlobalState globalState, int nrColumns) { TiDBSelect select = generateSelect(globalState, nrColumns); - return new QueryAdapter(TiDBVisitor.asString(select)); + return new SQLQueryAdapter(TiDBVisitor.asString(select)); } public static TiDBSelect generateSelect(TiDBGlobalState globalState, int nrColumns) { diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 5bfffe13b..431724332 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; public final class TiDBSetGenerator { @@ -68,7 +68,7 @@ public static Query getQuery(TiDBGlobalState globalState) throws SQLException { sb.append(option.name); sb.append("="); sb.append(option.prod.apply(globalState.getRandomly())); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index ae10f1b50..c6835eecf 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -9,7 +9,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -48,7 +48,7 @@ public Query getQuery(TiDBGlobalState globalState) throws SQLException { } else { createNewTable(gen, sb); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 9e9b7cf39..aea2d7faf 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -48,7 +48,7 @@ public static Query getQuery(TiDBGlobalState globalState) throws SQLException { } TiDBErrors.addInsertErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index a77585758..3b36e027f 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -4,7 +4,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; -import sqlancer.common.query.QueryAdapter; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -45,7 +45,7 @@ public static Query getQuery(TiDBGlobalState globalState) { // TODO: CREATE VIEW v0(c0) AS SELECT '\\' FROM t0; causes an unexpected failure throw new IgnoreMeException(); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/test/sqlancer/TestParameterFormat.java b/test/sqlancer/TestParameterFormat.java index 6d9522ca0..f9d89545a 100644 --- a/test/sqlancer/TestParameterFormat.java +++ b/test/sqlancer/TestParameterFormat.java @@ -21,7 +21,7 @@ public class TestParameterFormat { @Test public void testOptionFormat() throws Exception { - List> providers = Main.getDBMSProviders(); + List> providers = Main.getDBMSProviders(); MainOptions options = new MainOptions(); Builder commandBuilder = JCommander.newBuilder().addObject(options); List parameterDescriptions = new ArrayList<>(); From f6bf8a81dba7d04d4ef491999477353b39056b49 Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 23 Nov 2020 01:40:01 +0100 Subject: [PATCH 0279/1284] PostgresQueryCatalogGenerator removed This deletion was decided with mrigger, because of errors in Connection Rework. --- src/sqlancer/citus/CitusProvider.java | 6 +-- src/sqlancer/postgres/PostgresProvider.java | 5 +- .../gen/PostgresQueryCatalogGenerator.java | 53 ------------------- 3 files changed, 2 insertions(+), 62 deletions(-) delete mode 100644 src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 2e69a6b1d..4a1f280ca 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -1,6 +1,5 @@ package sqlancer.citus; -import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; @@ -44,7 +43,6 @@ import sqlancer.postgres.gen.PostgresDiscardGenerator; import sqlancer.postgres.gen.PostgresDropIndexGenerator; import sqlancer.postgres.gen.PostgresNotifyGenerator; -import sqlancer.postgres.gen.PostgresQueryCatalogGenerator; import sqlancer.postgres.gen.PostgresReindexGenerator; import sqlancer.postgres.gen.PostgresSequenceGenerator; import sqlancer.postgres.gen.PostgresStatisticsGenerator; @@ -104,8 +102,7 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(CitusViewGenerator::create), // - QUERY_CATALOG((g) -> PostgresQueryCatalogGenerator.query()); + CREATE_VIEW(CitusViewGenerator::create); private final QueryProvider queryProvider; @@ -147,7 +144,6 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { case DELETE: case RESET_ROLE: case SET: - case QUERY_CATALOG: nrPerformed = r.getInteger(0, 5); break; case ANALYZE: diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 786988eba..5cdb49977 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -29,7 +29,6 @@ import sqlancer.postgres.gen.PostgresIndexGenerator; import sqlancer.postgres.gen.PostgresInsertGenerator; import sqlancer.postgres.gen.PostgresNotifyGenerator; -import sqlancer.postgres.gen.PostgresQueryCatalogGenerator; import sqlancer.postgres.gen.PostgresReindexGenerator; import sqlancer.postgres.gen.PostgresSequenceGenerator; import sqlancer.postgres.gen.PostgresSetGenerator; @@ -112,8 +111,7 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(PostgresViewGenerator::create), // - QUERY_CATALOG((g) -> PostgresQueryCatalogGenerator.query()); + CREATE_VIEW(PostgresViewGenerator::create); private final QueryProvider queryProvider; @@ -155,7 +153,6 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case DELETE: case RESET_ROLE: case SET: - case QUERY_CATALOG: nrPerformed = r.getInteger(0, 5); break; case ANALYZE: diff --git a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java deleted file mode 100644 index 0e44b0884..000000000 --- a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java +++ /dev/null @@ -1,53 +0,0 @@ -package sqlancer.postgres.gen; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; - -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; -import sqlancer.common.query.Query; -import sqlancer.common.query.SQLQueryAdapter; - -public final class PostgresQueryCatalogGenerator { - - private PostgresQueryCatalogGenerator() { - } - - public static Query query() { - StringBuilder sb = new StringBuilder(); - sb.append("SELECT * FROM "); - sb.append( - Randomly.fromOptions("pg_aggregate", "pg_am", "pg_amop", "pg_amproc", "pg_attrdef", "pg_attribute", - "pg_authid", "pg_auth_members", "pg_cast", "pg_class", "pg_collation", "pg_constraint", - "pg_conversion", "pg_database", "pg_db_role_setting", "pg_default_acl", "pg_depend", - "pg_description", "pg_enum", "pg_event_trigger", "pg_extension", "pg_foreign_data_wrapper", - "pg_foreign_server", "pg_foreign_table", "pg_index", "pg_inherits", "pg_init_privs", - "pg_language", "pg_largeobject", "pg_largeobject_metadata", "pg_namespace", "pg_opclass", - "pg_operator", "pg_opfamily", "pg_partitioned_table", /* "pg_pltemplate", */ - "pg_policy", "pg_proc", "pg_publication", "pg_publication_rel", "pg_range", - "pg_replication_origin", "pg_rewrite", "pg_seclabel", "pg_sequence", "pg_shdepend", - "pg_shdescription", "pg_shseclabel", "pg_statistic", "pg_statistic_ext", - "pg_statistic_ext_data", "pg_subscription", "pg_subscription_rel", "pg_tablespace", - "pg_transform", "pg_trigger", "pg_ts_config", "pg_ts_config_map", "pg_ts_dict", "pg_ts_parser", - "pg_ts_template", "pg_type", "pg_user_mapping")); - return new SQLQueryAdapter(sb.toString()) { - @Override - public boolean execute(SQLGlobalState globalState, String... fills) throws SQLException { - try (Statement s = globalState.getConnection().createStatement()) { - try (ResultSet rs = s.executeQuery(getQueryString())) { - // CHECKSTYLE:OFF - while (rs.next()) { - // only force the DBMS to fetch the records and hope that they cause an - // invalid state - } - // CHECKSTYLE:ON - } - } - return true; - } - }; - } - -} From 95e4fe5a1b90f5c5b1ecddfad5a0cc14d54d4bf1 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 24 Nov 2020 06:40:56 +0000 Subject: [PATCH 0280/1284] Bump mariadb-java-client from 2.7.0 to 2.7.1 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.7.0 to 2.7.1. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/commits) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9aa769c21..f958a05c1 100644 --- a/pom.xml +++ b/pom.xml @@ -206,7 +206,7 @@ org.mariadb.jdbc mariadb-java-client - 2.7.0 + 2.7.1 org.duckdb From 62e302ebaa56a56528713b7875af126ee33e8cb2 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 26 Nov 2020 11:57:02 +0100 Subject: [PATCH 0281/1284] Continuation of Reworking Connection to DB to support non-SQL connections These adaptations to the code fix now the compilation issues of the earlier commit. Additionally, a lot of times it was necessary to change Query to SQLQueryAdapter in the DBMS specific packages. --- src/sqlancer/AbstractAction.java | 2 +- src/sqlancer/ComparatorHelper.java | 4 +-- src/sqlancer/DatabaseProvider.java | 3 +- src/sqlancer/GlobalState.java | 26 +++++------------ src/sqlancer/Main.java | 20 ++++++------- src/sqlancer/ProviderAdapter.java | 5 ++-- src/sqlancer/SQLConnection.java | 4 +-- src/sqlancer/SQLGlobalState.java | 17 ++++++++++- src/sqlancer/SQLProviderAdapter.java | 13 +++++---- src/sqlancer/StateToReproduce.java | 12 ++++---- src/sqlancer/StatementExecutor.java | 4 ++- src/sqlancer/citus/CitusProvider.java | 24 ++++++++-------- src/sqlancer/citus/CitusSchema.java | 1 - .../citus/gen/CitusAlterTableGenerator.java | 5 ++-- .../citus/gen/CitusDeleteGenerator.java | 6 ++-- .../citus/gen/CitusIndexGenerator.java | 6 ++-- .../citus/gen/CitusInsertGenerator.java | 6 ++-- src/sqlancer/citus/gen/CitusSetGenerator.java | 6 ++-- .../citus/gen/CitusUpdateGenerator.java | 6 ++-- .../citus/gen/CitusViewGenerator.java | 6 ++-- .../clickhouse/ClickHouseProvider.java | 20 ++++++------- src/sqlancer/clickhouse/ClickHouseSchema.java | 5 ++-- .../gen/ClickHouseInsertGenerator.java | 5 ++-- .../gen/ClickHouseTableGenerator.java | 4 +-- .../cockroachdb/CockroachDBProvider.java | 23 ++++++++------- .../cockroachdb/CockroachDBSchema.java | 5 ++-- .../gen/CockroachDBCommentOnGenerator.java | 3 +- .../CockroachDBCreateStatisticsGenerator.java | 3 +- .../gen/CockroachDBDeleteGenerator.java | 3 +- .../gen/CockroachDBIndexGenerator.java | 4 +-- .../gen/CockroachDBInsertGenerator.java | 3 +- .../CockroachDBRandomQuerySynthesizer.java | 3 +- ...CockroachDBSetClusterSettingGenerator.java | 3 +- .../gen/CockroachDBSetSessionGenerator.java | 3 +- .../gen/CockroachDBShowGenerator.java | 3 +- .../gen/CockroachDBTableGenerator.java | 4 +-- .../gen/CockroachDBTruncateGenerator.java | 3 +- .../gen/CockroachDBUpdateGenerator.java | 3 +- .../gen/CockroachDBViewGenerator.java | 3 +- .../oracle/CockroachDBNoRECOracle.java | 9 +++--- .../common/gen/AbstractGenerator.java | 3 +- src/sqlancer/common/log/LoggableFactory.java | 4 +-- .../common/log/SQLLoggableFactory.java | 4 +-- .../common/oracle/CompositeTestOracle.java | 4 +-- src/sqlancer/common/oracle/NoRECBase.java | 1 - .../oracle/PivotedQuerySynthesisBase.java | 17 +++++------ src/sqlancer/common/query/Query.java | 13 +++++---- src/sqlancer/common/query/QueryProvider.java | 6 ---- .../common/query/SQLQueryAdapter.java | 12 ++++---- .../common/query/SQLQueryProvider.java | 6 ++++ .../query/SQLQueryResultCheckAdapter.java | 7 +++-- .../schema/AbstractRelationalTable.java | 6 ++-- src/sqlancer/common/schema/AbstractTable.java | 10 +++---- src/sqlancer/duckdb/DuckDBProvider.java | 20 ++++++------- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- .../duckdb/gen/DuckDBAlterTableGenerator.java | 3 +- .../duckdb/gen/DuckDBDeleteGenerator.java | 3 +- .../duckdb/gen/DuckDBIndexGenerator.java | 3 +- .../duckdb/gen/DuckDBInsertGenerator.java | 5 ++-- .../duckdb/gen/DuckDBTableGenerator.java | 3 +- .../duckdb/gen/DuckDBUpdateGenerator.java | 3 +- .../duckdb/gen/DuckDBViewGenerator.java | 3 +- .../duckdb/test/DuckDBNoRECOracle.java | 9 +++--- src/sqlancer/h2/H2DeleteGenerator.java | 3 +- src/sqlancer/h2/H2IndexGenerator.java | 5 ++-- src/sqlancer/h2/H2InsertGenerator.java | 5 ++-- src/sqlancer/h2/H2Provider.java | 19 ++++++------- src/sqlancer/h2/H2Schema.java | 2 +- src/sqlancer/h2/H2SetGenerator.java | 3 +- src/sqlancer/h2/H2TableGenerator.java | 3 +- src/sqlancer/h2/H2UpdateGenerator.java | 3 +- src/sqlancer/h2/H2ViewGenerator.java | 3 +- src/sqlancer/mariadb/MariaDBProvider.java | 10 +++---- src/sqlancer/mariadb/MariaDBSchema.java | 3 +- .../gen/MariaDBExpressionGenerator.java | 1 - .../mariadb/gen/MariaDBIndexGenerator.java | 3 +- .../mariadb/gen/MariaDBInsertGenerator.java | 3 +- .../mariadb/gen/MariaDBSetGenerator.java | 5 ++-- .../MariaDBTableAdminCommandGenerator.java | 15 +++++----- .../mariadb/gen/MariaDBTableGenerator.java | 5 ++-- .../mariadb/gen/MariaDBTruncateGenerator.java | 3 +- .../mariadb/gen/MariaDBUpdateGenerator.java | 3 +- src/sqlancer/mysql/MySQLGlobalState.java | 1 - src/sqlancer/mysql/MySQLProvider.java | 15 +++++----- src/sqlancer/mysql/MySQLSchema.java | 2 +- src/sqlancer/mysql/gen/MySQLAlterTable.java | 5 ++-- .../mysql/gen/MySQLDeleteGenerator.java | 5 ++-- src/sqlancer/mysql/gen/MySQLDropIndex.java | 3 +- .../mysql/gen/MySQLInsertGenerator.java | 9 +++--- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 5 ++-- .../mysql/gen/MySQLTableGenerator.java | 5 ++-- .../gen/MySQLTruncateTableGenerator.java | 3 +- src/sqlancer/mysql/gen/admin/MySQLFlush.java | 5 ++-- src/sqlancer/mysql/gen/admin/MySQLReset.java | 3 +- .../gen/datadef/MySQLIndexGenerator.java | 5 ++-- .../gen/tblmaintenance/MySQLAnalyzeTable.java | 5 ++-- .../gen/tblmaintenance/MySQLCheckTable.java | 5 ++-- .../gen/tblmaintenance/MySQLChecksum.java | 5 ++-- .../gen/tblmaintenance/MySQLOptimize.java | 5 ++-- .../mysql/gen/tblmaintenance/MySQLRepair.java | 5 ++-- .../MySQLPivotedQuerySynthesisOracle.java | 7 +++-- .../postgres/PostgresGlobalState.java | 2 +- src/sqlancer/postgres/PostgresProvider.java | 23 ++++++++------- src/sqlancer/postgres/PostgresSchema.java | 5 ++-- .../gen/PostgresAlterTableGenerator.java | 6 ++-- .../gen/PostgresAnalyzeGenerator.java | 3 +- .../gen/PostgresClusterGenerator.java | 3 +- .../gen/PostgresCommentGenerator.java | 3 +- .../postgres/gen/PostgresDeleteGenerator.java | 3 +- .../gen/PostgresDiscardGenerator.java | 3 +- .../gen/PostgresDropIndexGenerator.java | 3 +- .../postgres/gen/PostgresIndexGenerator.java | 3 +- .../postgres/gen/PostgresInsertGenerator.java | 3 +- .../postgres/gen/PostgresNotifyGenerator.java | 7 ++--- .../gen/PostgresReindexGenerator.java | 3 +- .../gen/PostgresSequenceGenerator.java | 3 +- .../postgres/gen/PostgresSetGenerator.java | 3 +- .../gen/PostgresStatisticsGenerator.java | 5 ++-- .../postgres/gen/PostgresTableGenerator.java | 5 ++-- .../gen/PostgresTransactionGenerator.java | 3 +- .../gen/PostgresTruncateGenerator.java | 3 +- .../postgres/gen/PostgresUpdateGenerator.java | 3 +- .../postgres/gen/PostgresVacuumGenerator.java | 3 +- .../postgres/gen/PostgresViewGenerator.java | 3 +- .../postgres/oracle/PostgresNoRECOracle.java | 3 +- .../PostgresPivotedQuerySynthesisOracle.java | 7 +++-- src/sqlancer/sqlite3/SQLite3Provider.java | 28 +++++++++---------- .../sqlite3/gen/SQLite3AnalyzeGenerator.java | 3 +- ...Lite3CreateVirtualRtreeTabelGenerator.java | 3 +- .../sqlite3/gen/SQLite3ExplainGenerator.java | 5 ++-- .../sqlite3/gen/SQLite3ReindexGenerator.java | 3 +- .../gen/SQLite3TransactionGenerator.java | 7 ++--- .../sqlite3/gen/SQLite3VacuumGenerator.java | 3 +- ...QLite3VirtualFTSTableCommandGenerator.java | 5 ++-- .../sqlite3/gen/ddl/SQLite3AlterTable.java | 5 ++-- .../ddl/SQLite3CreateTriggerGenerator.java | 3 +- ...SQLite3CreateVirtualFTSTableGenerator.java | 5 ++-- .../gen/ddl/SQLite3DropIndexGenerator.java | 3 +- .../gen/ddl/SQLite3DropTableGenerator.java | 3 +- .../gen/ddl/SQLite3IndexGenerator.java | 5 ++-- .../gen/ddl/SQLite3TableGenerator.java | 3 +- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 5 ++-- .../gen/dml/SQLite3DeleteGenerator.java | 5 ++-- .../gen/dml/SQLite3InsertGenerator.java | 5 ++-- .../gen/dml/SQLite3StatTableGenerator.java | 8 +++--- .../gen/dml/SQLite3UpdateGenerator.java | 7 ++--- .../SQLite3PivotedQuerySynthesisOracle.java | 7 +++-- .../sqlite3/schema/SQLite3Schema.java | 5 ++-- src/sqlancer/tidb/TiDBProvider.java | 19 ++++++------- src/sqlancer/tidb/TiDBSchema.java | 2 +- .../tidb/gen/TiDBAlterTableGenerator.java | 3 +- .../tidb/gen/TiDBAnalyzeTableGenerator.java | 3 +- .../tidb/gen/TiDBDeleteGenerator.java | 3 +- src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 3 +- .../tidb/gen/TiDBInsertGenerator.java | 5 ++-- .../tidb/gen/TiDBRandomQuerySynthesizer.java | 3 +- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 3 +- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 3 +- .../tidb/gen/TiDBUpdateGenerator.java | 3 +- src/sqlancer/tidb/gen/TiDBViewGenerator.java | 3 +- 160 files changed, 413 insertions(+), 507 deletions(-) delete mode 100644 src/sqlancer/common/query/QueryProvider.java create mode 100644 src/sqlancer/common/query/SQLQueryProvider.java diff --git a/src/sqlancer/AbstractAction.java b/src/sqlancer/AbstractAction.java index 92996a7fa..db218ebe8 100644 --- a/src/sqlancer/AbstractAction.java +++ b/src/sqlancer/AbstractAction.java @@ -4,7 +4,7 @@ public interface AbstractAction { - Query getQuery(G globalState) throws Exception; + Query getQuery(G globalState) throws Exception; /** * Specifies whether it makes sense to request a {@link Query}, when the previous call to {@link #getQuery(Object)} diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 3233ab243..f05fe5ee3 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -35,8 +35,8 @@ static boolean equals(double a, double b) { return Math.abs(a - b) < 0.001 * Math.max(Math.abs(a), Math.abs(b)) + 0.001; } - public static List getResultSetFirstColumnAsString(String queryString, - ExpectedErrors errors, SQLGlobalState state) throws SQLException { + public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, + SQLGlobalState state) throws SQLException { if (state.getOptions().logEachSelect()) { // TODO: refactor me state.getLogger().writeCurrent(queryString); diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 6530eddcc..17f2a5ada 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -2,8 +2,7 @@ import sqlancer.common.log.LoggableFactory; -public interface DatabaseProvider, O extends DBMSSpecificOptions, - C extends SQLancerDBConnection> { +public interface DatabaseProvider, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> { /** * Gets the the {@link GlobalState} class. diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 7476171c9..97c5a7a9d 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -14,7 +14,7 @@ public abstract class GlobalState, S extends Ab private S schema; private Main.StateLogger logger; private StateToReproduce state; - private Main.QueryManager manager; + private Main.QueryManager manager; private String databaseName; public void setConnection(C con) { @@ -34,7 +34,6 @@ public O getDmbsSpecificOptions() { return dmbsSpecificOptions; } - public void setRandomly(Randomly r) { this.r = r; } @@ -67,11 +66,11 @@ public StateToReproduce getState() { return state; } - public Main.QueryManager getManager() { + public Main.QueryManager getManager() { return manager; } - public void setManager(Main.QueryManager manager) { + public void setManager(Main.QueryManager manager) { this.manager = manager; } @@ -83,7 +82,7 @@ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } - private ExecutionTimer executePrologue(Query q) throws Exception { + private ExecutionTimer executePrologue(Query q) throws Exception { boolean logExecutionTime = getOptions().logExecutionTime(); ExecutionTimer timer = null; if (logExecutionTime) { @@ -102,27 +101,16 @@ private ExecutionTimer executePrologue(Query q) throws Exception { return timer; } - private void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { - boolean logExecutionTime = getOptions().logExecutionTime(); - if (success && getOptions().printSucceedingStatements()) { - System.out.println(q.getQueryString()); - } - if (logExecutionTime) { - getLogger().writeCurrent(" -- " + timer.end().asString()); - } - if (q.couldAffectSchema()) { - updateSchema(); - } - } + protected abstract void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception; - public boolean executeStatement(Query q, String... fills) throws Exception { + public boolean executeStatement(Query q, String... fills) throws Exception { ExecutionTimer timer = executePrologue(q); boolean success = manager.execute(q, fills); executeEpilogue(q, success, timer); return success; } - public SQLancerResultSet executeStatementAndGet(Query q, String... fills) throws Exception { + public SQLancerResultSet executeStatementAndGet(Query q, String... fills) throws Exception { ExecutionTimer timer = executePrologue(q); SQLancerResultSet result = manager.executeAndGet(q, fills); boolean success = result != null; diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 1a8e2ec93..755ce36a8 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -208,7 +208,7 @@ private void printState(FileWriter writer, StateToReproduce state) { sb.append(databaseProvider.getLoggableFactory() .getInfo(state.getDatabaseName(), state.getDatabaseVersion(), state.getSeedValue()).getLogString()); - for (Query s : state.getStatements()) { + for (Query s : state.getStatements()) { sb.append(s.getQueryString()); sb.append('\n'); } @@ -221,15 +221,15 @@ private void printState(FileWriter writer, StateToReproduce state) { } - public static class QueryManager { + public static class QueryManager { - private final GlobalState globalState; + private final GlobalState globalState; - QueryManager(GlobalState globalState) { + QueryManager(GlobalState globalState) { this.globalState = globalState; } - public boolean execute(Query q, String... fills) throws Exception { + public boolean execute(Query q, String... fills) throws Exception { globalState.getState().logStatement(q); boolean success; success = q.execute(globalState, fills); @@ -237,7 +237,7 @@ public boolean execute(Query q, String... fills) throws Exception { return success; } - public SQLancerResultSet executeAndGet(Query q, String... fills) throws Exception { + public SQLancerResultSet executeAndGet(Query q, String... fills) throws Exception { globalState.getState().logStatement(q); SQLancerResultSet result; result = q.executeAndGet(globalState, fills); @@ -259,8 +259,7 @@ public static void main(String[] args) { System.exit(executeMain(args)); } - public static class DBMSExecutor, O extends DBMSSpecificOptions, C - extends SQLancerDBConnection> { + public static class DBMSExecutor, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> { private final DatabaseProvider provider; private final MainOptions options; @@ -309,7 +308,7 @@ public void run() throws Exception { state.setMainOptions(options); state.setDmbsSpecificOptions(command); try (C con = provider.createDatabase(state)) { - QueryManager manager = new QueryManager(state); + QueryManager manager = new QueryManager<>(state); try { stateToRepro.databaseVersion = con.getDatabaseVersion(); } catch (Exception e) { @@ -354,8 +353,7 @@ public StateToReproduce getStateToReproduce() { } } - public static class DBMSExecutorFactory, O extends DBMSSpecificOptions, - C extends SQLancerDBConnection> { + public static class DBMSExecutorFactory, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> { private final DatabaseProvider provider; private final MainOptions options; diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index caeb20314..1e3158563 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -6,9 +6,10 @@ import sqlancer.StateToReproduce.OracleRunReproductionState; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.schema.AbstractSchema; -public abstract class ProviderAdapter, O extends DBMSSpecificOptions>, - C extends SQLancerDBConnection> implements DatabaseProvider { +public abstract class ProviderAdapter, C>, O extends DBMSSpecificOptions>, C extends SQLancerDBConnection> + implements DatabaseProvider { private final Class globalClass; private final Class optionClass; diff --git a/src/sqlancer/SQLConnection.java b/src/sqlancer/SQLConnection.java index cb72b40d0..ae56c781f 100644 --- a/src/sqlancer/SQLConnection.java +++ b/src/sqlancer/SQLConnection.java @@ -14,7 +14,7 @@ public SQLConnection(Connection connection) { } @Override - public String getDatabaseVersion() throws SQLException{ + public String getDatabaseVersion() throws SQLException { DatabaseMetaData meta = connection.getMetaData(); return meta.getDatabaseProductVersion(); } @@ -28,7 +28,7 @@ public Statement prepareStatement(String arg) throws SQLException { return connection.prepareStatement(arg); } - public Statement createStatement() throws SQLException{ + public Statement createStatement() throws SQLException { return connection.createStatement(); } } diff --git a/src/sqlancer/SQLGlobalState.java b/src/sqlancer/SQLGlobalState.java index 0101fb577..534086472 100644 --- a/src/sqlancer/SQLGlobalState.java +++ b/src/sqlancer/SQLGlobalState.java @@ -1,5 +1,6 @@ package sqlancer; +import sqlancer.common.query.Query; import sqlancer.common.schema.AbstractSchema; /** @@ -11,5 +12,19 @@ * the schema parameter */ public abstract class SQLGlobalState, S extends AbstractSchema> - extends GlobalState { + extends GlobalState { + + @Override + protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { + boolean logExecutionTime = getOptions().logExecutionTime(); + if (success && getOptions().printSucceedingStatements()) { + System.out.println(q.getQueryString()); + } + if (logExecutionTime) { + getLogger().writeCurrent(" -- " + timer.end().asString()); + } + if (q.couldAffectSchema()) { + updateSchema(); + } + } } diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java index 91468f27e..1469ec0da 100644 --- a/src/sqlancer/SQLProviderAdapter.java +++ b/src/sqlancer/SQLProviderAdapter.java @@ -1,13 +1,14 @@ package sqlancer; +import java.util.List; + import sqlancer.common.log.LoggableFactory; import sqlancer.common.log.SQLLoggableFactory; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractRelationalTable; - -import java.util.List; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; -public abstract class SQLProviderAdapter, O extends DBMSSpecificOptions>> +public abstract class SQLProviderAdapter>, O extends DBMSSpecificOptions>> extends ProviderAdapter { public SQLProviderAdapter(Class globalClass, Class optionClass) { super(globalClass, optionClass); @@ -20,8 +21,8 @@ public LoggableFactory getLoggableFactory() { @Override protected void checkViewsAreValid(G globalState) { - List> views = globalState.getSchema().getViews(); - for (AbstractRelationalTable view : views) { + List> views = globalState.getSchema().getViews(); + for (AbstractTable view : views) { SQLQueryAdapter q = new SQLQueryAdapter("SELECT 1 FROM " + view.getName() + " LIMIT 1"); try { q.execute(globalState); diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 4477c25cb..f6408ce58 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -9,7 +9,7 @@ public class StateToReproduce { - private final List statements = new ArrayList<>(); + private final List> statements = new ArrayList<>(); private final String databaseName; @@ -59,22 +59,22 @@ public void logStatement(String queryString) { * @param query * the query to be logged */ - public void logStatement(Query query) { + public void logStatement(Query query) { if (query == null) { throw new IllegalArgumentException(); } statements.add(query); } - public List getStatements() { + public List> getStatements() { return Collections.unmodifiableList(statements); } @Deprecated public void commentStatements() { for (int i = 0; i < statements.size(); i++) { - Query statement = statements.get(i); - Query newQuery = databaseProvider.getLoggableFactory().commentOutQuery(statement); + Query statement = statements.get(i); + Query newQuery = databaseProvider.getLoggableFactory().commentOutQuery(statement); statements.set(i, newQuery); } } @@ -98,7 +98,7 @@ public OracleRunReproductionState getLocalState() { */ public class OracleRunReproductionState implements Closeable { - private final List statements = new ArrayList<>(); + private final List> statements = new ArrayList<>(); public boolean success; diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index 8b09e2a6f..26265d136 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -14,7 +14,7 @@ public class StatementExecutor, A extends Abstrac @FunctionalInterface public interface AfterQueryAction { - void notify(Query q) throws Exception; + void notify(Query q) throws Exception; } @FunctionalInterface @@ -29,6 +29,7 @@ public StatementExecutor(G globalState, A[] actions, ActionMapper mapping, this.queryConsumer = queryConsumer; } + @SuppressWarnings("unchecked") public void executeStatements() throws Exception { Randomly r = globalState.getRandomly(); int[] nrRemaining = new int[actions.length]; @@ -59,6 +60,7 @@ public void executeStatements() throws Exception { assert nextAction != null; assert nrRemaining[i] > 0; nrRemaining[i]--; + @SuppressWarnings("rawtypes") Query query = null; try { boolean success; diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 4a1f280ca..aed8c083b 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -26,9 +26,8 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresOptions; @@ -64,7 +63,7 @@ public enum Action implements AbstractAction { generateOnlyKnown)), // CLUSTER(PostgresClusterGenerator::create), // COMMIT(g -> { - Query query; + SQLQueryAdapter query; if (Randomly.getBoolean()) { query = new SQLQueryAdapter("COMMIT", true); } else if (Randomly.getBoolean()) { @@ -95,24 +94,24 @@ public enum Action implements AbstractAction { RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // COMMENT_ON(PostgresCommentGenerator::generate), // RESET((g) -> new SQLQueryAdapter("RESET ALL") /* - * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also - * configuration parameter - */), // + * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also + * configuration parameter + */), // NOTIFY(PostgresNotifyGenerator::createNotify), // LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // CREATE_VIEW(CitusViewGenerator::create); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(PostgresGlobalState state) throws Exception { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(PostgresGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -344,7 +343,8 @@ private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List { @@ -30,15 +30,15 @@ public enum Action implements AbstractAction { INSERT(ClickHouseInsertGenerator::getQuery); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(ClickHouseGlobalState state) throws Exception { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(ClickHouseGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -86,7 +86,7 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws Exception boolean success; do { String tableName = ClickHouseCommon.createTableName(i); - Query qt = ClickHouseTableGenerator.createTableStatement(tableName, globalState); + SQLQueryAdapter qt = ClickHouseTableGenerator.createTableStatement(tableName, globalState); success = globalState.executeStatement(qt); } while (!success); } diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index 86011b4da..21b39e021 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -14,9 +14,9 @@ import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ast.ClickHouseConstant; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; @@ -168,7 +168,8 @@ private static ClickHouseLancerDataType getColumnType(String typeString) { return new ClickHouseLancerDataType(typeString); } - public static class ClickHouseTable extends AbstractRelationalTable { + public static class ClickHouseTable + extends AbstractRelationalTable { public ClickHouseTable(String tableName, List columns, List indexes, boolean isView) { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 21c8cc870..d92887541 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -11,7 +11,6 @@ import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public class ClickHouseInsertGenerator extends AbstractInsertGenerator { @@ -31,11 +30,11 @@ public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { ClickHouseErrors.addExpectedExpressionErrors(errors); } - public static Query getQuery(ClickHouseGlobalState globalState) throws SQLException { + public static SQLQueryAdapter getQuery(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseInsertGenerator(globalState).get(); } - private Query get() { + private SQLQueryAdapter get() { ClickHouseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset().stream() .filter(c -> !c.isAlias() && !c.isMaterialized()).collect(Collectors.toList()); diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index 13f7a152f..280fade9b 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -11,7 +11,6 @@ import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public class ClickHouseTableGenerator { @@ -33,7 +32,8 @@ public ClickHouseTableGenerator(String tableName, ClickHouseProvider.ClickHouseG this.globalState = globalState; } - public static Query createTableStatement(String tableName, ClickHouseProvider.ClickHouseGlobalState globalState) { + public static SQLQueryAdapter createTableStatement(String tableName, + ClickHouseProvider.ClickHouseGlobalState globalState) { ClickHouseTableGenerator chTableGenerator = new ClickHouseTableGenerator(tableName, globalState); chTableGenerator.start(); ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index d0cee36f3..ccfa4b13a 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -8,12 +8,12 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; import sqlancer.MainOptions; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; @@ -31,9 +31,8 @@ import sqlancer.cockroachdb.gen.CockroachDBUpdateGenerator; import sqlancer.cockroachdb.gen.CockroachDBViewGenerator; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLQueryProvider; public class CockroachDBProvider extends SQLProviderAdapter { @@ -100,14 +99,14 @@ public enum Action { return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("must be of type")); }); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } - public Query getQuery(CockroachDBGlobalState state) throws Exception { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(CockroachDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -122,7 +121,7 @@ protected CockroachDBSchema readSchema() throws SQLException { @Override public void generateDatabase(CockroachDBGlobalState globalState) throws Exception { - QueryManager manager = globalState.getManager(); + QueryManager manager = globalState.getManager(); MainOptions options = globalState.getOptions(); List standardSettings = new ArrayList<>(); standardSettings.add("--Don't send automatic bug reports\n" @@ -150,7 +149,7 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio boolean success = false; do { try { - Query q = CockroachDBTableGenerator.generate(globalState); + SQLQueryAdapter q = CockroachDBTableGenerator.generate(globalState); success = globalState.executeStatement(q); } catch (IgnoreMeException e) { // continue trying @@ -226,7 +225,7 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio assert nextAction != null; assert nrRemaining[nextAction.ordinal()] > 0; nrRemaining[nextAction.ordinal()]--; - Query query = null; + SQLQueryAdapter query = null; try { boolean success; int nrTries = 0; diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index f51fdce88..5c21a1107 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -10,8 +10,8 @@ import sqlancer.SQLConnection; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; @@ -264,7 +264,8 @@ private static CockroachDBCompositeDataType getColumnType(String typeString) { } } - public static class CockroachDBTable extends AbstractRelationalTable { + public static class CockroachDBTable + extends AbstractRelationalTable { public CockroachDBTable(String tableName, List columns, List indexes, boolean isView) { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java index 3a052edaf..a7d6f3a30 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java @@ -9,7 +9,6 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.TableIndex; @@ -22,7 +21,7 @@ private enum Option { TABLE, INDEX, COLUMN } - public static Query comment(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter comment(CockroachDBGlobalState globalState) { StringBuilder sb = new StringBuilder("COMMENT ON "); CockroachDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); switch (Randomly.fromOptions(Option.values())) { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java index b98639651..81920cd4f 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java @@ -4,7 +4,6 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBCreateStatisticsGenerator { @@ -12,7 +11,7 @@ public final class CockroachDBCreateStatisticsGenerator { private CockroachDBCreateStatisticsGenerator() { } - public static Query create(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter create(CockroachDBGlobalState globalState) { CockroachDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); StringBuilder sb = new StringBuilder("CREATE STATISTICS s"); sb.append(Randomly.smallNumber()); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index 7df102338..23fbae389 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -7,7 +7,6 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBDeleteGenerator { @@ -15,7 +14,7 @@ public final class CockroachDBDeleteGenerator { private CockroachDBDeleteGenerator() { } - public static Query delete(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 45831c31b..004c8be63 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -6,7 +6,7 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; // https://www.cockroachlabs.com/docs/stable/create-index.html public class CockroachDBIndexGenerator extends CockroachDBGenerator { @@ -15,7 +15,7 @@ public CockroachDBIndexGenerator(CockroachDBGlobalState globalState) { super(globalState); } - public static Query create(CockroachDBGlobalState s) { + public static SQLQueryAdapter create(CockroachDBGlobalState s) { return new CockroachDBIndexGenerator(s).getQuery(); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java index 9a1496152..47de622d8 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java @@ -10,7 +10,6 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBInsertGenerator { @@ -18,7 +17,7 @@ public final class CockroachDBInsertGenerator { private CockroachDBInsertGenerator() { } - public static Query insert(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter insert(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addExpressionErrors(errors); // e.g., caused by computed columns diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java index 88d2ee906..d3679b015 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java @@ -14,7 +14,6 @@ import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBRandomQuerySynthesizer { @@ -22,7 +21,7 @@ public final class CockroachDBRandomQuerySynthesizer { private CockroachDBRandomQuerySynthesizer() { } - public static Query generate(CockroachDBGlobalState globalState, int nrColumns) { + public static SQLQueryAdapter generate(CockroachDBGlobalState globalState, int nrColumns) { CockroachDBSelect select = generateSelect(globalState, nrColumns); return new SQLQueryAdapter(CockroachDBVisitor.asString(select)); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 4690143de..5b39c5b1f 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -6,7 +6,6 @@ import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBSetClusterSettingGenerator { @@ -40,7 +39,7 @@ private enum CockroachDBClusterSetting { } } - public static Query create(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter create(CockroachDBGlobalState globalState) { CockroachDBClusterSetting s = Randomly.fromOptions(CockroachDBClusterSetting.values()); StringBuilder sb = new StringBuilder("SET CLUSTER SETTING "); sb.append(s.name); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index 2b3f549a7..e3d9bd147 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -6,7 +6,6 @@ import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBSetSessionGenerator { @@ -45,7 +44,7 @@ private enum CockroachDBSetting { } } - public static Query create(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter create(CockroachDBGlobalState globalState) { CockroachDBSetting s = Randomly.fromOptions(CockroachDBSetting.values()); StringBuilder sb = new StringBuilder("SET SESSION "); sb.append(s); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java index d299b7e30..09abbfbd0 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java @@ -4,7 +4,6 @@ import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBShowGenerator { @@ -17,7 +16,7 @@ private enum Option { DATABASES, JOBS, RANGES, LOCALITY, SEQUENCES, TRACE_FOR_SESSION } - public static Query show(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter show(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); switch (Randomly.fromOptions(Option.values())) { diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index ab3b7afc8..22a863b78 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -14,7 +14,7 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; public class CockroachDBTableGenerator extends CockroachDBGenerator { @@ -28,7 +28,7 @@ public CockroachDBTableGenerator(CockroachDBGlobalState globalState) { canAffectSchema = true; } - public static Query generate(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { return new CockroachDBTableGenerator(globalState).getQuery(); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index 9230ccd75..c5bdc2881 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -3,7 +3,6 @@ import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBTruncateGenerator { @@ -12,7 +11,7 @@ private CockroachDBTruncateGenerator() { } // https://www.cockroachlabs.com/docs/v19.2/truncate.html - public static Query truncate(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add("is interleaved by table"); errors.add("is referenced by foreign key"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index 6b18f9700..b8a96e4c7 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -10,7 +10,6 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBUpdateGenerator { @@ -18,7 +17,7 @@ public final class CockroachDBUpdateGenerator { private CockroachDBUpdateGenerator() { } - public static Query gen(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java index f8b112e6b..8754f54d1 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java @@ -4,7 +4,6 @@ import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBViewGenerator { @@ -12,7 +11,7 @@ public final class CockroachDBViewGenerator { private CockroachDBViewGenerator() { } - public static Query generate(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { int nrColumns = Randomly.smallNumber() + 1; StringBuilder sb = new StringBuilder("CREATE "); sb.append("VIEW "); diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index f0f741194..3020f3238 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -6,9 +6,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLGlobalState; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -26,7 +26,6 @@ import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; @@ -113,7 +112,7 @@ private int getOptimizedResult(CockroachDBExpression whereCondition, List globalState, Query q) throws AssertionError { + private int getCount(SQLGlobalState globalState, SQLQueryAdapter q) throws AssertionError { int count = 0; try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs == null) { diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java index b5b94e8c0..c478610c4 100644 --- a/src/sqlancer/common/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -1,7 +1,6 @@ package sqlancer.common.gen; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public abstract class AbstractGenerator { @@ -10,7 +9,7 @@ public abstract class AbstractGenerator { protected final StringBuilder sb = new StringBuilder(); protected boolean canAffectSchema; - public Query getQuery() { + public SQLQueryAdapter getQuery() { buildStatement(); return new SQLQueryAdapter(sb.toString(), errors, canAffectSchema); } diff --git a/src/sqlancer/common/log/LoggableFactory.java b/src/sqlancer/common/log/LoggableFactory.java index 2386ef10f..5a439714d 100644 --- a/src/sqlancer/common/log/LoggableFactory.java +++ b/src/sqlancer/common/log/LoggableFactory.java @@ -18,10 +18,10 @@ public Loggable createLoggable(String input) { protected abstract Loggable createLoggable(String input, String suffix); - public abstract Query getQueryForStateToReproduce(String queryString); + public abstract Query getQueryForStateToReproduce(String queryString); @Deprecated - public abstract Query commentOutQuery(Query query); + public abstract Query commentOutQuery(Query query); public Loggable getInfo(String databaseName, String databaseVersion, long seedValue) { Date date = new Date(); diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java index ce9170ed7..24a28a603 100644 --- a/src/sqlancer/common/log/SQLLoggableFactory.java +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -21,12 +21,12 @@ protected Loggable createLoggable(String input, String suffix) { } @Override - public Query getQueryForStateToReproduce(String queryString) { + public SQLQueryAdapter getQueryForStateToReproduce(String queryString) { return new SQLQueryAdapter(queryString); } @Override - public Query commentOutQuery(Query query) { + public SQLQueryAdapter commentOutQuery(Query query) { String queryString = query.getLogString(); String newQueryString = "-- " + queryString; return new SQLQueryAdapter(newQueryString); diff --git a/src/sqlancer/common/oracle/CompositeTestOracle.java b/src/sqlancer/common/oracle/CompositeTestOracle.java index 478e67702..fbb66960a 100644 --- a/src/sqlancer/common/oracle/CompositeTestOracle.java +++ b/src/sqlancer/common/oracle/CompositeTestOracle.java @@ -1,9 +1,9 @@ package sqlancer.common.oracle; -import sqlancer.GlobalState; - import java.util.List; +import sqlancer.GlobalState; + public class CompositeTestOracle implements TestOracle { private final TestOracle[] oracles; diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java index b6c83f524..734e04a86 100644 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ b/src/sqlancer/common/oracle/NoRECBase.java @@ -1,6 +1,5 @@ package sqlancer.common.oracle; -import sqlancer.GlobalState; import sqlancer.Main.StateLogger; import sqlancer.MainOptions; import sqlancer.SQLConnection; diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index d541872ae..b790e6e99 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -3,14 +3,15 @@ import java.util.ArrayList; import java.util.List; -import sqlancer.SQLGlobalState; +import sqlancer.GlobalState; import sqlancer.IgnoreMeException; +import sqlancer.SQLancerDBConnection; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractRowValue; -public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E> +public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E, C extends SQLancerDBConnection> implements TestOracle { protected final ExpectedErrors errors = new ExpectedErrors(); @@ -35,11 +36,11 @@ public PivotedQuerySynthesisBase(S globalState) { @Override public final void check() throws Exception { rectifiedPredicates.clear(); - Query pivotRowQuery = getRectifiedQuery(); + Query pivotRowQuery = getRectifiedQuery(); if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(pivotRowQuery.getQueryString()); } - Query isContainedQuery = getContainmentCheckQuery(pivotRowQuery); + Query isContainedQuery = getContainmentCheckQuery(pivotRowQuery); if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(isContainedQuery.getQueryString()); } @@ -61,7 +62,7 @@ public final void check() throws Exception { * * @throws Exception */ - private boolean containsRows(Query query) throws Exception { + private boolean containsRows(Query query) throws Exception { try (SQLancerResultSet result = query.executeAndGet(globalState)) { if (result == null) { throw new IgnoreMeException(); @@ -70,7 +71,7 @@ private boolean containsRows(Query query) throws Exception { } } - protected void reportMissingPivotRow(Query query) { + protected void reportMissingPivotRow(Query query) { globalState.getState().getLocalState().log("-- pivot row values:"); String expectedPivotRowString = pivotRow.asStringGroupedByTables(); globalState.getState().getLocalState().log(expectedPivotRowString); @@ -108,7 +109,7 @@ protected void reportMissingPivotRow(Query query) { * * @throws Exception */ - protected abstract Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception; + protected abstract Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception; /** * Obtains a rectified query (i.e., a query that is guaranteed to fetch the pivot row. This corresponds to steps 2-5 @@ -118,7 +119,7 @@ protected void reportMissingPivotRow(Query query) { * * @throws Exception */ - protected abstract Query getRectifiedQuery() throws Exception; + protected abstract Query getRectifiedQuery() throws Exception; /** * Prints the value to which the expression is expected to evaluate, and then recursively prints the subexpressions' diff --git a/src/sqlancer/common/query/Query.java b/src/sqlancer/common/query/Query.java index ec2a3b67b..44efc0084 100644 --- a/src/sqlancer/common/query/Query.java +++ b/src/sqlancer/common/query/Query.java @@ -4,7 +4,7 @@ import sqlancer.SQLancerDBConnection; import sqlancer.common.log.Loggable; -public abstract class Query, C extends SQLancerDBConnection> implements Loggable { +public abstract class Query implements Loggable { /** * Gets the query string, which is guaranteed to be terminated with a semicolon. @@ -27,7 +27,7 @@ public abstract class Query, C extends SQLancerDB */ public abstract boolean couldAffectSchema(); - public abstract boolean execute(G globalState, String... fills) throws Exception; + public abstract > boolean execute(G globalState, String... fills) throws Exception; public abstract ExpectedErrors getExpectedErrors(); @@ -36,21 +36,22 @@ public String toString() { return getQueryString(); } - public SQLancerResultSet executeAndGet(G globalState, String... fills) throws Exception { + public > SQLancerResultSet executeAndGet(G globalState, String... fills) + throws Exception { throw new AssertionError(); } - public boolean executeLogged(G globalState) throws Exception { + public > boolean executeLogged(G globalState) throws Exception { logQueryString(globalState); return execute(globalState); } - public SQLancerResultSet executeAndGetLogged(G globalState) throws Exception { + public > SQLancerResultSet executeAndGetLogged(G globalState) throws Exception { logQueryString(globalState); return executeAndGet(globalState); } - private void logQueryString(G globalState) { + private > void logQueryString(G globalState) { if (globalState.getOptions().logEachSelect()) { globalState.getLogger().writeCurrent(getQueryString()); } diff --git a/src/sqlancer/common/query/QueryProvider.java b/src/sqlancer/common/query/QueryProvider.java deleted file mode 100644 index a3a60bbbe..000000000 --- a/src/sqlancer/common/query/QueryProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.common.query; - -@FunctionalInterface -public interface QueryProvider { - Query getQuery(S globalState) throws Exception; -} diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index c80a9eb69..1e52b1c1a 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -5,11 +5,11 @@ import java.sql.SQLException; import java.sql.Statement; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; +import sqlancer.GlobalState; import sqlancer.Main; +import sqlancer.SQLConnection; -public class SQLQueryAdapter> extends Query { +public class SQLQueryAdapter extends Query { private final String query; private final ExpectedErrors expectedErrors; @@ -69,7 +69,8 @@ public String getUnterminatedQueryString() { } @Override - public boolean execute(G globalState, String... fills) throws SQLException { + public > boolean execute(G globalState, String... fills) + throws SQLException { Statement s; if (fills.length > 0) { s = globalState.getConnection().prepareStatement(fills[0]); @@ -101,7 +102,8 @@ public void checkException(Exception e) throws AssertionError { } @Override - public SQLancerResultSet executeAndGet(G globalState, String... fills) throws SQLException { + public > SQLancerResultSet executeAndGet(G globalState, String... fills) + throws SQLException { Statement s; if (fills.length > 0) { s = globalState.getConnection().prepareStatement(fills[0]); diff --git a/src/sqlancer/common/query/SQLQueryProvider.java b/src/sqlancer/common/query/SQLQueryProvider.java new file mode 100644 index 000000000..39aa0e05c --- /dev/null +++ b/src/sqlancer/common/query/SQLQueryProvider.java @@ -0,0 +1,6 @@ +package sqlancer.common.query; + +@FunctionalInterface +public interface SQLQueryProvider { + SQLQueryAdapter getQuery(S globalState) throws Exception; +} diff --git a/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java index b2692fecf..8f6a01a4b 100644 --- a/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java +++ b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java @@ -5,10 +5,10 @@ import java.sql.Statement; import java.util.function.Consumer; +import sqlancer.GlobalState; import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; -public class SQLQueryResultCheckAdapter> extends SQLQueryAdapter { +public class SQLQueryResultCheckAdapter extends SQLQueryAdapter { private final Consumer rsChecker; @@ -18,7 +18,8 @@ public SQLQueryResultCheckAdapter(String query, Consumer rsChecker) { } @Override - public boolean execute(G globalState, String... fills) throws SQLException { + public > boolean execute(G globalState, String... fills) + throws SQLException { try (Statement s = globalState.getConnection().createStatement()) { ResultSet rs = s.executeQuery(getQueryString()); rsChecker.accept(rs); diff --git a/src/sqlancer/common/schema/AbstractRelationalTable.java b/src/sqlancer/common/schema/AbstractRelationalTable.java index cd4d2287f..7f26efc1a 100644 --- a/src/sqlancer/common/schema/AbstractRelationalTable.java +++ b/src/sqlancer/common/schema/AbstractRelationalTable.java @@ -1,12 +1,12 @@ package sqlancer.common.schema; +import java.util.List; + import sqlancer.IgnoreMeException; import sqlancer.SQLGlobalState; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -import java.util.List; - public class AbstractRelationalTable, I extends TableIndex, G extends SQLGlobalState> extends AbstractTable { @@ -14,6 +14,7 @@ public AbstractRelationalTable(String name, List columns, List indexes, bo super(name, columns, indexes, isView); } + @Override public long getNrRows(G globalState) { if (rowCount == NO_ROW_COUNT_AVAILABLE) { SQLQueryAdapter q = new SQLQueryAdapter("SELECT COUNT(*) FROM " + name); @@ -32,4 +33,5 @@ public long getNrRows(G globalState) { return rowCount; } } + } diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 4755164df..c40f1a197 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -1,14 +1,14 @@ package sqlancer.common.schema; -import sqlancer.GlobalState; -import sqlancer.Randomly; - import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import sqlancer.GlobalState; +import sqlancer.Randomly; + public abstract class AbstractTable, I extends TableIndex, G extends GlobalState> - implements Comparable>{ + implements Comparable> { protected static final int NO_ROW_COUNT_AVAILABLE = -1; protected final String name; @@ -29,7 +29,7 @@ public String getName() { } @Override - public int compareTo(AbstractTable o) { + public int compareTo(AbstractTable o) { return o.getName().compareTo(getName()); } diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index a20f41d50..58a4ff4a4 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -1,20 +1,18 @@ package sqlancer.duckdb; -import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import sqlancer.AbstractAction; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLQueryProvider; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.gen.DuckDBDeleteGenerator; import sqlancer.duckdb.gen.DuckDBIndexGenerator; @@ -49,15 +47,15 @@ public enum Action implements AbstractAction { errors); }); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(DuckDBGlobalState state) throws Exception { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(DuckDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -100,7 +98,7 @@ public void generateDatabase(DuckDBGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { boolean success = false; do { - Query qt = new DuckDBTableGenerator().getQuery(globalState); + SQLQueryAdapter qt = new DuckDBTableGenerator().getQuery(globalState); success = globalState.executeStatement(qt); } while (!success); } diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 7f8a51fb0..b63f5bb96 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -9,8 +9,8 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; diff --git a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java index e1c1088f6..577ad566b 100644 --- a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -19,7 +18,7 @@ enum Action { ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN } - public static Query getQuery(DuckDBGlobalState globalState) { + public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add(" does not have a column with name \"rowid\""); errors.add("Table does not contain column rowid referenced in alter statement"); diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index da7c16c4f..5fadce30a 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -14,7 +13,7 @@ public final class DuckDBDeleteGenerator { private DuckDBDeleteGenerator() { } - public static Query generate(DuckDBGlobalState globalState) { + public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { StringBuilder sb = new StringBuilder("DELETE FROM "); ExpectedErrors errors = new ExpectedErrors(); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index f9820797c..4b6b47612 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; @@ -18,7 +17,7 @@ public final class DuckDBIndexGenerator { private DuckDBIndexGenerator() { } - public static Query getQuery(DuckDBGlobalState globalState) { + public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); sb.append("CREATE "); diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 2193109c0..1e5eb60b9 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -23,11 +22,11 @@ public DuckDBInsertGenerator(DuckDBGlobalState globalState) { this.globalState = globalState; } - public static Query getQuery(DuckDBGlobalState globalState) { + public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { return new DuckDBInsertGenerator(globalState).generate(); } - private Query generate() { + private SQLQueryAdapter generate() { sb.append("INSERT INTO "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index 3e6f12a61..b10151258 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -8,7 +8,6 @@ import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -20,7 +19,7 @@ public class DuckDBTableGenerator { - public Query getQuery(DuckDBGlobalState globalState) { + public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); String tableName = globalState.getSchema().getFreeTableName(); diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 2969bd21d..ba8c867fd 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -19,7 +18,7 @@ public final class DuckDBUpdateGenerator { private DuckDBUpdateGenerator() { } - public static Query getQuery(DuckDBGlobalState globalState) { + public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { StringBuilder sb = new StringBuilder("UPDATE "); ExpectedErrors errors = new ExpectedErrors(); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java index 495e000a3..c56f23b1c 100644 --- a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -13,7 +12,7 @@ public final class DuckDBViewGenerator { private DuckDBViewGenerator() { } - public static Query generate(DuckDBGlobalState globalState) { + public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { int nrColumns = Randomly.smallNumber() + 1; StringBuilder sb = new StringBuilder("CREATE "); sb.append("VIEW "); diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index a701a9fc3..550043615 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -1,6 +1,5 @@ package sqlancer.duckdb.test; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -17,7 +16,6 @@ import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.duckdb.DuckDBErrors; @@ -84,7 +82,7 @@ private int getSecondQuery(List> tableList, Node> tableList, Node> tableList, List columns, - Node randomWhereCondition, List> joins) throws SQLException { + private int getFirstQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, List> joins) + throws SQLException { DuckDBSelect select = new DuckDBSelect(); // select.setGroupByClause(groupBys); // DuckDBAggregate aggr = new DuckDBAggregate( diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java index 49aff01d6..58afdcb48 100644 --- a/src/sqlancer/h2/H2DeleteGenerator.java +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; @@ -12,7 +11,7 @@ public final class H2DeleteGenerator { private H2DeleteGenerator() { } - public static Query getQuery(H2GlobalState globalState) { + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { StringBuilder sb = new StringBuilder("DELETE FROM "); ExpectedErrors errors = new ExpectedErrors(); H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/h2/H2IndexGenerator.java b/src/sqlancer/h2/H2IndexGenerator.java index 18d23fb6d..b0d65e096 100644 --- a/src/sqlancer/h2/H2IndexGenerator.java +++ b/src/sqlancer/h2/H2IndexGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; @@ -15,11 +14,11 @@ public H2IndexGenerator(H2GlobalState globalState) { this.globalState = globalState; } - public static Query getQuery(H2GlobalState globalState) { + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { return new H2IndexGenerator(globalState).generate(); } - private Query generate() { + private SQLQueryAdapter generate() { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); sb.append("CREATE "); diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index cc59b9af6..cfb17846a 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; @@ -23,11 +22,11 @@ public H2InsertGenerator(H2GlobalState globalState) { gen = new H2ExpressionGenerator(globalState); } - public static Query getQuery(H2GlobalState globalState) { + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { return new H2InsertGenerator(globalState).generate(); } - private Query generate() { + private SQLQueryAdapter generate() { boolean mergeInto = false; // Randomly.getBooleanWithRatherLowProbability(); if (mergeInto) { sb.append("MERGE INTO "); diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index ce36eca2c..463add537 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -5,15 +5,14 @@ import java.sql.SQLException; import sqlancer.AbstractAction; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLQueryProvider; import sqlancer.h2.H2Provider.H2GlobalState; public class H2Provider extends SQLProviderAdapter { @@ -32,15 +31,15 @@ public enum Action implements AbstractAction { DELETE(H2DeleteGenerator::getQuery), // SET(H2SetGenerator::getQuery); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(H2GlobalState state) throws Exception { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(H2GlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -81,7 +80,7 @@ public void generateDatabase(H2GlobalState globalState) throws Exception { boolean success = false; for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { do { - Query qt = new H2TableGenerator().getQuery(globalState); + SQLQueryAdapter qt = new H2TableGenerator().getQuery(globalState); success = globalState.executeStatement(qt); } while (!success); } diff --git a/src/sqlancer/h2/H2Schema.java b/src/sqlancer/h2/H2Schema.java index 506a67864..7a64ef56e 100644 --- a/src/sqlancer/h2/H2Schema.java +++ b/src/sqlancer/h2/H2Schema.java @@ -9,8 +9,8 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; diff --git a/src/sqlancer/h2/H2SetGenerator.java b/src/sqlancer/h2/H2SetGenerator.java index af14f016a..d157d9ce7 100644 --- a/src/sqlancer/h2/H2SetGenerator.java +++ b/src/sqlancer/h2/H2SetGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; @@ -49,7 +48,7 @@ public static Option getRandom() { } } - public static Query getQuery(H2GlobalState globalState) { + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { StringBuilder sb = new StringBuilder(); Option o = Option.getRandom(); sb.append("SET "); diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java index d28bdad66..1faa0bdb7 100644 --- a/src/sqlancer/h2/H2TableGenerator.java +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -7,7 +7,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; @@ -16,7 +15,7 @@ public class H2TableGenerator { - public Query getQuery(H2GlobalState globalState) { + public SQLQueryAdapter getQuery(H2GlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add("already exists"); StringBuilder sb = new StringBuilder("CREATE TABLE " + globalState.getSchema().getFreeTableName() + "("); diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index 965483e28..0dfc15938 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; @@ -15,7 +14,7 @@ public final class H2UpdateGenerator { private H2UpdateGenerator() { } - public static Query getQuery(H2GlobalState globalState) { + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { StringBuilder sb = new StringBuilder("UPDATE "); ExpectedErrors errors = new ExpectedErrors(); H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/h2/H2ViewGenerator.java b/src/sqlancer/h2/H2ViewGenerator.java index 2c56f5549..78b174025 100644 --- a/src/sqlancer/h2/H2ViewGenerator.java +++ b/src/sqlancer/h2/H2ViewGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; @@ -11,7 +10,7 @@ public final class H2ViewGenerator { private H2ViewGenerator() { } - public static Query getQuery(H2GlobalState globalState) { + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { int nrColumns = Randomly.smallNumber() + 1; StringBuilder sb = new StringBuilder("CREATE "); sb.append("VIEW "); diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index bc70f266a..bb514256f 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -7,13 +7,13 @@ import java.util.ArrayList; import java.util.List; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; -import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.gen.MariaDBIndexGenerator; import sqlancer.mariadb.gen.MariaDBInsertGenerator; @@ -51,7 +51,7 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = MariaDBTableGenerator.generate(tableName, globalState.getRandomly(), + SQLQueryAdapter createTable = MariaDBTableGenerator.generate(tableName, globalState.getRandomly(), globalState.getSchema()); globalState.executeStatement(createTable); } @@ -103,7 +103,7 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { assert nextAction != null; assert nrRemaining[nextAction.ordinal()] > 0; nrRemaining[nextAction.ordinal()]--; - Query query; + SQLQueryAdapter query; try { switch (nextAction) { case CHECKSUM: diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index 2acf3d79e..ed432e8da 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -12,9 +12,8 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; -import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.TableIndex; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; diff --git a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java index a39ec3270..062bf509c 100644 --- a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java @@ -1,6 +1,5 @@ package sqlancer.mariadb.gen; -import java.sql.Connection; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 4dc7ac4d1..d98c71ce2 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; @@ -16,7 +15,7 @@ public final class MariaDBIndexGenerator { private MariaDBIndexGenerator() { } - public static Query generate(MariaDBSchema s) { + public static SQLQueryAdapter generate(MariaDBSchema s) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE "); errors.add("Key/Index cannot be defined on a virtual generated column"); diff --git a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java index c6346c569..5fa906727 100644 --- a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; @@ -14,7 +13,7 @@ public final class MariaDBInsertGenerator { private MariaDBInsertGenerator() { } - public static Query insert(MariaDBSchema s, Randomly r) { + public static SQLQueryAdapter insert(MariaDBSchema s, Randomly r) { MariaDBTable randomTable = s.getRandomTable(); StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO "); diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index 4ecc3e02d..d437d4537 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -8,7 +8,6 @@ import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public class MariaDBSetGenerator { @@ -24,7 +23,7 @@ public MariaDBSetGenerator(Randomly r, MainOptions options) { this.isSingleThreaded = options.getNumberConcurrentThreads() == 1; } - public static Query set(Randomly r, MainOptions options) { + public static SQLQueryAdapter set(Randomly r, MainOptions options) { return new MariaDBSetGenerator(r, options).get(); } @@ -163,7 +162,7 @@ public Scope[] getScopes() { } } - private Query get() { + private SQLQueryAdapter get() { sb.append("SET "); Action a; if (isSingleThreaded) { diff --git a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java index 5a2f4f5c4..bff83e1e0 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java @@ -6,7 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryResultCheckAdapter; import sqlancer.mariadb.MariaDBSchema; @@ -17,7 +16,7 @@ public final class MariaDBTableAdminCommandGenerator { private MariaDBTableAdminCommandGenerator() { } - public static Query checksumTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter checksumTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "CHECKSUM TABLE"); if (Randomly.getBoolean()) { sb.append(" "); @@ -26,7 +25,7 @@ public static Query checksumTable(MariaDBSchema newSchema) { return new SQLQueryAdapter(sb.toString()); } - public static Query repairTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter repairTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "REPAIR TABLE"); if (Randomly.getBoolean()) { List subset = Randomly.nonEmptySubset("QUICK", "EXTENDED"); // , "USE_FRM" @@ -37,12 +36,12 @@ public static Query repairTable(MariaDBSchema newSchema) { s -> s.equals("OK") || s.equals("The storage engine for the table doesn't support repair")); } - public static Query analyzeTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter analyzeTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "ANALYZE TABLE"); return checkForMsgText(sb, s -> s.equals("OK") || s.equals("Table is already up to date")); } - public static Query checkTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter checkTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "CHECK TABLE"); if (Randomly.getBoolean()) { List subset = Randomly.nonEmptySubset("FOR UPGRADE", "QUICK", "FAST", "MEDIUM", "EXTENDED", @@ -53,7 +52,7 @@ public static Query checkTable(MariaDBSchema newSchema) { return checkForMsgText(sb, s -> s.equals("OK") || s.equals("Table is already up to date")); } - public static Query optimizeTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter optimizeTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "OPTIMIZE TABLE"); MariaDBCommon.addWaitClause(sb); return checkForMsgText(sb, @@ -61,8 +60,8 @@ public static Query optimizeTable(MariaDBSchema newSchema) { || s.contentEquals("Table is already up to date")); } - private static Query checkForMsgText(StringBuilder sb, Function checker) { - return new SQLQueryResultCheckAdapter<>(sb.toString(), rs -> { + private static SQLQueryAdapter checkForMsgText(StringBuilder sb, Function checker) { + return new SQLQueryResultCheckAdapter(sb.toString(), rs -> { try { while (rs.next()) { String s = rs.getString("Msg_text"); diff --git a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java index 128eb61c9..033e08c3d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBBugs; import sqlancer.mariadb.MariaDBSchema; @@ -31,11 +30,11 @@ public MariaDBTableGenerator(String tableName, Randomly r, MariaDBSchema newSche this.r = r; } - public static Query generate(String tableName, Randomly r, MariaDBSchema newSchema) { + public static SQLQueryAdapter generate(String tableName, Randomly r, MariaDBSchema newSchema) { return new MariaDBTableGenerator(tableName, r, newSchema).gen(); } - private Query gen() { + private SQLQueryAdapter gen() { if (Randomly.getBoolean() || s.getDatabaseTables().isEmpty()) { newTable(); } else { diff --git a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java index 4913b48b3..04e567444 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java @@ -1,6 +1,5 @@ package sqlancer.mariadb.gen; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBSchema; @@ -9,7 +8,7 @@ public final class MariaDBTruncateGenerator { private MariaDBTruncateGenerator() { } - public static Query truncate(MariaDBSchema s) { + public static SQLQueryAdapter truncate(MariaDBSchema s) { StringBuilder sb = new StringBuilder("TRUNCATE "); sb.append(s.getRandomTable().getName()); sb.append(" "); diff --git a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java index db01383b5..8a9127961 100644 --- a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; @@ -14,7 +13,7 @@ public final class MariaDBUpdateGenerator { private MariaDBUpdateGenerator() { } - public static Query update(MariaDBSchema s, Randomly r) { + public static SQLQueryAdapter update(MariaDBSchema s, Randomly r) { MariaDBTable randomTable = s.getRandomTable(); StringBuilder sb = new StringBuilder("UPDATE "); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 0bbd1f915..041ac440b 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -3,7 +3,6 @@ import java.sql.SQLException; -import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index a1d86a55f..114150369 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -11,9 +11,8 @@ import sqlancer.SQLConnection; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLQueryProvider; import sqlancer.mysql.gen.MySQLAlterTable; import sqlancer.mysql.gen.MySQLDeleteGenerator; import sqlancer.mysql.gen.MySQLDropIndex; @@ -60,15 +59,15 @@ enum Action implements AbstractAction { DELETE(MySQLDeleteGenerator::delete), // DROP_INDEX(MySQLDropIndex::generate); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(MySQLGlobalState globalState) throws Exception { - return queryProvider.getQuery(globalState); + public SQLQueryAdapter getQuery(MySQLGlobalState globalState) throws Exception { + return sqlQueryProvider.getQuery(globalState); } } @@ -135,7 +134,7 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { public void generateDatabase(MySQLGlobalState globalState) throws Exception { while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = MySQLTableGenerator.generate(globalState, tableName); + SQLQueryAdapter createTable = MySQLTableGenerator.generate(globalState, tableName); globalState.executeStatement(createTable); } diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index aef4c61c9..0384f34df 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -12,9 +12,9 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; diff --git a/src/sqlancer/mysql/gen/MySQLAlterTable.java b/src/sqlancer/mysql/gen/MySQLAlterTable.java index 5425d26b9..cb5db6bde 100644 --- a/src/sqlancer/mysql/gen/MySQLAlterTable.java +++ b/src/sqlancer/mysql/gen/MySQLAlterTable.java @@ -7,7 +7,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; @@ -25,7 +24,7 @@ public MySQLAlterTable(MySQLSchema newSchema) { this.schema = newSchema; } - public static Query create(MySQLGlobalState globalState) { + public static SQLQueryAdapter create(MySQLGlobalState globalState) { return new MySQLAlterTable(globalState.getSchema()).create(); } @@ -61,7 +60,7 @@ PACK_KEYS, RENAME("doesn't exist", "already exists"), /* WITH_WITHOUT_VALIDATION } - private Query create() { + private SQLQueryAdapter create() { ExpectedErrors errors = ExpectedErrors.from("does not support the create option", "doesn't have this option", "is not supported for this operation", "Data truncation", "Specified key was too long"); errors.add("Data truncated for functional index "); diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index 49fbe7b87..f12c23a72 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; @@ -20,11 +19,11 @@ public MySQLDeleteGenerator(MySQLGlobalState globalState) { this.globalState = globalState; } - public static Query delete(MySQLGlobalState globalState) { + public static SQLQueryAdapter delete(MySQLGlobalState globalState) { return new MySQLDeleteGenerator(globalState).generate(); } - private Query generate() { + private SQLQueryAdapter generate() { MySQLTable randomTable = globalState.getSchema().getRandomTable(); MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()); ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/mysql/gen/MySQLDropIndex.java b/src/sqlancer/mysql/gen/MySQLDropIndex.java index ec1ee7cb5..4e56c9a30 100644 --- a/src/sqlancer/mysql/gen/MySQLDropIndex.java +++ b/src/sqlancer/mysql/gen/MySQLDropIndex.java @@ -3,7 +3,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -25,7 +24,7 @@ private MySQLDropIndex() { // lock_option: // LOCK [=] {DEFAULT|NONE|SHARED|EXCLUSIVE} - public static Query generate(MySQLGlobalState globalState) { + public static SQLQueryAdapter generate(MySQLGlobalState globalState) { MySQLTable table = globalState.getSchema().getRandomTable(); if (!table.hasIndexes()) { throw new IgnoreMeException(); diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index c3112c9ff..496600905 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; @@ -25,7 +24,7 @@ public MySQLInsertGenerator(MySQLGlobalState globalState) { table = globalState.getSchema().getRandomTable(); } - public static Query insertRow(MySQLGlobalState globalState) throws SQLException { + public static SQLQueryAdapter insertRow(MySQLGlobalState globalState) throws SQLException { if (Randomly.getBoolean()) { return new MySQLInsertGenerator(globalState).generateInsert(); } else { @@ -33,7 +32,7 @@ public static Query insertRow(MySQLGlobalState globalState) throws SQLException } } - private Query generateReplace() { + private SQLQueryAdapter generateReplace() { sb.append("REPLACE"); if (Randomly.getBoolean()) { sb.append(" "); @@ -43,7 +42,7 @@ private Query generateReplace() { } - private Query generateInsert() { + private SQLQueryAdapter generateInsert() { sb.append("INSERT"); if (Randomly.getBoolean()) { sb.append(" "); @@ -55,7 +54,7 @@ private Query generateInsert() { return generateInto(); } - private Query generateInto() { + private SQLQueryAdapter generateInto() { sb.append(" INTO "); sb.append(table.getName()); List columns = table.getRandomNonEmptyColumnSubset(); diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index f14a361d4..2db20fee9 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -6,7 +6,6 @@ import sqlancer.MainOptions; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; @@ -23,7 +22,7 @@ public MySQLSetGenerator(Randomly r, MainOptions options) { this.isSingleThreaded = options.getNumberConcurrentThreads() == 1; } - public static Query set(MySQLGlobalState globalState) { + public static SQLQueryAdapter set(MySQLGlobalState globalState) { return new MySQLSetGenerator(globalState.getRandomly(), globalState.getOptions()).get(); } @@ -160,7 +159,7 @@ public Scope[] getScopes() { } } - private Query get() { + private SQLQueryAdapter get() { sb.append("SET "); Action a; if (isSingleThreaded) { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 5b0f85a39..ffa85984e 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -9,7 +9,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; @@ -41,11 +40,11 @@ public MySQLTableGenerator(MySQLGlobalState globalState, String tableName) { this.globalState = globalState; } - public static Query generate(MySQLGlobalState globalState, String tableName) { + public static SQLQueryAdapter generate(MySQLGlobalState globalState, String tableName) { return new MySQLTableGenerator(globalState, tableName).create(); } - private Query create() { + private SQLQueryAdapter create() { ExpectedErrors errors = new ExpectedErrors(); sb.append("CREATE"); diff --git a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java index a06428f97..58b6958c1 100644 --- a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java @@ -1,7 +1,6 @@ package sqlancer.mysql.gen; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; @@ -10,7 +9,7 @@ public final class MySQLTruncateTableGenerator { private MySQLTruncateTableGenerator() { } - public static Query generate(MySQLGlobalState globalState) { + public static SQLQueryAdapter generate(MySQLGlobalState globalState) { StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); sb.append(globalState.getSchema().getRandomTable().getName()); return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("doesn't have this option")); diff --git a/src/sqlancer/mysql/gen/admin/MySQLFlush.java b/src/sqlancer/mysql/gen/admin/MySQLFlush.java index ad9be8601..2558dbc7a 100644 --- a/src/sqlancer/mysql/gen/admin/MySQLFlush.java +++ b/src/sqlancer/mysql/gen/admin/MySQLFlush.java @@ -4,7 +4,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -21,11 +20,11 @@ public MySQLFlush(List tables) { this.tables = tables; } - public static Query create(MySQLGlobalState globalState) { + public static SQLQueryAdapter create(MySQLGlobalState globalState) { return new MySQLFlush(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty()).generate(); } - private Query generate() { + private SQLQueryAdapter generate() { sb.append("FLUSH"); if (Randomly.getBoolean()) { sb.append(" "); diff --git a/src/sqlancer/mysql/gen/admin/MySQLReset.java b/src/sqlancer/mysql/gen/admin/MySQLReset.java index c0fdab986..dedb34a21 100644 --- a/src/sqlancer/mysql/gen/admin/MySQLReset.java +++ b/src/sqlancer/mysql/gen/admin/MySQLReset.java @@ -3,7 +3,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; @@ -12,7 +11,7 @@ public final class MySQLReset { private MySQLReset() { } - public static Query create(MySQLGlobalState globalState) { + public static SQLQueryAdapter create(MySQLGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("RESET "); sb.append(Randomly.nonEmptySubset("MASTER", "SLAVE").stream().collect(Collectors.joining(", "))); diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index 6772ce95d..408463f24 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; @@ -32,11 +31,11 @@ public MySQLIndexGenerator(MySQLSchema schema, Randomly r, MySQLGlobalState glob this.globalState = globalState; } - public static Query create(MySQLGlobalState globalState) { + public static SQLQueryAdapter create(MySQLGlobalState globalState) { return new MySQLIndexGenerator(globalState.getSchema(), globalState.getRandomly(), globalState).create(); } - public Query create() { + public SQLQueryAdapter create() { ExpectedErrors errors = new ExpectedErrors(); MySQLErrors.addExpressionErrors(errors); sb.append("CREATE "); diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java index 3d4cae59a..7714031ff 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java @@ -4,7 +4,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; @@ -24,12 +23,12 @@ public MySQLAnalyzeTable(List tables, Randomly r) { this.r = r; } - public static Query analyze(MySQLGlobalState globalState) { + public static SQLQueryAdapter analyze(MySQLGlobalState globalState) { return new MySQLAnalyzeTable(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty(), globalState.getRandomly()).generate(); } - private Query generate() { + private SQLQueryAdapter generate() { sb.append("ANALYZE "); if (Randomly.getBoolean()) { sb.append(Randomly.fromOptions("NO_WRITE_TO_BINLOG", "LOCAL")); diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java index e1e4d0f34..5858343bf 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java @@ -4,7 +4,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -21,7 +20,7 @@ public MySQLCheckTable(List tables) { this.tables = tables; } - public static Query check(MySQLGlobalState globalState) { + public static SQLQueryAdapter check(MySQLGlobalState globalState) { return new MySQLCheckTable(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty()).generate(); } @@ -35,7 +34,7 @@ public static Query check(MySQLGlobalState globalState) { // | EXTENDED // | CHANGED // } - private Query generate() { + private SQLQueryAdapter generate() { sb.append("CHECK TABLE "); sb.append(tables.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); sb.append(" "); diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java index c3e96adcd..4196405a5 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java @@ -4,7 +4,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -21,12 +20,12 @@ public MySQLChecksum(List tables) { this.tables = tables; } - public static Query checksum(MySQLGlobalState globalState) { + public static SQLQueryAdapter checksum(MySQLGlobalState globalState) { return new MySQLChecksum(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty()).checksum(); } // CHECKSUM TABLE tbl_name [, tbl_name] ... [QUICK | EXTENDED] - private Query checksum() { + private SQLQueryAdapter checksum() { sb.append("CHECKSUM TABLE "); sb.append(tables.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java index f3b3895a7..cbe57c46d 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java @@ -4,7 +4,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -21,13 +20,13 @@ public MySQLOptimize(List tables) { this.tables = tables; } - public static Query optimize(MySQLGlobalState globalState) { + public static SQLQueryAdapter optimize(MySQLGlobalState globalState) { return new MySQLOptimize(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty()).optimize(); } // OPTIMIZE [NO_WRITE_TO_BINLOG | LOCAL] // TABLE tbl_name [, tbl_name] ... - private Query optimize() { + private SQLQueryAdapter optimize() { sb.append("OPTIMIZE"); if (Randomly.getBoolean()) { sb.append(" "); diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java index a546301af..4ce4db692 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java @@ -4,7 +4,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -22,7 +21,7 @@ public MySQLRepair(List tables) { this.tables = tables; } - public static Query repair(MySQLGlobalState globalState) { + public static SQLQueryAdapter repair(MySQLGlobalState globalState) { List tables = globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty(); for (MySQLTable table : tables) { // see https://bugs.mysql.com/bug.php?id=95820 @@ -36,7 +35,7 @@ public static Query repair(MySQLGlobalState globalState) { // REPAIR [NO_WRITE_TO_BINLOG | LOCAL] // TABLE tbl_name [, tbl_name] ... // [QUICK] [EXTENDED] [USE_FRM] - private Query repair() { + private SQLQueryAdapter repair() { sb.append("REPAIR"); if (Randomly.getBoolean()) { sb.append(" "); diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 0c122d41e..6e50fbb33 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -6,6 +6,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; @@ -28,7 +29,7 @@ import sqlancer.mysql.gen.MySQLExpressionGenerator; public class MySQLPivotedQuerySynthesisOracle - extends PivotedQuerySynthesisBase { + extends PivotedQuerySynthesisBase { private List fetchColumns; private List columns; @@ -40,7 +41,7 @@ public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQL } @Override - public Query getRectifiedQuery() throws SQLException { + public Query getRectifiedQuery() throws SQLException { MySQLTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); List tables = randomFromTables.getTables(); @@ -114,7 +115,7 @@ private MySQLExpression generateRectifiedExpression(List columns, M } @Override - protected Query getContainmentCheckQuery(Query query) throws SQLException { + protected Query getContainmentCheckQuery(Query query) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions sb.append(query.getUnterminatedQueryString()); diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 5b5d60d0e..f36536975 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -10,9 +10,9 @@ import java.util.List; import java.util.Map; +import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; -import sqlancer.Randomly; public class PostgresGlobalState extends SQLGlobalState { diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 5cdb49977..f887e11ff 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -14,9 +14,8 @@ import sqlancer.SQLConnection; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; import sqlancer.postgres.gen.PostgresAlterTableGenerator; @@ -73,7 +72,7 @@ public enum Action implements AbstractAction { generateOnlyKnown)), // CLUSTER(PostgresClusterGenerator::create), // COMMIT(g -> { - Query query; + SQLQueryAdapter query; if (Randomly.getBoolean()) { query = new SQLQueryAdapter("COMMIT", true); } else if (Randomly.getBoolean()) { @@ -104,24 +103,24 @@ public enum Action implements AbstractAction { RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // COMMENT_ON(PostgresCommentGenerator::generate), // RESET((g) -> new SQLQueryAdapter("RESET ALL") /* - * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also - * configuration parameter - */), // + * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also + * configuration parameter + */), // NOTIFY(PostgresNotifyGenerator::createNotify), // LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // CREATE_VIEW(PostgresViewGenerator::create); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(PostgresGlobalState state) throws Exception { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(PostgresGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -270,7 +269,7 @@ protected void createTables(PostgresGlobalState globalState, int numTables) thro while (globalState.getSchema().getDatabaseTables().size() < numTables) { try { String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), + SQLQueryAdapter createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), generateOnlyKnown, globalState); globalState.executeStatement(createTable); } catch (IgnoreMeException e) { diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 2a52fb645..6b6fb0c06 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -15,9 +15,9 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; @@ -152,7 +152,8 @@ protected PostgresRowValue(PostgresTables tables, Map { + public static class PostgresTable + extends AbstractRelationalTable { public enum TableType { STANDARD, TEMPORARY diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 3859a7f5c..ac2964661 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -5,7 +5,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -61,7 +60,8 @@ public PostgresAlterTableGenerator(PostgresTable randomTable, PostgresGlobalStat this.opClasses = globalState.getOpClasses(); } - public static Query create(PostgresTable randomTable, PostgresGlobalState globalState, boolean generateOnlyKnown) { + public static SQLQueryAdapter create(PostgresTable randomTable, PostgresGlobalState globalState, + boolean generateOnlyKnown) { return new PostgresAlterTableGenerator(randomTable, globalState, generateOnlyKnown).generate(); } @@ -115,7 +115,7 @@ public List getActions(ExpectedErrors errors) { return action; } - public Query generate() { + public SQLQueryAdapter generate() { ExpectedErrors errors = new ExpectedErrors(); int i = 0; List action = getActions(errors); diff --git a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java index 46408c960..3acbfde10 100644 --- a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -14,7 +13,7 @@ public final class PostgresAnalyzeGenerator { private PostgresAnalyzeGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(); StringBuilder sb = new StringBuilder("ANALYZE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java index 1b1d6d456..9d966e8e5 100644 --- a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -12,7 +11,7 @@ public final class PostgresClusterGenerator { private PostgresClusterGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add("there is no previously clustered index for table"); errors.add("cannot cluster a partitioned table"); diff --git a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java index f21826760..343242ce9 100644 --- a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java @@ -2,7 +2,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -19,7 +18,7 @@ private enum Action { INDEX, COLUMN, STATISTICS, TABLE } - public static Query generate(PostgresGlobalState globalState) { + public static SQLQueryAdapter generate(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("COMMENT ON "); Action type = Randomly.fromOptions(Action.values()); diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 132e5ae5e..f827331c0 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -14,7 +13,7 @@ public final class PostgresDeleteGenerator { private PostgresDeleteGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); diff --git a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java index 1793387e0..c977fa272 100644 --- a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; @@ -12,7 +11,7 @@ public final class PostgresDiscardGenerator { private PostgresDiscardGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("DISCARD "); // prevent that DISCARD discards all tables (if they are TEMP tables) diff --git a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java index eaf43dd1b..6a6aa9473 100644 --- a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresIndex; @@ -15,7 +14,7 @@ public final class PostgresDropIndexGenerator { private PostgresDropIndexGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { List indexes = globalState.getSchema().getRandomTable().getIndexes(); StringBuilder sb = new StringBuilder(); sb.append("DROP INDEX "); diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index ba6ed17cf..43e1825f5 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -25,7 +24,7 @@ public enum IndexType { BTREE, HASH, GIST, GIN } - public static Query generate(PostgresGlobalState globalState) { + public static SQLQueryAdapter generate(PostgresGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); sb.append("CREATE"); diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 8c6ceca46..f23387977 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -18,7 +17,7 @@ public final class PostgresInsertGenerator { private PostgresInsertGenerator() { } - public static Query insert(PostgresGlobalState globalState) { + public static SQLQueryAdapter insert(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); ExpectedErrors errors = new ExpectedErrors(); errors.add("cannot insert into column"); diff --git a/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java b/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java index c7ecfad2b..ef044dc3d 100644 --- a/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java @@ -1,7 +1,6 @@ package sqlancer.postgres.gen; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -14,7 +13,7 @@ private static String getChannel() { return Randomly.fromOptions("asdf", "test"); } - public static Query createNotify(PostgresGlobalState globalState) { + public static SQLQueryAdapter createNotify(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("NOTIFY "); sb.append(getChannel()); @@ -27,14 +26,14 @@ public static Query createNotify(PostgresGlobalState globalState) { return new SQLQueryAdapter(sb.toString()); } - public static Query createListen() { + public static SQLQueryAdapter createListen() { StringBuilder sb = new StringBuilder(); sb.append("LISTEN "); sb.append(getChannel()); return new SQLQueryAdapter(sb.toString()); } - public static Query createUnlisten() { + public static SQLQueryAdapter createUnlisten() { StringBuilder sb = new StringBuilder(); sb.append("UNLISTEN "); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index 34081e140..9bb5ec5cd 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -6,7 +6,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresIndex; @@ -20,7 +19,7 @@ private enum Scope { INDEX, TABLE, DATABASE; } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add("could not create unique index"); // CONCURRENT INDEX StringBuilder sb = new StringBuilder(); diff --git a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java index e5f5b342f..281e89e3d 100644 --- a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -11,7 +10,7 @@ public final class PostgresSequenceGenerator { private PostgresSequenceGenerator() { } - public static Query createSequence(PostgresGlobalState globalState) { + public static SQLQueryAdapter createSequence(PostgresGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 27034ed1a..bfc0bbc04 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -5,7 +5,6 @@ import java.util.function.Function; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -128,7 +127,7 @@ private enum ConfigurationOption { } } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); ArrayList options = new ArrayList<>(Arrays.asList(ConfigurationOption.values())); options.remove(ConfigurationOption.DEFAULT_WITH_OIDS); diff --git a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java index 9b7f88453..e27f82800 100644 --- a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java @@ -6,7 +6,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -18,7 +17,7 @@ public final class PostgresStatisticsGenerator { private PostgresStatisticsGenerator() { } - public static Query insert(PostgresGlobalState globalState) { + public static SQLQueryAdapter insert(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("CREATE STATISTICS "); if (Randomly.getBoolean()) { @@ -48,7 +47,7 @@ public static Query insert(PostgresGlobalState globalState) { true); } - public static Query remove(PostgresGlobalState globalState) { + public static SQLQueryAdapter remove(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder("DROP STATISTICS "); PostgresTable randomTable = globalState.getSchema().getRandomTable(); List statistics = randomTable.getStatistics(); diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 861e8ac11..ddf9753de 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; @@ -59,12 +58,12 @@ public PostgresTableGenerator(String tableName, PostgresSchema newSchema, boolea PostgresCommon.addCommonTableErrors(errors); } - public static Query generate(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, + public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, PostgresGlobalState globalState) { return new PostgresTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); } - private Query generate() { + private SQLQueryAdapter generate() { columnCanHavePrimaryKey = true; sb.append("CREATE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java index 4fa67a3b1..d58f37d3f 100644 --- a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; public final class PostgresTransactionGenerator { @@ -10,7 +9,7 @@ public final class PostgresTransactionGenerator { private PostgresTransactionGenerator() { } - public static Query executeBegin() { + public static SQLQueryAdapter executeBegin() { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("BEGIN"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java index 774e87916..197668ac5 100644 --- a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -13,7 +12,7 @@ public final class PostgresTruncateGenerator { private PostgresTruncateGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("TRUNCATE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 8b0d331c2..1122b2b5a 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -18,7 +17,7 @@ public final class PostgresUpdateGenerator { private PostgresUpdateGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { PostgresTable randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); StringBuilder sb = new StringBuilder(); sb.append("UPDATE "); diff --git a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java index da0ef9930..ef32db45a 100644 --- a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -16,7 +15,7 @@ public final class PostgresVacuumGenerator { private PostgresVacuumGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(); StringBuilder sb = new StringBuilder("VACUUM "); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index 1366c7f41..8dee2b57a 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresVisitor; @@ -14,7 +13,7 @@ public final class PostgresViewGenerator { private PostgresViewGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE"); boolean materialized; diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 61cf44312..995eb207f 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -12,7 +12,6 @@ import sqlancer.Randomly; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresCompoundDataType; @@ -120,7 +119,7 @@ private int getUnoptimizedQueryCount(List fromTables, Postgr logger.writeCurrent(unoptimizedQueryString); } errors.add("canceling statement due to statement timeout"); - Query q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { rs = q.executeAndGet(state); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 2dda07258..778a7ca90 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -6,6 +6,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; @@ -26,7 +27,7 @@ import sqlancer.postgres.gen.PostgresExpressionGenerator; public class PostgresPivotedQuerySynthesisOracle - extends PivotedQuerySynthesisBase { + extends PivotedQuerySynthesisBase { private List fetchColumns; @@ -37,7 +38,7 @@ public PostgresPivotedQuerySynthesisOracle(PostgresGlobalState globalState) thro } @Override - public Query getRectifiedQuery() throws SQLException { + public SQLQueryAdapter getRectifiedQuery() throws SQLException { PostgresTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); PostgresSelect selectStatement = new PostgresSelect(); @@ -118,7 +119,7 @@ private PostgresExpression generateRectifiedExpression(List colu } @Override - protected Query getContainmentCheckQuery(Query query) throws SQLException { + protected Query getContainmentCheckQuery(Query query) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions sb.append(query.getUnterminatedQueryString()); diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 46c05c9a7..a01fcb0fb 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -1,7 +1,6 @@ package sqlancer.sqlite3; import java.io.File; -import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; @@ -9,16 +8,15 @@ import java.util.List; import sqlancer.AbstractAction; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLQueryProvider; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; @@ -101,15 +99,15 @@ public boolean canBeRetried() { CREATE_TRIGGER(SQLite3CreateTriggerGenerator::create), // MANIPULATE_STAT_TABLE(SQLite3StatTableGenerator::getQuery); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(SQLite3GlobalState state) throws Exception { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(SQLite3GlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -195,7 +193,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws Exception { int i = 0; do { - Query tableQuery = getTableQuery(globalState, i++); + SQLQueryAdapter tableQuery = getTableQuery(globalState, i++); globalState.executeStatement(tableQuery); } while (globalState.getSchema().getDatabaseTables().size() < nrTablesToCreate); assert globalState.getSchema().getTables().getTables().size() == nrTablesToCreate; @@ -213,7 +211,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws Exception { }); se.executeStatements(); - Query query = SQLite3TransactionGenerator.generateCommit(globalState); + SQLQueryAdapter query = SQLite3TransactionGenerator.generateCommit(globalState); globalState.executeStatement(query); // also do an abort for DEFERRABLE INITIALLY DEFERRED @@ -224,7 +222,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws Exception { private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws Exception { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { - Query q = new SQLQueryAdapter("SELECT * FROM " + table.getName(), + SQLQueryAdapter q = new SQLQueryAdapter("SELECT * FROM " + table.getName(), ExpectedErrors.from("needs an odd number of arguments", " requires an even number of arguments", "generated column loop", "integer overflow", "malformed JSON", "JSON cannot hold BLOB values", "JSON path error", "labels must be TEXT", @@ -235,8 +233,8 @@ private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) } } - private Query getTableQuery(SQLite3GlobalState globalState, int i) throws AssertionError { - Query tableQuery; + private SQLQueryAdapter getTableQuery(SQLite3GlobalState globalState, int i) throws AssertionError { + SQLQueryAdapter tableQuery; List options = new ArrayList<>(Arrays.asList(TableType.values())); if (!globalState.getDmbsSpecificOptions().testFts) { options.remove(TableType.FTS); diff --git a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java index 4c0ac67cb..5d04d25e1 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -15,7 +14,7 @@ private enum AnalyzeTarget { SCHEMA, TABLE, INDEX, SQL_MASTER } - public static Query generateAnalyze(SQLite3GlobalState globalState) { + public static SQLQueryAdapter generateAnalyze(SQLite3GlobalState globalState) { final StringBuilder sb = new StringBuilder("ANALYZE"); if (Randomly.getBoolean()) { sb.append(" "); diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index 702746e15..ba1637d81 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; @@ -15,7 +14,7 @@ public final class SQLite3CreateVirtualRtreeTabelGenerator { private SQLite3CreateVirtualRtreeTabelGenerator() { } - public static Query createTableStatement(String rTreeTableName, SQLite3GlobalState globalState) { + public static SQLQueryAdapter createTableStatement(String rTreeTableName, SQLite3GlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); List columns = new ArrayList<>(); StringBuilder sb = new StringBuilder("CREATE VIRTUAL TABLE "); diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java index 239ecfc7a..cfc7aa838 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java @@ -1,7 +1,6 @@ package sqlancer.sqlite3.gen; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.sqlite3.SQLite3Provider.Action; @@ -12,7 +11,7 @@ public final class SQLite3ExplainGenerator { private SQLite3ExplainGenerator() { } - public static Query explain(SQLite3GlobalState globalState) throws Exception { + public static SQLQueryAdapter explain(SQLite3GlobalState globalState) throws Exception { StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN "); if (Randomly.getBoolean()) { @@ -22,7 +21,7 @@ public static Query explain(SQLite3GlobalState globalState) throws Exception { do { action = Randomly.fromOptions(SQLite3Provider.Action.values()); } while (action == Action.EXPLAIN); - Query query = action.getQuery(globalState); + SQLQueryAdapter query = action.getQuery(globalState); sb.append(query); return new SQLQueryAdapter(sb.toString(), query.getExpectedErrors()); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java index 33105bc7c..df6fc3806 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -19,7 +18,7 @@ private enum Target { TABLE, INDEX, COLLATION_NAME } - public static Query executeReindex(SQLite3GlobalState globalState) { + public static SQLQueryAdapter executeReindex(SQLite3GlobalState globalState) { SQLite3Schema s = globalState.getSchema(); StringBuilder sb = new StringBuilder("REINDEX"); ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java index 38cd1f033..f38f2a0d0 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -11,7 +10,7 @@ public final class SQLite3TransactionGenerator { private SQLite3TransactionGenerator() { } - public static Query generateCommit(SQLite3GlobalState globalState) { + public static SQLQueryAdapter generateCommit(SQLite3GlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append(Randomly.fromOptions("COMMIT", "END")); if (Randomly.getBoolean()) { @@ -21,7 +20,7 @@ public static Query generateCommit(SQLite3GlobalState globalState) { "The database file is locked", "FOREIGN KEY constraint failed"), true); } - public static Query generateBeginTransaction(SQLite3GlobalState globalState) { + public static SQLQueryAdapter generateBeginTransaction(SQLite3GlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("BEGIN "); if (Randomly.getBoolean()) { @@ -32,7 +31,7 @@ public static Query generateBeginTransaction(SQLite3GlobalState globalState) { ExpectedErrors.from("cannot start a transaction within a transaction", "The database file is locked")); } - public static Query generateRollbackTransaction(SQLite3GlobalState globalState) { + public static SQLQueryAdapter generateRollbackTransaction(SQLite3GlobalState globalState) { // TODO: could be extended by savepoint return new SQLQueryAdapter("ROLLBACK TRANSACTION;", ExpectedErrors.from("no transaction is active", "The database file is locked"), true); diff --git a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java index 2389adb94..24bed3333 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -14,7 +13,7 @@ public final class SQLite3VacuumGenerator { private SQLite3VacuumGenerator() { } - public static Query executeVacuum(SQLite3GlobalState globalState) { + public static SQLQueryAdapter executeVacuum(SQLite3GlobalState globalState) { StringBuilder sb = new StringBuilder("VACUUM"); if (Randomly.getBoolean()) { sb.append(" "); diff --git a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java index 6693ad41c..fafe50cc6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -15,7 +14,7 @@ public class SQLite3VirtualFTSTableCommandGenerator { private final Randomly r; private final ExpectedErrors errors = new ExpectedErrors(); - public static Query create(SQLite3GlobalState globalState) { + public static SQLQueryAdapter create(SQLite3GlobalState globalState) { return new SQLite3VirtualFTSTableCommandGenerator(globalState.getSchema(), globalState.getRandomly()) .generate(); } @@ -29,7 +28,7 @@ private enum Action { AUTOMERGE, CRISISMERGE, INTEGRITYCHECK, MERGE, OPTIMIZE, REBUILD, USER_MERGE, PGSZ, RANK; // TODO: delete all } - private Query generate() { + private SQLQueryAdapter generate() { errors.add("has no column named rank"); SQLite3Table vTable = s.getRandomTableOrBailout(t -> t.isVirtual() && t.getName().startsWith("vt")); Action a = Randomly.fromOptions(Action.values()); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java index d0dc74f53..2e1b4d4e8 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; @@ -18,7 +17,7 @@ public class SQLite3AlterTable { private final StringBuilder sb = new StringBuilder(); private final SQLite3GlobalState globalState; - public static Query alterTable(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter alterTable(SQLite3GlobalState globalState) throws SQLException { SQLite3AlterTable alterTable = new SQLite3AlterTable(globalState); return alterTable.getQuery(globalState.getSchema(), alterTable); } @@ -31,7 +30,7 @@ public SQLite3AlterTable(SQLite3GlobalState globalState) { this.globalState = globalState; } - private Query getQuery(SQLite3Schema s, SQLite3AlterTable alterTable) throws AssertionError { + private SQLQueryAdapter getQuery(SQLite3Schema s, SQLite3AlterTable alterTable) throws AssertionError { ExpectedErrors errors = new ExpectedErrors(); errors.add("error in view"); errors.add("no such column"); // trigger diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java index 9933a1459..a57f67445 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java @@ -7,7 +7,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; @@ -31,7 +30,7 @@ private enum TriggerAction { INSERT, DELETE, UPDATE, RAISE } - public static Query create(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter create(SQLite3GlobalState globalState) throws SQLException { SQLite3Schema s = globalState.getSchema(); StringBuilder sb = new StringBuilder(); SQLite3Table table = s.getRandomTableOrBailout(t -> !t.isVirtual()); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java index 07c81fba7..ed8cb3556 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.gen.SQLite3Common; @@ -21,7 +20,7 @@ public SQLite3CreateVirtualFTSTableGenerator(String tableName, Randomly r) { this.r = r; } - public static Query createTableStatement(String tableName, Randomly r) { + public static SQLQueryAdapter createTableStatement(String tableName, Randomly r) { return new SQLite3CreateVirtualFTSTableGenerator(tableName, r).create(); } @@ -37,7 +36,7 @@ private enum Fts4Options { MATCHINFO, TOKENIZE, PREFIX, ORDER, LANGUAGEID, COMPRESS, NOT_INDEXED } - public Query create() { + public SQLQueryAdapter create() { sb.append("CREATE VIRTUAL TABLE "); sb.append(tableName); sb.append(" USING "); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java index 2c39f6771..1bfeb2753 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -12,7 +11,7 @@ public final class SQLite3DropIndexGenerator { private SQLite3DropIndexGenerator() { } - public static Query dropIndex(SQLite3GlobalState globalState) { + public static SQLQueryAdapter dropIndex(SQLite3GlobalState globalState) { String indexName = globalState.getSchema().getRandomIndexOrBailout(); StringBuilder sb = new StringBuilder(); sb.append("DROP INDEX "); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java index c538da076..023cffe02 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java @@ -3,7 +3,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -12,7 +11,7 @@ public final class SQLite3DropTableGenerator { private SQLite3DropTableGenerator() { } - public static Query dropTable(SQLite3GlobalState globalState) { + public static SQLQueryAdapter dropTable(SQLite3GlobalState globalState) { if (globalState.getSchema().getTables(t -> !t.isView()).size() == 1) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 0b806038e..6c9adbe9b 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider; @@ -23,7 +22,7 @@ public class SQLite3IndexGenerator { private final ExpectedErrors errors = new ExpectedErrors(); private final SQLite3GlobalState globalState; - public static Query insertIndex(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter insertIndex(SQLite3GlobalState globalState) throws SQLException { return new SQLite3IndexGenerator(globalState).create(); } @@ -31,7 +30,7 @@ public SQLite3IndexGenerator(SQLite3GlobalState globalState) throws SQLException this.globalState = globalState; } - private Query create() throws SQLException { + private SQLQueryAdapter create() throws SQLException { SQLite3Table t = globalState.getSchema() .getRandomTableOrBailout(tab -> !tab.isView() && !tab.isVirtual() && !tab.isReadOnly()); String q = createIndex(t, t.getColumns()); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index b439dceda..bd817407a 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -7,7 +7,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; @@ -46,7 +45,7 @@ public SQLite3TableGenerator(String tableName, SQLite3GlobalState globalState) { this.existingSchema = globalState.getSchema(); } - public static Query createTableStatement(String tableName, SQLite3GlobalState globalState) { + public static SQLQueryAdapter createTableStatement(String tableName, SQLite3GlobalState globalState) { SQLite3TableGenerator sqLite3TableGenerator = new SQLite3TableGenerator(tableName, globalState); sqLite3TableGenerator.start(); ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 480908f3d..79fa9f4b2 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; @@ -21,14 +20,14 @@ public final class SQLite3ViewGenerator { private SQLite3ViewGenerator() { } - public static Query dropView(SQLite3GlobalState globalState) { + public static SQLQueryAdapter dropView(SQLite3GlobalState globalState) { SQLite3Schema s = globalState.getSchema(); StringBuilder sb = new StringBuilder("DROP VIEW "); sb.append(s.getRandomViewOrBailout().getName()); return new SQLQueryAdapter(sb.toString(), true); } - public static Query generate(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter generate(SQLite3GlobalState globalState) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("CREATE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index c4d678297..44c5f0172 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -17,12 +16,12 @@ public final class SQLite3DeleteGenerator { private SQLite3DeleteGenerator() { } - public static Query deleteContent(SQLite3GlobalState globalState) { + public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState) { SQLite3Table tableName = globalState.getSchema().getRandomTable(t -> !t.isView() && !t.isReadOnly()); return deleteContent(globalState, tableName); } - public static Query deleteContent(SQLite3GlobalState globalState, SQLite3Table tableName) { + public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLite3Table tableName) { StringBuilder sb = new StringBuilder(); sb.append("DELETE FROM "); sb.append(tableName.getName()); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java index 7a8d6d2ab..134da1ee7 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -30,12 +29,12 @@ public SQLite3InsertGenerator(SQLite3GlobalState globalState, Randomly r) { errors = new ExpectedErrors(); } - public static Query insertRow(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter insertRow(SQLite3GlobalState globalState) throws SQLException { SQLite3Table randomTable = globalState.getSchema().getRandomTableOrBailout(t -> !t.isView() && !t.isReadOnly()); return insertRow(globalState, randomTable); } - public static Query insertRow(SQLite3GlobalState globalState, SQLite3Table randomTable) { + public static SQLQueryAdapter insertRow(SQLite3GlobalState globalState, SQLite3Table randomTable) { SQLite3InsertGenerator generator = new SQLite3InsertGenerator(globalState, globalState.getRandomly()); String query = generator.insertRow(randomTable); return new SQLQueryAdapter(query, generator.errors, true); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java index c5a1d8be7..5e37f79e5 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java @@ -9,7 +9,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; @@ -24,11 +23,11 @@ private SQLite3StatTableGenerator(SQLite3GlobalState globalState) { this.globalState = globalState; } - public static Query getQuery(SQLite3GlobalState globalState) { + public static SQLQueryAdapter getQuery(SQLite3GlobalState globalState) { return new SQLite3StatTableGenerator(globalState).getQuery(); } - private Query getQuery() { + private SQLQueryAdapter getQuery() { List columns = new ArrayList<>(); SQLite3Table t = new SQLite3Table("sqlite_stat1", columns, TableKind.MAIN, false, false, false, false); if (Randomly.getBoolean()) { @@ -81,7 +80,8 @@ private Query getQuery() { sb.append(" noskipscan"); } sb.append("')"); - return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("no such table", "The database file is locked")); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("no such table", "The database file is locked")); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 0ecb8ee11..0c20ea98b 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -27,18 +26,18 @@ public SQLite3UpdateGenerator(SQLite3GlobalState globalState, Randomly r) { this.r = r; } - public static Query updateRow(SQLite3GlobalState globalState) { + public static SQLQueryAdapter updateRow(SQLite3GlobalState globalState) { SQLite3Table randomTableNoViewOrBailout = globalState.getSchema() .getRandomTableOrBailout(t -> !t.isView() && !t.isReadOnly()); return updateRow(globalState, randomTableNoViewOrBailout); } - public static Query updateRow(SQLite3GlobalState globalState, SQLite3Table table) { + public static SQLQueryAdapter updateRow(SQLite3GlobalState globalState, SQLite3Table table) { SQLite3UpdateGenerator generator = new SQLite3UpdateGenerator(globalState, globalState.getRandomly()); return generator.update(table); } - private Query update(SQLite3Table table) { + private SQLQueryAdapter update(SQLite3Table table) { sb.append("UPDATE "); if (Randomly.getBoolean()) { sb.append("OR IGNORE "); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 3dcd6824c..5dc0bfd97 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; import sqlancer.StateToReproduce.OracleRunReproductionState; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; @@ -41,7 +42,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; public class SQLite3PivotedQuerySynthesisOracle - extends PivotedQuerySynthesisBase { + extends PivotedQuerySynthesisBase { private List fetchColumns; private OracleRunReproductionState localState; @@ -51,7 +52,7 @@ public SQLite3PivotedQuerySynthesisOracle(SQLite3GlobalState globalState) { } @Override - public Query getRectifiedQuery() throws SQLException { + public Query getRectifiedQuery() throws SQLException { SQLite3Select selectStatement = getQuery(); SQLite3Errors.addExpectedExpressionErrors(errors); return new SQLQueryAdapter(SQLite3Visitor.asString(selectStatement), errors); @@ -174,7 +175,7 @@ private SQLite3Expression generateOffset() { } @Override - protected Query getContainmentCheckQuery(Query query) throws SQLException { + protected Query getContainmentCheckQuery(Query query) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("SELECT "); String checkForContainmentValues = getGeneralizedPivotRowValues(); diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index d0916cfa9..fcef1aad3 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -14,9 +14,9 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; @@ -378,7 +378,8 @@ private static List getTableColumns(SQLConnection con, String tab columnTypeString.contentEquals("INTEGER"), isPrimaryKey, collate)); } } - } catch (Exception e) { + } catch (SQLException e) { + } if (databaseColumns.isEmpty()) { // only generated columns diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 86f4e63de..1ec747c0d 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -6,16 +6,15 @@ import java.sql.Statement; import sqlancer.AbstractAction; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.QueryProvider; +import sqlancer.common.query.SQLQueryProvider; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; @@ -55,15 +54,15 @@ public enum Action implements AbstractAction { errors); }); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(TiDBGlobalState state) throws Exception { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(TiDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -108,7 +107,7 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { boolean success = false; do { - Query qt = new TiDBTableGenerator().getQuery(globalState); + SQLQueryAdapter qt = new TiDBTableGenerator().getQuery(globalState); success = globalState.executeStatement(qt); } while (!success); } diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index be78dab38..9f426c42f 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -8,8 +8,8 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 1e35c4640..3753a09f4 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -5,7 +5,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -22,7 +21,7 @@ private enum Action { MODIFY_COLUMN, ENABLE_DISABLE_KEYS, FORCE, DROP_PRIMARY_KEY, ADD_PRIMARY_KEY, CHANGE, DROP_COLUMN, ORDER_BY } - public static Query getQuery(TiDBGlobalState globalState) { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 93d6ebc1d..85525b6ff 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -4,7 +4,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -14,7 +13,7 @@ public final class TiDBAnalyzeTableGenerator { private TiDBAnalyzeTableGenerator() { } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); StringBuilder sb = new StringBuilder("ANALYZE "); diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index f4bf5d6bc..6c194a010 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; @@ -18,7 +17,7 @@ public final class TiDBDeleteGenerator { private TiDBDeleteGenerator() { } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { ExpectedErrors errors = new ExpectedErrors(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index 00fbd568a..3444d3281 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; @@ -16,7 +15,7 @@ public final class TiDBIndexGenerator { private TiDBIndexGenerator() { } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { ExpectedErrors errors = new ExpectedErrors(); TiDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java index cb9aed477..a78d6c2fb 100644 --- a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; @@ -26,11 +25,11 @@ public TiDBInsertGenerator(TiDBGlobalState globalState) { TiDBErrors.addInsertErrors(errors); } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { return new TiDBInsertGenerator(globalState).get(); } - private Query get() { + private SQLQueryAdapter get() { TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); StringBuilder sb = new StringBuilder(); diff --git a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java index 6450ffc6d..f0609ad56 100644 --- a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java +++ b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java @@ -5,7 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -20,7 +19,7 @@ public final class TiDBRandomQuerySynthesizer { private TiDBRandomQuerySynthesizer() { } - public static Query generate(TiDBGlobalState globalState, int nrColumns) { + public static SQLQueryAdapter generate(TiDBGlobalState globalState, int nrColumns) { TiDBSelect select = generateSelect(globalState, nrColumns); return new SQLQueryAdapter(TiDBVisitor.asString(select)); } diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 431724332..5ed96f707 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -4,7 +4,6 @@ import java.util.function.Function; import sqlancer.Randomly; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -61,7 +60,7 @@ private enum Action { } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { StringBuilder sb = new StringBuilder(); Action option = Randomly.fromOptions(Action.values()); sb.append("set @@"); diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index c6835eecf..c252a7b9c 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -8,7 +8,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBExpressionGenerator; @@ -26,7 +25,7 @@ public class TiDBTableGenerator { private boolean primaryKeyAsTableConstraints; private final ExpectedErrors errors = new ExpectedErrors(); - public Query getQuery(TiDBGlobalState globalState) throws SQLException { + public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { errors.add("Information schema is changed during the execution of the statement"); String tableName = globalState.getSchema().getFreeTableName(); int nrColumns = Randomly.smallNumber() + 1; diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index aea2d7faf..38eba8e78 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; @@ -19,7 +18,7 @@ public final class TiDBUpdateGenerator { private TiDBUpdateGenerator() { } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { ExpectedErrors errors = new ExpectedErrors(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index 3b36e027f..9da8e433e 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -3,7 +3,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -13,7 +12,7 @@ public final class TiDBViewGenerator { private TiDBViewGenerator() { } - public static Query getQuery(TiDBGlobalState globalState) { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { int nrColumns = Randomly.smallNumber() + 1; StringBuilder sb = new StringBuilder("CREATE "); if (Randomly.getBoolean()) { From 255d86670f4858b2a697287d60ef221fd8948696 Mon Sep 17 00:00:00 2001 From: Gregg Date: Mon, 30 Nov 2020 18:09:19 +0000 Subject: [PATCH 0282/1284] Clean logic to skip internal indexes. --- src/sqlancer/postgres/PostgresSchema.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 6b6fb0c06..68e2a9ce8 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -280,13 +280,9 @@ protected static List getIndexes(SQLConnection con, String tableN List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String - .format("SELECT indexname FROM pg_indexes WHERE tablename='%s' ORDER BY indexname;", tableName))) { + .format("SELECT indexname FROM pg_indexes WHERE tablename='%s' AND LENGTH(indexname)=2 ORDER BY indexname;", tableName))) { while (rs.next()) { String indexName = rs.getString("indexname"); - if (indexName.length() != 2) { - // FIXME: implement cleanly - continue; // skip internal indexes - } indexes.add(PostgresIndex.create(indexName)); } } From 81846338b5403ea3637d33e905ef4272ec2831b2 Mon Sep 17 00:00:00 2001 From: Gregg Date: Sat, 5 Dec 2020 17:49:57 +0000 Subject: [PATCH 0283/1284] Create DBMSCommon utility class and implement a matchesIndexName method. --- src/sqlancer/common/DBMSCommon.java | 30 +++++++++++++++++++ src/sqlancer/mariadb/MariaDBProvider.java | 4 +-- .../mariadb/gen/MariaDBIndexGenerator.java | 4 +-- .../mariadb/gen/MariaDBTableGenerator.java | 4 +-- src/sqlancer/mysql/MySQLProvider.java | 6 ++-- .../mysql/gen/MySQLTableGenerator.java | 4 +-- src/sqlancer/postgres/PostgresProvider.java | 4 +-- src/sqlancer/postgres/PostgresSchema.java | 7 +++-- .../gen/PostgresDropIndexGenerator.java | 4 +-- .../postgres/gen/PostgresIndexGenerator.java | 4 +-- .../postgres/gen/PostgresTableGenerator.java | 4 +-- .../postgres/gen/PostgresViewGenerator.java | 4 +-- src/sqlancer/sqlite3/SQLite3Provider.java | 6 ++-- src/sqlancer/sqlite3/gen/SQLite3Common.java | 21 ++++--------- ...Lite3CreateVirtualRtreeTabelGenerator.java | 3 +- ...SQLite3CreateVirtualFTSTableGenerator.java | 4 +-- .../gen/ddl/SQLite3TableGenerator.java | 5 ++-- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 3 +- 18 files changed, 73 insertions(+), 48 deletions(-) create mode 100644 src/sqlancer/common/DBMSCommon.java diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java new file mode 100644 index 000000000..cb5b88ff3 --- /dev/null +++ b/src/sqlancer/common/DBMSCommon.java @@ -0,0 +1,30 @@ +package sqlancer.common; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class DBMSCommon { + + private static final Pattern sqlancerIndexPattern = Pattern.compile("i\\d+"); + + private DBMSCommon() { + } + + public static String createTableName(int nr) { + return String.format("t%d", nr); + } + + public static String createColumnName(int nr) { + return String.format("c%d", nr); + } + + public static String createIndexName(int nr) { + return String.format("i%d", nr); + } + + public static boolean matchesIndexName(String indexName) { + Matcher matcher = sqlancerIndexPattern.matcher(indexName); + return matcher.matches(); + } + +} diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index bb514256f..602eaccae 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -13,6 +13,7 @@ import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.gen.MariaDBIndexGenerator; @@ -22,7 +23,6 @@ import sqlancer.mariadb.gen.MariaDBTableGenerator; import sqlancer.mariadb.gen.MariaDBTruncateGenerator; import sqlancer.mariadb.gen.MariaDBUpdateGenerator; -import sqlancer.sqlite3.gen.SQLite3Common; public class MariaDBProvider extends SQLProviderAdapter { @@ -50,7 +50,7 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { MainOptions options = globalState.getOptions(); while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { - String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); SQLQueryAdapter createTable = MariaDBTableGenerator.generate(tableName, globalState.getRandomly(), globalState.getSchema()); globalState.executeStatement(createTable); diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index d98c71ce2..19519ca2e 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -3,12 +3,12 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -import sqlancer.sqlite3.gen.SQLite3Common; public final class MariaDBIndexGenerator { @@ -26,7 +26,7 @@ public static SQLQueryAdapter generate(MariaDBSchema s) { } sb.append("INDEX "); sb.append("i"); - sb.append(SQLite3Common.createColumnName(Randomly.smallNumber())); + sb.append(DBMSCommon.createColumnName(Randomly.smallNumber())); if (Randomly.getBoolean()) { sb.append(" USING "); sb.append(Randomly.fromOptions("BTREE", "HASH")); // , "RTREE") diff --git a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java index 033e08c3d..4376b62db 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java @@ -5,6 +5,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBBugs; @@ -12,7 +13,6 @@ import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; import sqlancer.mariadb.MariaDBSchema.MariaDBTable.MariaDBEngine; import sqlancer.mariadb.ast.MariaDBVisitor; -import sqlancer.sqlite3.gen.SQLite3Common; public class MariaDBTableGenerator { @@ -54,7 +54,7 @@ private void newTable() { if (i != 0) { sb.append(", "); } - String columnName = SQLite3Common.createColumnName(i); + String columnName = DBMSCommon.createColumnName(i); columnNames.add(columnName); sb.append(columnName); sb.append(" "); diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 114150369..dd51f4982 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -11,6 +11,7 @@ import sqlancer.SQLConnection; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.mysql.gen.MySQLAlterTable; @@ -28,7 +29,6 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLChecksum; import sqlancer.mysql.gen.tblmaintenance.MySQLOptimize; import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; -import sqlancer.sqlite3.gen.SQLite3Common; public class MySQLProvider extends SQLProviderAdapter { @@ -53,7 +53,7 @@ enum Action implements AbstractAction { + "'")), // CREATE_TABLE((g) -> { // TODO refactor - String tableName = SQLite3Common.createTableName(g.getSchema().getDatabaseTables().size()); + String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); return MySQLTableGenerator.generate(g, tableName); }), // DELETE(MySQLDeleteGenerator::delete), // @@ -133,7 +133,7 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { @Override public void generateDatabase(MySQLGlobalState globalState) throws Exception { while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { - String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); SQLQueryAdapter createTable = MySQLTableGenerator.generate(globalState, tableName); globalState.executeStatement(createTable); } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index ffa85984e..579d9a4c9 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -8,6 +8,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; @@ -15,7 +16,6 @@ import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; -import sqlancer.sqlite3.gen.SQLite3Common; public class MySQLTableGenerator { @@ -247,7 +247,7 @@ private void appendTableOptions() { } private void appendColumn() { - String columnName = SQLite3Common.createColumnName(columnId); + String columnName = DBMSCommon.createColumnName(columnId); columns.add(columnName); sb.append(columnName); appendColumnDefinition(); diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index f887e11ff..c5d295611 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -14,6 +14,7 @@ import sqlancer.SQLConnection; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; @@ -38,7 +39,6 @@ import sqlancer.postgres.gen.PostgresUpdateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; import sqlancer.postgres.gen.PostgresViewGenerator; -import sqlancer.sqlite3.gen.SQLite3Common; // EXISTS // IN @@ -268,7 +268,7 @@ protected void readFunctions(PostgresGlobalState globalState) throws SQLExceptio protected void createTables(PostgresGlobalState globalState, int numTables) throws Exception { while (globalState.getSchema().getDatabaseTables().size() < numTables) { try { - String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); SQLQueryAdapter createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), generateOnlyKnown, globalState); globalState.executeStatement(createTable); diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 68e2a9ce8..889b2e2a6 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -15,6 +15,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; @@ -280,10 +281,12 @@ protected static List getIndexes(SQLConnection con, String tableN List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String - .format("SELECT indexname FROM pg_indexes WHERE tablename='%s' AND LENGTH(indexname)=2 ORDER BY indexname;", tableName))) { + .format("SELECT indexname FROM pg_indexes WHERE tablename='%s' ORDER BY indexname;", tableName))) { while (rs.next()) { String indexName = rs.getString("indexname"); - indexes.add(PostgresIndex.create(indexName)); + if (DBMSCommon.matchesIndexName(indexName)) { + indexes.add(PostgresIndex.create(indexName)); + } } } } diff --git a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java index 6a6aa9473..4ed133f0b 100644 --- a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java @@ -3,11 +3,11 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresIndex; -import sqlancer.sqlite3.gen.SQLite3Common; public final class PostgresDropIndexGenerator { @@ -25,7 +25,7 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { sb.append(", "); } if (indexes.isEmpty() || Randomly.getBoolean()) { - sb.append(SQLite3Common.createIndexName(Randomly.smallNumber())); + sb.append(DBMSCommon.createIndexName(Randomly.smallNumber())); } else { sb.append(Randomly.fromList(indexes).getIndexName()); } diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 43e1825f5..a28f42332 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -4,6 +4,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -13,7 +14,6 @@ import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.sqlite3.gen.SQLite3Common; public final class PostgresIndexGenerator { @@ -142,7 +142,7 @@ private static String getNewIndexName(PostgresTable randomTable) { List indexes = randomTable.getIndexes(); int indexI = 0; while (true) { - String indexName = SQLite3Common.createIndexName(indexI++); + String indexName = DBMSCommon.createIndexName(indexI++); if (indexes.stream().noneMatch(i -> i.getIndexName().equals(indexName))) { return indexName; } diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index ddf9753de..d1b5058af 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -5,6 +5,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -14,7 +15,6 @@ import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.sqlite3.gen.SQLite3Common; public class PostgresTableGenerator { @@ -93,7 +93,7 @@ private void createStandard() throws AssertionError { if (i != 0) { sb.append(", "); } - String name = SQLite3Common.createColumnName(i); + String name = DBMSCommon.createColumnName(i); createColumn(name); } if (Randomly.getBoolean()) { diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index 8dee2b57a..b0a2a8b9d 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -1,12 +1,12 @@ package sqlancer.postgres.gen; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.sqlite3.gen.SQLite3Common; public final class PostgresViewGenerator { @@ -51,7 +51,7 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { if (i != 0) { sb.append(", "); } - sb.append(SQLite3Common.createColumnName(i)); + sb.append(DBMSCommon.createColumnName(i)); } sb.append(")"); // if (Randomly.getBoolean() && false) { diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index a01fcb0fb..77ceb3367 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -14,13 +14,13 @@ import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; -import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.gen.SQLite3CreateVirtualRtreeTabelGenerator; import sqlancer.sqlite3.gen.SQLite3ExplainGenerator; import sqlancer.sqlite3.gen.SQLite3PragmaGenerator; @@ -244,11 +244,11 @@ private SQLQueryAdapter getTableQuery(SQLite3GlobalState globalState, int i) thr } switch (Randomly.fromList(options)) { case NORMAL: - String tableName = SQLite3Common.createTableName(i); + String tableName = DBMSCommon.createTableName(i); tableQuery = SQLite3TableGenerator.createTableStatement(tableName, globalState); break; case FTS: - String ftsTableName = "v" + SQLite3Common.createTableName(i); + String ftsTableName = "v" + DBMSCommon.createTableName(i); tableQuery = SQLite3CreateVirtualFTSTableGenerator.createTableStatement(ftsTableName, globalState.getRandomly()); break; diff --git a/src/sqlancer/sqlite3/gen/SQLite3Common.java b/src/sqlancer/sqlite3/gen/SQLite3Common.java index 97cb45787..830ab1c7f 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3Common.java +++ b/src/sqlancer/sqlite3/gen/SQLite3Common.java @@ -4,6 +4,7 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; @@ -22,18 +23,6 @@ public static String getRandomCollate() { return Randomly.fromOptions(" COLLATE BINARY", " COLLATE RTRIM", " COLLATE NOCASE"/* , " COLLATE UINT" */); } - public static String createTableName(int nr) { - return String.format("t%d", nr); - } - - public static String createColumnName(int nr) { - return String.format("c%d", nr); - } - - public static String createIndexName(int nr) { - return String.format("i%d", nr); - } - public static String getCheckConstraint(SQLite3GlobalState globalState, List columns) { SQLite3Expression expression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) .generateExpression(); @@ -75,7 +64,7 @@ public static String getFreeTableName(SQLite3Schema s) { int nr = 0; String[] name = new String[1]; do { - name[0] = SQLite3Common.createTableName(nr++); + name[0] = DBMSCommon.createTableName(nr++); } while (s.getDatabaseTables().stream().anyMatch(tab -> tab.getName().contentEquals(name[0]))); return name[0]; } @@ -93,7 +82,7 @@ public static String getFreeIndexName(SQLite3Schema s) { List indexNames = s.getIndexNames(); String candidateName; do { - candidateName = SQLite3Common.createIndexName((int) Randomly.getNotCachedInteger(0, 100)); + candidateName = DBMSCommon.createIndexName((int) Randomly.getNotCachedInteger(0, 100)); } while (indexNames.contains(candidateName)); return candidateName; } @@ -102,7 +91,7 @@ public static String getFreeColumnName(SQLite3Table t) { List indexNames = t.getColumns(); final String[] candidateName = new String[1]; do { - candidateName[0] = SQLite3Common.createColumnName((int) Randomly.getNotCachedInteger(0, 100)); + candidateName[0] = DBMSCommon.createColumnName((int) Randomly.getNotCachedInteger(0, 100)); } while (indexNames.stream().anyMatch(c -> c.getName().contentEquals(candidateName[0]))); return candidateName[0]; } @@ -131,7 +120,7 @@ public static List getOrderBy(List columns, SQ } public static SQLite3Column createColumn(int i) { - return new SQLite3Column(createColumnName(i), SQLite3DataType.NONE, false, false, null); + return new SQLite3Column(DBMSCommon.createColumnName(i), SQLite3DataType.NONE, false, false, null); } public static List getTableRefs(List tables, SQLite3Schema s) { diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index ba1637d81..7d9ff763f 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -4,6 +4,7 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; @@ -34,7 +35,7 @@ public static SQLQueryAdapter createTableStatement(String rTreeTableName, SQLite for (int i = 0; i < Randomly.smallNumber(); i++) { sb.append(", "); sb.append("+"); - String columnName = SQLite3Common.createColumnName(size + i); + String columnName = DBMSCommon.createColumnName(size + i); SQLite3ColumnBuilder columnBuilder = new SQLite3ColumnBuilder().allowPrimaryKey(false).allowNotNull(false) .allowUnique(false).allowCheck(false); String c = columnBuilder.createColumn(columnName, globalState, columns); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java index ed8cb3556..a0f79fa7e 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java @@ -5,9 +5,9 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.gen.SQLite3Common; public class SQLite3CreateVirtualFTSTableGenerator { @@ -174,7 +174,7 @@ private void createTable(String ftsVersion, Ac columnAction, Ac tableAction) { if (i != 0) { sb.append(", "); } - sb.append(SQLite3Common.createColumnName(i)); + sb.append(DBMSCommon.createColumnName(i)); columnAction.action(); } tableAction.action(); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index bd817407a..1a6f69855 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -6,6 +6,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; @@ -78,13 +79,13 @@ public void start() { boolean allowPrimaryKeyInColumn = Randomly.getBoolean(); int nrColumns = 1 + Randomly.smallNumber(); for (int i = 0; i < nrColumns; i++) { - columns.add(SQLite3Column.createDummy(SQLite3Common.createColumnName(i))); + columns.add(SQLite3Column.createDummy(DBMSCommon.createColumnName(i))); } for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } - String columnName = SQLite3Common.createColumnName(columnId); + String columnName = DBMSCommon.createColumnName(columnId); SQLite3ColumnBuilder columnBuilder = new SQLite3ColumnBuilder() .allowPrimaryKey(allowPrimaryKeyInColumn && !containsPrimaryKey); sb.append(columnBuilder.createColumn(columnName, globalState, columns)); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 79fa9f4b2..ea8bf1e12 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; @@ -87,7 +88,7 @@ private static void columnNamesAs(StringBuilder sb, int size) { if (i != 0) { sb.append(", "); } - sb.append(SQLite3Common.createColumnName(i)); + sb.append(DBMSCommon.createColumnName(i)); } sb.append(")"); sb.append(" AS "); From 10c9ddaa07351280ef0b7178f8fb0889c8e75169 Mon Sep 17 00:00:00 2001 From: Gregg Date: Sat, 5 Dec 2020 17:55:55 +0000 Subject: [PATCH 0284/1284] Add starting anchor to regular expression. --- src/sqlancer/common/DBMSCommon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java index cb5b88ff3..22e2c64c9 100644 --- a/src/sqlancer/common/DBMSCommon.java +++ b/src/sqlancer/common/DBMSCommon.java @@ -5,7 +5,7 @@ public final class DBMSCommon { - private static final Pattern sqlancerIndexPattern = Pattern.compile("i\\d+"); + private static final Pattern sqlancerIndexPattern = Pattern.compile("^i\\d+"); private DBMSCommon() { } From 2f99231b7e9b5ea495e52665faba807b1d2a09a4 Mon Sep 17 00:00:00 2001 From: Gregg Date: Sun, 6 Dec 2020 13:49:50 +0000 Subject: [PATCH 0285/1284] Use Java naming conventions for constants --- src/sqlancer/common/DBMSCommon.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java index 22e2c64c9..2c7531d10 100644 --- a/src/sqlancer/common/DBMSCommon.java +++ b/src/sqlancer/common/DBMSCommon.java @@ -5,7 +5,7 @@ public final class DBMSCommon { - private static final Pattern sqlancerIndexPattern = Pattern.compile("^i\\d+"); + private static final Pattern SQLANCER_INDEX_PATTERN = Pattern.compile("^i\\d+"); private DBMSCommon() { } @@ -23,7 +23,7 @@ public static String createIndexName(int nr) { } public static boolean matchesIndexName(String indexName) { - Matcher matcher = sqlancerIndexPattern.matcher(indexName); + Matcher matcher = SQLANCER_INDEX_PATTERN.matcher(indexName); return matcher.matches(); } From 86359ab74f71ec4eb8aeb0e34053adb4da8f776e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 26 Nov 2020 13:48:36 +0000 Subject: [PATCH 0286/1284] Bump maven-pmd-plugin from 3.13.0 to 3.14.0 Bumps [maven-pmd-plugin](https://github.com/apache/maven-pmd-plugin) from 3.13.0 to 3.14.0. - [Release notes](https://github.com/apache/maven-pmd-plugin/releases) - [Commits](https://github.com/apache/maven-pmd-plugin/compare/maven-pmd-plugin-3.13.0...maven-pmd-plugin-3.14.0) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f958a05c1..8f4534ed6 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ org.apache.maven.plugins maven-pmd-plugin - 3.13.0 + 3.14.0 pmd From c8e8b9e56b96b9dd58e2ae91c2df4d3265991d91 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 3 Dec 2020 20:36:06 +0100 Subject: [PATCH 0287/1284] Address various new PMD issues --- configs/pmd-rules.xml | 3 ++- .../cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java | 2 +- .../cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java | 4 +--- src/sqlancer/common/query/SQLQueryAdapter.java | 2 +- src/sqlancer/duckdb/DuckDBProvider.java | 2 +- .../duckdb/test/DuckDBQueryPartitioningAggregateTester.java | 2 +- .../duckdb/test/DuckDBQueryPartitioningGroupByTester.java | 4 +--- src/sqlancer/h2/H2Provider.java | 2 +- src/sqlancer/h2/H2TableGenerator.java | 4 +++- src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPAggregateOracle.java | 2 +- src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java | 2 +- src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java | 4 +--- src/sqlancer/sqlite3/schema/SQLite3Schema.java | 1 - src/sqlancer/tidb/TiDBProvider.java | 2 +- 15 files changed, 17 insertions(+), 21 deletions(-) diff --git a/configs/pmd-rules.xml b/configs/pmd-rules.xml index fd2ad9fc4..656e29f85 100644 --- a/configs/pmd-rules.xml +++ b/configs/pmd-rules.xml @@ -17,12 +17,13 @@ - + + 2 diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index 8d1a53719..bc2601ef5 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -107,7 +107,7 @@ private String createMetamorphicUnionQuery(CockroachDBSelect select, CockroachDB CockroachDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); CockroachDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); CockroachDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate).toString() + " FROM ("; + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; metamorphicQuery += CockroachDBVisitor.asString(leftSelect) + " UNION ALL " + CockroachDBVisitor.asString(middleSelect) + " UNION ALL " + CockroachDBVisitor.asString(rightSelect); metamorphicQuery += ")"; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java index 34d372d30..535fea3e1 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java @@ -42,10 +42,8 @@ public void check() throws SQLException { @Override List generateFetchColumns() { - List columns = new ArrayList<>(); - columns = Randomly.nonEmptySubset(targetTables.getColumns().stream().map(c -> new CockroachDBColumnReference(c)) + return Randomly.nonEmptySubset(targetTables.getColumns().stream().map(c -> new CockroachDBColumnReference(c)) .collect(Collectors.toList())); - return columns; } } diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 1e52b1c1a..c2e6706cc 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -113,7 +113,7 @@ public void checkException(Exception e) throws AssertionError { } else { s = globalState.getConnection().createStatement(); } - ResultSet result = null; + ResultSet result; try { if (fills.length > 0) { result = ((PreparedStatement) s).executeQuery(); diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 58a4ff4a4..3acfc56cb 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -96,7 +96,7 @@ protected DuckDBSchema readSchema() throws SQLException { @Override public void generateDatabase(DuckDBGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { - boolean success = false; + boolean success; do { SQLQueryAdapter qt = new DuckDBTableGenerator().getQuery(globalState); success = globalState.executeStatement(qt); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index 21eedf0be..a6b8d8f27 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -89,7 +89,7 @@ private String createMetamorphicUnionQuery(DuckDBSelect select, DuckDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DuckDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DuckDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate).toString() + " FROM ("; + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; metamorphicQuery += DuckDBToStringVisitor.asString(leftSelect) + " UNION ALL " + DuckDBToStringVisitor.asString(middleSelect) + " UNION ALL " + DuckDBToStringVisitor.asString(rightSelect); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java index 3cd6356fa..7ac650f44 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java @@ -46,10 +46,8 @@ public void check() throws SQLException { @Override List> generateFetchColumns() { - List> columns = new ArrayList<>(); - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); - return columns; } } diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 463add537..916849199 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -77,7 +77,7 @@ public void generateDatabase(H2GlobalState globalState) throws Exception { if (Randomly.getBoolean()) { H2SetGenerator.getQuery(globalState).execute(globalState); } - boolean success = false; + boolean success; for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { do { SQLQueryAdapter qt = new H2TableGenerator().getQuery(globalState); diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java index 1faa0bdb7..146919e8b 100644 --- a/src/sqlancer/h2/H2TableGenerator.java +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -18,7 +18,9 @@ public class H2TableGenerator { public SQLQueryAdapter getQuery(H2GlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add("already exists"); - StringBuilder sb = new StringBuilder("CREATE TABLE " + globalState.getSchema().getFreeTableName() + "("); + StringBuilder sb = new StringBuilder("CREATE TABLE "); + sb.append(globalState.getSchema().getFreeTableName()); + sb.append("("); List columnNames = new ArrayList<>(); for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { columnNames.add("c" + i); diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index 836983134..882b49c08 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -106,7 +106,7 @@ private int getOptimizedQuery(MariaDBTable randomTable, MariaDBExpression random select.setFromTables(Arrays.asList(randomTable)); select.setWhereClause(randomWhereCondition); select.setSelectType(MariaDBSelectType.ALL); - int firstCount = 0; + int firstCount; optimizedQueryString = MariaDBVisitor.asString(select); SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); try (SQLancerResultSet rs = q.executeAndGet(state)) { diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index d87a37ce3..b9ea695e2 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -94,7 +94,7 @@ private String createMetamorphicUnionQuery(PostgresSelect select, PostgresAggreg PostgresSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); PostgresSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); PostgresSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate).toString() + " FROM ("; + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; metamorphicQuery += PostgresVisitor.asString(leftSelect) + " UNION ALL " + PostgresVisitor.asString(middleSelect) + " UNION ALL " + PostgresVisitor.asString(rightSelect); metamorphicQuery += ") as asdf"; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index 1a6f69855..a5f79b0c4 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -152,7 +152,7 @@ private void addColumnConstraints(String s) { */ private void addForeignKey() { assert globalState.getDmbsSpecificOptions().testForeignKeys; - List foreignKeyColumns = new ArrayList<>(); + List foreignKeyColumns; if (Randomly.getBoolean()) { foreignKeyColumns = Arrays.asList(Randomly.fromList(columnNames)); } else { diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java index 677bc73ba..72aec6bb0 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java @@ -42,10 +42,8 @@ public void check() throws SQLException { @Override List generateFetchColumns() { - List columns = new ArrayList<>(); - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new SQLite3ColumnName(c, null)) + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new SQLite3ColumnName(c, null)) .collect(Collectors.toList()); - return columns; } } diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index fcef1aad3..ca616eed5 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -290,7 +290,6 @@ public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throw || tableName.endsWith("_config") || tableName.endsWith("_segdir") || tableName.endsWith("_stat") || tableName.endsWith("_segments") || tableName.contains("_")) { - isReadOnly = true; continue; // TODO } else if (sqlString.contains("using dbstat")) { isReadOnly = true; diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 1ec747c0d..1e1abf2a9 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -105,7 +105,7 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { @Override public void generateDatabase(TiDBGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { - boolean success = false; + boolean success; do { SQLQueryAdapter qt = new TiDBTableGenerator().getQuery(globalState); success = globalState.executeStatement(qt); From 74c6977178c60ac66fd10579091cd441cc56ae6b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 9 Dec 2020 00:22:29 +0100 Subject: [PATCH 0288/1284] Start using GitHub actions (#252) --- .github/workflows/main.yml | 27 +++++++++++++++++++++++++++ .github/workflows~HEAD | 21 +++++++++++++++++++++ .github/workflows~HEAD_0 | 21 +++++++++++++++++++++ .github/workflows~HEAD_1 | 21 +++++++++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows~HEAD create mode 100644 .github/workflows~HEAD_0 create mode 100644 .github/workflows~HEAD_1 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..7804661c0 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,27 @@ +name: CI tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + misc: + name: Misc tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build + run: mvn -B package -DskipTests=true + - name: Verify + run: mvn -B verify -DskipTests=true + - name: Misc Tests + run: mvn -B '-Dtest=!sqlancer.dbms.**' test diff --git a/.github/workflows~HEAD b/.github/workflows~HEAD new file mode 100644 index 000000000..0e057d405 --- /dev/null +++ b/.github/workflows~HEAD @@ -0,0 +1,21 @@ +name: CI tests + +on: [push, pull_request] + +jobs: + misc: + name: Misc tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build + run: mvn package -DskipTests=true + - name: Verify + run: mvn verify -DskipTests=true + - name: Misc Tests + run: mvn '-Dtest=!sqlancer.dbms.**' test diff --git a/.github/workflows~HEAD_0 b/.github/workflows~HEAD_0 new file mode 100644 index 000000000..0e057d405 --- /dev/null +++ b/.github/workflows~HEAD_0 @@ -0,0 +1,21 @@ +name: CI tests + +on: [push, pull_request] + +jobs: + misc: + name: Misc tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build + run: mvn package -DskipTests=true + - name: Verify + run: mvn verify -DskipTests=true + - name: Misc Tests + run: mvn '-Dtest=!sqlancer.dbms.**' test diff --git a/.github/workflows~HEAD_1 b/.github/workflows~HEAD_1 new file mode 100644 index 000000000..0e057d405 --- /dev/null +++ b/.github/workflows~HEAD_1 @@ -0,0 +1,21 @@ +name: CI tests + +on: [push, pull_request] + +jobs: + misc: + name: Misc tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build + run: mvn package -DskipTests=true + - name: Verify + run: mvn verify -DskipTests=true + - name: Misc Tests + run: mvn '-Dtest=!sqlancer.dbms.**' test From a0b4adfd1ad5c232a0584f16107bf840169d61cc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 9 Dec 2020 21:07:48 +0100 Subject: [PATCH 0289/1284] Test the SQLite implementation using GitHub actions --- .github/workflows/main.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7804661c0..b7bf6ae38 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,6 +15,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: @@ -25,3 +27,22 @@ jobs: run: mvn -B verify -DskipTests=true - name: Misc Tests run: mvn -B '-Dtest=!sqlancer.dbms.**' test + + sqlite: + name: SQLite Tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build + run: mvn -B package -DskipTests=true + - name: SQLite Tests + run: | + mvn -Dtest=TestSQLitePQS test + mvn -Dtest=TestSQLite3 test From 253ec53dc53bf540c0f57559ad8906807db0ddf4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 9 Dec 2020 21:34:50 +0100 Subject: [PATCH 0290/1284] Test DuckDB using GitHub actions --- .github/workflows/main.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7bf6ae38..1331768e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,3 +46,20 @@ jobs: run: | mvn -Dtest=TestSQLitePQS test mvn -Dtest=TestSQLite3 test + + duckdb: + name: DuckDB Tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build + run: mvn -B package -DskipTests=true + - name: DuckDB Tests + run: mvn -Dtest=TestDuckDB test From d94974da4f4b4ab4a0ca0060bc30c47218185c37 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 11 Dec 2020 06:22:36 +0000 Subject: [PATCH 0291/1284] Bump sqlite-jdbc from 3.32.3.2 to 3.34.0 Bumps [sqlite-jdbc](https://github.com/xerial/sqlite-jdbc) from 3.32.3.2 to 3.34.0. - [Release notes](https://github.com/xerial/sqlite-jdbc/releases) - [Changelog](https://github.com/xerial/sqlite-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/xerial/sqlite-jdbc/compare/3.32.3.2...3.34.0) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f4534ed6..7f10f3546 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ org.xerial sqlite-jdbc - 3.32.3.2 + 3.34.0 mysql From 9502c70985cdad004e610fac362cf6dfd44830fc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 12 Dec 2020 21:14:56 +0100 Subject: [PATCH 0292/1284] Address the updated SQLite3 JDBC error messages --- src/sqlancer/sqlite3/SQLite3Errors.java | 18 ++++++++++++++++++ .../sqlite3/gen/SQLite3PragmaGenerator.java | 13 +------------ .../sqlite3/gen/ddl/SQLite3IndexGenerator.java | 2 +- .../gen/dml/SQLite3InsertGenerator.java | 12 +----------- .../gen/dml/SQLite3UpdateGenerator.java | 13 ++----------- 5 files changed, 23 insertions(+), 35 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 0fa4058e3..b872fcd1d 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -17,6 +17,7 @@ public static void addDeleteErrors(ExpectedErrors errors) { } public static void addExpectedExpressionErrors(ExpectedErrors errors) { + errors.add("[SQLITE_BUSY] The database file is locked"); errors.add("FTS expression tree is too large"); errors.add("String or BLOB exceeds size limit"); errors.add("[SQLITE_ERROR] SQL error or missing database (integer overflow)"); @@ -93,4 +94,21 @@ public static void addInsertNowErrors(ExpectedErrors errors) { errors.add("non-deterministic use of date()"); } + public static void addInsertUpdateErrors(ExpectedErrors errors) { + errors.add("String or BLOB exceeds size limit"); + errors.add("[SQLITE_CONSTRAINT_CHECK]"); + errors.add("[SQLITE_CONSTRAINT_PRIMARYKEY]"); + errors.add("[SQLITE_CONSTRAINT]"); + errors.add("[SQLITE_CONSTRAINT_NOTNULL]"); + errors.add("[SQLITE_CONSTRAINT_UNIQUE]"); + errors.add("cannot INSERT into generated column"); // TODO: filter out generated columns + errors.add("A table in the database is locked"); // https://www.sqlite.org/src/tktview?name=56a74875be + errors.add("The database file is locked"); + errors.add("too many levels of trigger recursion"); + errors.add("cannot UPDATE generated column"); + errors.add("[SQLITE_ERROR] SQL error or missing database (no such table:"); + errors.add("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch"); + errors.add("no such column"); // trigger + } + } diff --git a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java index 65bed34e5..fda897d61 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java @@ -143,18 +143,7 @@ public SQLQueryAdapter insert(SQLite3GlobalState globalState) { } break; case INTEGRITY_CHECK: - // errors.add("malformed JSON"); - // errors.add("JSON cannot hold BLOB values"); - // errors.add("json_object() labels must be TEXT"); - // errors.add("requires an even number of arguments"); - // errors.add("needs an odd number of arguments"); - // errors.add("overflow"); - // errors.add("JSON path error"); - if (Randomly.getBoolean()) { - createPragma("integrity_check", () -> null); - } else { - sb.append(String.format("PRAGMA integrity_check(%d)", r.getInteger())); - } + createPragma("integrity_check", () -> null); break; case JOURNAL_MODE: // OFF is no longer generated, since it might corrupt the database upon failed diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 6c9adbe9b..184c9d9ab 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -64,7 +64,7 @@ private String createIndex(SQLite3Table t, List columns) { StringBuilder sb = new StringBuilder(); sb.append("CREATE"); if (Randomly.getBoolean()) { - errors.add("[SQLITE_CONSTRAINT] Abort due to constraint violation (UNIQUE constraint failed"); + errors.add("UNIQUE constraint failed "); sb.append(" UNIQUE"); } sb.append(" INDEX"); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java index 134da1ee7..0561e28b2 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java @@ -41,23 +41,13 @@ public static SQLQueryAdapter insertRow(SQLite3GlobalState globalState, SQLite3T } private String insertRow(SQLite3Table table) { - errors.add("cannot UPDATE generated column"); - errors.add("[SQLITE_CONSTRAINT]"); + SQLite3Errors.addInsertUpdateErrors(errors); errors.add("[SQLITE_FULL]"); - errors.add("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch"); - errors.add("[SQLITE_CONSTRAINT] Abort due to constraint violation (FOREIGN KEY constraint failed)"); // // TODO: also check if the table is really missing (caused by a DROP TABLE) - errors.add("[SQLITE_ERROR] SQL error or missing database (no such table:"); errors.add("ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint"); // trigger - errors.add("no such column"); // trigger errors.add("values were supplied"); // trigger errors.add("Data type mismatch (datatype mismatch)"); // trigger - errors.add("too many levels of trigger recursion"); - errors.add("String or BLOB exceeds size limit"); - errors.add("A table in the database is locked"); - errors.add("cannot INSERT into generated column"); // TODO: filter out generated columns - errors.add("The database file is locked"); errors.add("load_extension() prohibited in triggers and views"); SQLite3Errors.addInsertNowErrors(errors); SQLite3Errors.addExpectedExpressionErrors(errors); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 0c20ea98b..af762b846 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -95,24 +95,15 @@ private SQLQueryAdapter update(SQLite3Table table) { // sb.append(expressions.stream().map(e -> SQLite3Visitor.asString(e)).collect(Collectors.joining(", "))); // } - errors.add("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch"); - errors.add("[SQLITE_CONSTRAINT] Abort due to constraint violation"); + SQLite3Errors.addInsertUpdateErrors(errors); + errors.add("[SQLITE_ERROR] SQL error or missing database (parser stack overflow)"); errors.add( "[SQLITE_ERROR] SQL error or missing database (second argument to likelihood() must be a constant between 0.0 and 1.0)"); - errors.add("[SQLITE_ERROR] SQL error or missing database (no such table:"); // for views errors.add("ORDER BY term out of range"); - errors.add("no such column"); - errors.add("(too many levels of trigger recursion"); - errors.add("String or BLOB exceeds size limit"); - errors.add("cannot UPDATE generated column"); errors.add("unknown function: json_type"); - // TODO not update generated columns? - errors.add("cannot INSERT into generated column"); - errors.add("A table in the database is locked"); // https://www.sqlite.org/src/tktview?name=56a74875be - errors.add("The database file is locked"); SQLite3Errors.addInsertNowErrors(errors); SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addDeleteErrors(errors); From be350a62610d33bbb6d82bdeef80cd5841f3c121 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 12 Dec 2020 21:54:47 +0100 Subject: [PATCH 0293/1284] Check-in and use a specified org.eclipse.jdt.core.prefs file --- .gitignore | 3 ++- .settings/org.eclipse.jdt.core.prefs | 8 ++++++++ pom.xml | 3 +++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .settings/org.eclipse.jdt.core.prefs diff --git a/.gitignore b/.gitignore index 9fd02264e..3d3378a71 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ target/ .classpath -.settings +.settings/org.eclipse.core.resources.prefs +.settings/org.eclipse.m2e.core.prefs .project .checkstyle *.DS_Store diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 000000000..c24a64f8a --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,8 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.codeComplete.visibilityCheck=enabled +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/pom.xml b/pom.xml index 7f10f3546..c05876599 100644 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,9 @@ 8 8 + + .settings/org.eclipse.jdt.core.prefs + eclipse true true From 8f4b4ccc7a8358f0be4b7cd45af3362f7141e7ae Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 12 Dec 2020 22:32:59 +0100 Subject: [PATCH 0294/1284] Update the ECJ warnings based on a default project-specific setting in Eclipse --- .settings/org.eclipse.jdt.core.prefs | 106 +++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index c24a64f8a..0a659c1a7 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,8 +1,114 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.codeComplete.visibilityCheck=enabled +org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled +org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore +org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull +org.eclipse.jdt.core.compiler.annotation.nonnull.secondary= +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary= +org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable +org.eclipse.jdt.core.compiler.annotation.nullable.secondary= +org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.APILeak=warning +org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.autoboxing=ignore +org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning +org.eclipse.jdt.core.compiler.problem.deadCode=warning +org.eclipse.jdt.core.compiler.problem.deprecation=warning +org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled +org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled +org.eclipse.jdt.core.compiler.problem.discouragedReference=warning +org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore +org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore +org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled +org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore +org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning +org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning +org.eclipse.jdt.core.compiler.problem.forbiddenReference=error +org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning +org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled +org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning +org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning +org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore +org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore +org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning +org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore +org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore +org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled +org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled +org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning +org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning +org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning +org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore +org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning +org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error +org.eclipse.jdt.core.compiler.problem.nullReference=warning +org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error +org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning +org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore +org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning +org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore +org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore +org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore +org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning +org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning +org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore +org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore +org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled +org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning +org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled +org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled +org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled +org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.terminalDeprecation=warning +org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning +org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled +org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning +org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning +org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore +org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentType=warning +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentTypeStrict=disabled +org.eclipse.jdt.core.compiler.problem.unlikelyEqualsArgumentType=info +org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore +org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore +org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore +org.eclipse.jdt.core.compiler.problem.unstableAutoModuleName=warning +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled +org.eclipse.jdt.core.compiler.problem.unusedExceptionParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedImport=warning +org.eclipse.jdt.core.compiler.problem.unusedLabel=warning +org.eclipse.jdt.core.compiler.problem.unusedLocal=warning +org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled +org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning +org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning +org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.source=1.8 From 59a85973ff85c4d2d8429c2e6cdd881c6238e564 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 12 Dec 2020 22:51:13 +0100 Subject: [PATCH 0295/1284] Use GitHub Actions to check for warnings --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1331768e5..28b87fec1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,8 +21,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build - run: mvn -B package -DskipTests=true + - name: Build and check for warnings + run: mvn -B package -DskipTests 2>/dev/null | grep -vz "WARNING" - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests From b70b0ce83ecbced0099922897f56d94192bfb8ef Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 12 Dec 2020 22:54:48 +0100 Subject: [PATCH 0296/1284] Suppress a Citus warning This warning has previously been shown in the Eclipse IDE. This kind of warnings now lets the CI build fail. --- src/sqlancer/citus/CitusProvider.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index aed8c083b..49a548298 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -382,6 +382,7 @@ private void addCitusWorkerNodes(PostgresGlobalState globalState, SQLConnection } } + @SuppressWarnings("deprecation") @Override public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLException { synchronized (CitusProvider.class) { From 890a0aef663d55f5695cf76d514df0dacfd4a5ed Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 12 Dec 2020 23:26:33 +0100 Subject: [PATCH 0297/1284] Use GitHub Actions to test MariaDB --- .github/workflows/main.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 28b87fec1..d08b92bee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,3 +63,29 @@ jobs: run: mvn -B package -DskipTests=true - name: DuckDB Tests run: mvn -Dtest=TestDuckDB test + + mariadb: + name: MariaDB Tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Install MariaDB + run: | + sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 + sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main' + sudo apt update + sudo apt install mariadb-server + sudo systemctl start mariadb + - name: Create SQLancer User + run: sudo mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" + - name: Run Tests + run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test From 49c44c96eac3fa8ea204133265cfa792e7e4fda1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 9 Dec 2020 22:47:35 +0100 Subject: [PATCH 0298/1284] Test MySQL using GitHub actions --- .github/workflows/main.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d08b92bee..2fb2cb1b3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,3 +89,31 @@ jobs: run: sudo mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - name: Run Tests run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test + + mysql: + name: MySQL Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up MySQL + run: | + sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 + sudo apt-get remove mysql-* + wget -q https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar + tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar + sudo dpkg -i *.deb + sudo systemctl start mysql + - name: Create SQLancer user + run: mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" + - name: Run Tests + run: | + MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS + MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP From b4d9ef0e380814c430c0aa6fb391d01370a20a75 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 00:24:37 +0100 Subject: [PATCH 0299/1284] Test CockroachDB using GitHub actions --- .github/workflows/main.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2fb2cb1b3..285a1591b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,3 +117,27 @@ jobs: run: | MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP + + + cockroachdb: + name: CockroachDB Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up CockroachDB + run: | + wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.2.linux-amd64.tgz | tar xvz + cd cockroach-v20.1.2.linux-amd64/ && ./cockroach start-single-node --insecure & + sleep 10 + - name: Create SQLancer user + run: cd cockroach-v20.1.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + - name: Run Tests + run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test From 687474466825342f51a43943c2246a3d3f0ac4ed Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 01:02:06 +0100 Subject: [PATCH 0300/1284] Test ClickHouse using GitHub Actions --- .github/workflows/main.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 285a1591b..4259d7708 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -141,3 +141,25 @@ jobs: run: cd cockroach-v20.1.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test + + + clickhouse: + name: ClickHouse Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up ClickHouse + run: | + docker pull yandex/clickhouse-server:latest + docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest + sleep 5 + - name: Run Tests + run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test From 9b4d3c7d2bd221b8409622775936e2d9b6eaceff Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 01:18:31 +0100 Subject: [PATCH 0301/1284] Test TiDB using GitHub Actions --- .github/workflows/main.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4259d7708..b5795b143 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -163,3 +163,27 @@ jobs: sleep 5 - name: Run Tests run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test + + + tidb: + name: TiDB Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up TiDB + run: | + docker pull pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + sleep 10 + - name: Create SQLancer user + run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + - name: Run Tests + run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test From 199a9c72af0b3d806ef03a70ac6d5cfe8bd292ef Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 01:46:14 +0100 Subject: [PATCH 0302/1284] Use the GitHub Actions build status badge rather than the Travis one --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 058a498be..2a54b5641 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![Travis](https://travis-ci.com/sqlancer/sqlancer.svg?branch=master) +[![Build Status](https://github.com/sqlancer/sqlancer/workflows/main/badge.svg)](https://github.com/sqlancer/sqlancer/actions) [![codecov](https://codecov.io/gh/sqlancer/sqlancer/branch/master/graph/badge.svg)](https://codecov.io/gh/sqlancer/sqlancer) [![Twitter](https://img.shields.io/twitter/follow/sqlancer_dbms?style=social)](https://twitter.com/sqlancer_dbms) # SQLancer From 49ff67386742fef243ef76ead8cd2dc211959069 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 01:52:34 +0100 Subject: [PATCH 0303/1284] Rename the GitHub Actions workflow name to CI --- .github/workflows/main.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b5795b143..5fcecfec0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: CI tests +name: ci on: push: diff --git a/README.md b/README.md index 2a54b5641..235d45d55 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://github.com/sqlancer/sqlancer/workflows/main/badge.svg)](https://github.com/sqlancer/sqlancer/actions) +[![Build Status](https://github.com/sqlancer/sqlancer/workflows/ci/badge.svg)](https://github.com/sqlancer/sqlancer/actions) [![codecov](https://codecov.io/gh/sqlancer/sqlancer/branch/master/graph/badge.svg)](https://codecov.io/gh/sqlancer/sqlancer) [![Twitter](https://img.shields.io/twitter/follow/sqlancer_dbms?style=social)](https://twitter.com/sqlancer_dbms) # SQLancer From 874a5c6948704e957bcb631d9529160067aaafac Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 12:20:27 +0100 Subject: [PATCH 0304/1284] Use GitHub Actions to test PostgreSQL --- .github/workflows/main.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5fcecfec0..5f77ecdac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -187,3 +187,26 @@ jobs: run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test + + postgres: + name: PostgreSQL Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up PostgreSQL + uses: harmon758/postgresql-action@v1 + with: + postgresql version: '12' + postgresql user: 'sqlancer' + postgresql password: 'sqlancer' + postgresql db: 'test' + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test From 5614328ecf78450b6fca919420929d3285bff488 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sun, 13 Dec 2020 13:14:03 +0000 Subject: [PATCH 0305/1284] Bump duckdb_jdbc from 0.2.0 to 0.2.3 Bumps [duckdb_jdbc](https://github.com/cwida/duckdb) from 0.2.0 to 0.2.3. - [Release notes](https://github.com/cwida/duckdb/releases) - [Commits](https://github.com/cwida/duckdb/compare/v0.2.0...v0.2.3) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c05876599..9f6ba56f4 100644 --- a/pom.xml +++ b/pom.xml @@ -214,7 +214,7 @@ org.duckdb duckdb_jdbc - 0.2.0 + 0.2.3 org.junit.jupiter From a9360deda656be0caab1200a1b3d1e9d88df786c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 17:13:32 +0100 Subject: [PATCH 0306/1284] Account for the recent DuckDB changes and disable the oracles for now in the CI The latest DuckDB version seems to be affected by a bug, see https://github.com/cwida/duckdb/issues/1224. --- src/sqlancer/duckdb/DuckDBErrors.java | 6 ++++-- src/sqlancer/duckdb/DuckDBSchema.java | 12 +++++++++++- test/sqlancer/dbms/TestDuckDB.java | 6 +----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 50d781cc8..8e2e7b698 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -8,12 +8,14 @@ private DuckDBErrors() { } public static void addExpressionErrors(ExpectedErrors errors) { + errors.add("with non-constant precision is not supported"); + errors.add("Like pattern must not end with escape character"); errors.add("Could not convert string"); errors.add("ORDER term out of range - should be between "); errors.add("You might need to add explicit type casts."); errors.add("can't be cast because the value is out of range for the destination type"); errors.add("field value out of range"); - errors.add("Not implemented: Unimplemented type for cast"); + errors.add("Unimplemented type for cast"); errors.add("Escape string must be empty or one character."); errors.add("Type mismatch when combining rows"); // BETWEEN @@ -82,7 +84,7 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("can't be cast because the value is out of range for the destination type"); errors.add("Could not convert string"); errors.add("timestamp field value out of range"); - errors.add("Not implemented: Unimplemented type for cast"); // TODO: report? + errors.add("Unimplemented type for cast"); errors.add("date/time field value out of range"); errors.add("CHECK constraint failed"); errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index b63f5bb96..56e89686c 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -7,8 +7,10 @@ import java.util.Collections; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; @@ -95,7 +97,7 @@ public String toString() { case FLOAT: switch (size) { case 8: - return Randomly.fromOptions("DOUBLE", "NUMERIC"); + return Randomly.fromOptions("DOUBLE"); case 4: return Randomly.fromOptions("REAL", "FLOAT4"); default: @@ -167,6 +169,7 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { size = 2; break; case "BIGINT": + case "HUGEINT": // TODO: 16-bit int primitiveType = DuckDBDataType.INT; size = 8; break; @@ -194,6 +197,10 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { case "TIMESTAMP": primitiveType = DuckDBDataType.TIMESTAMP; break; + case "INTERVAL": + throw new IgnoreMeException(); + // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP + // '1969-12-29 06:50:27')) default: throw new AssertionError(typeString); } @@ -212,6 +219,9 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } List databaseColumns = getTableColumns(con, tableName); boolean isView = tableName.startsWith("v"); DuckDBTable t = new DuckDBTable(tableName, databaseColumns, isView); diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDB.java index ad62befbf..a2f3e89be 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDB.java @@ -12,11 +12,7 @@ public class TestDuckDB { public void testDuckDB() { // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "duckdb", "--oracle", "NoREC" })); - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "duckdb", "--oracle", - "QUERY_PARTITIONING" })); + "--num-threads", "1", "--num-queries", "0", "duckdb", "--oracle", "QUERY_PARTITIONING" })); } } From f54c13b77bd15f1b5b74ea11762313bc17f163bf Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 17:40:32 +0100 Subject: [PATCH 0307/1284] Add a new TiDB error that infrequently occurs for concurrent schema modifications --- src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 3753a09f4..e6ebe4dac 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -23,6 +23,8 @@ private enum Action { public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); + errors.add( + "Information schema is changed during the execution of the statement(for example, table definition may be updated by other DDL ran in parallel)"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); From 77102d2ca763773971cf9ad7ff9d5ff4496cbe65 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 17:42:03 +0100 Subject: [PATCH 0308/1284] Add a new PostgreSQL error --- src/sqlancer/postgres/gen/PostgresIndexGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index a28f42332..4cb2b8e3a 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -134,6 +134,7 @@ public static SQLQueryAdapter generate(PostgresGlobalState globalState) { errors.add("functions in index predicate must be marked IMMUTABLE"); errors.add("functions in index expression must be marked IMMUTABLE"); errors.add("result of range difference would not be contiguous"); + errors.add("which is part of the partition key"); PostgresCommon.addCommonExpressionErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } From 5f2055da2123dca9b8d92d0d5664aabf687ec721 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 14:11:18 +0100 Subject: [PATCH 0309/1284] Test H2 using GitHub Actions --- .github/workflows/main.yml | 22 ++++++++++++++++++++++ test/sqlancer/dbms/TestH2.java | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 test/sqlancer/dbms/TestH2.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f77ecdac..49368bf6d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -210,3 +210,25 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test + + h2: + name: H2 Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up H2 + run: | + git clone https://github.com/h2database/h2database + cd h2database/h2 + ./build.sh jar + cp bin/h2*.jar ../../target/lib/h2*.jar + - name: Run Tests + run: H2_AVAILABLE=true mvn -Dtest=TestH2 test diff --git a/test/sqlancer/dbms/TestH2.java b/test/sqlancer/dbms/TestH2.java new file mode 100644 index 000000000..8f13170ff --- /dev/null +++ b/test/sqlancer/dbms/TestH2.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestH2 { + + @Test + public void testDuckDB() { + String h2Available = System.getenv("H2_AVAILABLE"); + boolean mariaDBIsAvailable = h2Available != null && h2Available.equalsIgnoreCase("true"); + assumeTrue(mariaDBIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "h2" })); + + } + +} From a3a8821588d521fc5c3868bff4371576e396fc5d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 18:42:27 +0100 Subject: [PATCH 0310/1284] Update H2 to make it work with recent changes --- .github/workflows/main.yml | 4 +++- src/sqlancer/h2/H2Errors.java | 2 ++ src/sqlancer/h2/H2SetGenerator.java | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 49368bf6d..ee8916934 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -229,6 +229,8 @@ jobs: git clone https://github.com/h2database/h2database cd h2database/h2 ./build.sh jar - cp bin/h2*.jar ../../target/lib/h2*.jar + cp bin/h2-2.0.201.jar ../../target/lib/h2-1.4.200.jar + cp bin/h2-2.0.201.jar ~/.m2/repository/com/h2database/h2/1.4.200/h2-1.4.200.jar + ls -lh ../../target/lib - name: Run Tests run: H2_AVAILABLE=true mvn -Dtest=TestH2 test diff --git a/src/sqlancer/h2/H2Errors.java b/src/sqlancer/h2/H2Errors.java index 2a8d7af2d..aa1cba9cd 100644 --- a/src/sqlancer/h2/H2Errors.java +++ b/src/sqlancer/h2/H2Errors.java @@ -19,6 +19,8 @@ public static void addInsertErrors(ExpectedErrors errors) { } public static void addExpressionErrors(ExpectedErrors errors) { + errors.add("java.lang.ArithmeticException: BigInteger would overflow supported range"); + errors.add("Value too long for column"); errors.add("Numeric value out of range"); errors.add("are not comparable"); errors.add("Data conversion error converting"); diff --git a/src/sqlancer/h2/H2SetGenerator.java b/src/sqlancer/h2/H2SetGenerator.java index d157d9ce7..26211a93c 100644 --- a/src/sqlancer/h2/H2SetGenerator.java +++ b/src/sqlancer/h2/H2SetGenerator.java @@ -25,7 +25,7 @@ private enum Option { DEFAULT_NULL_ORDERING((r) -> Randomly.fromOptions("LOW", "HIGH", "FIRST", "LAST")), // DEFAULT_TABLE_TYPE((r) -> Randomly.fromOptions("MEMORY", "CACHED")), // IGNORECASE((r) -> Randomly.fromOptions("TRUE", "FALSE")), // - LAZY_QUERY_EXECUTION((r) -> Randomly.fromOptions(0, 1)), LOG((r) -> Randomly.fromOptions(0, 1, 2)), // + LAZY_QUERY_EXECUTION((r) -> Randomly.fromOptions(0, 1)), // MAX_MEMORY_ROWS((r) -> Randomly.getNotCachedInteger(0, 100000)), // MAX_MEMORY_UNDO((r) -> Randomly.getNotCachedInteger(0, 100000)), // MAX_OPERATION_MEMORY((r) -> Randomly.getNotCachedInteger(0, 100000)), // From ec86a54b9ac04349a0ead55e274cce0847046544 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 13 Dec 2020 01:09:51 +0100 Subject: [PATCH 0311/1284] Test Citus using GitHub Actions --- .github/workflows/main.yml | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee8916934..fb58eea40 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -234,3 +234,48 @@ jobs: ls -lh ../../target/lib - name: Run Tests run: H2_AVAILABLE=true mvn -Dtest=TestH2 test + + citus: + name: Citus Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up Citus + run: | + curl https://install.citusdata.com/community/deb.sh | sudo bash + sudo apt-get -y install postgresql-13-citus-9.5 + sudo chown -R $USER:$USER /var/run/postgresql + export PATH=/usr/lib/postgresql/13/bin:$PATH + cd ~ + mkdir -p citus/coordinator citus/worker1 citus/worker2 + initdb -D citus/coordinator + initdb -D citus/worker1 + initdb -D citus/worker2 + echo "shared_preload_libraries = 'citus'" >> citus/coordinator/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> citus/worker1/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> citus/worker2/postgresql.conf + pg_ctl -D citus/coordinator -o "-p 9700" -l coordinator_logfile start || cat coordinator_logfile || cat citus/coordinator/coordinator_logfile + pg_ctl -D citus/worker1 -o "-p 9701" -l worker1_logfile start + ls citus/worker1 + pg_ctl -D citus/worker2 -o "-p 9702" -l worker2_logfile start + psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9700 -d postgres -U $USER + createdb test -p 9700 -U $USER + psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9701 -d postgres -U $USER + createdb test -p 9701 -U $USER + psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9702 -d postgres -U $USER + createdb test -p 9702 -U $USER + psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test + psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test + psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test + psql -c "SELECT * from master_add_node('localhost', 9701);" -p 9700 -U $USER -d test + psql -c "SELECT * from master_add_node('localhost', 9702);" -p 9700 -U $USER -d test + - name: Run Tests + run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test From a75b0f00a1922a867b85cb4ee20a0286aa6ced52 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 14 Dec 2020 21:03:14 +0100 Subject: [PATCH 0312/1284] Order the GitHub Actions tasks alphabetically --- .github/workflows/main.yml | 231 ++++++++++++++++++------------------- 1 file changed, 115 insertions(+), 116 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fb58eea40..148b12092 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,7 +10,7 @@ on: jobs: misc: - name: Misc tests + name: General tests runs-on: ubuntu-latest steps: @@ -28,10 +28,9 @@ jobs: - name: Misc Tests run: mvn -B '-Dtest=!sqlancer.dbms.**' test - sqlite: - name: SQLite Tests + citus: + name: DBMS Tests (Citus) runs-on: ubuntu-latest - steps: - uses: actions/checkout@v2 with: @@ -40,17 +39,43 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build + - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: SQLite Tests - run: | - mvn -Dtest=TestSQLitePQS test - mvn -Dtest=TestSQLite3 test + - name: Set up Citus + run: | + curl https://install.citusdata.com/community/deb.sh | sudo bash + sudo apt-get -y install postgresql-13-citus-9.5 + sudo chown -R $USER:$USER /var/run/postgresql + export PATH=/usr/lib/postgresql/13/bin:$PATH + cd ~ + mkdir -p citus/coordinator citus/worker1 citus/worker2 + initdb -D citus/coordinator + initdb -D citus/worker1 + initdb -D citus/worker2 + echo "shared_preload_libraries = 'citus'" >> citus/coordinator/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> citus/worker1/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> citus/worker2/postgresql.conf + pg_ctl -D citus/coordinator -o "-p 9700" -l coordinator_logfile start || cat coordinator_logfile || cat citus/coordinator/coordinator_logfile + pg_ctl -D citus/worker1 -o "-p 9701" -l worker1_logfile start + ls citus/worker1 + pg_ctl -D citus/worker2 -o "-p 9702" -l worker2_logfile start + psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9700 -d postgres -U $USER + createdb test -p 9700 -U $USER + psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9701 -d postgres -U $USER + createdb test -p 9701 -U $USER + psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9702 -d postgres -U $USER + createdb test -p 9702 -U $USER + psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test + psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test + psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test + psql -c "SELECT * from master_add_node('localhost', 9701);" -p 9700 -U $USER -d test + psql -c "SELECT * from master_add_node('localhost', 9702);" -p 9700 -U $USER -d test + - name: Run Tests + run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test - duckdb: - name: DuckDB Tests + clickhouse: + name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest - steps: - uses: actions/checkout@v2 with: @@ -59,15 +84,20 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build + - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: DuckDB Tests - run: mvn -Dtest=TestDuckDB test + - name: Set up ClickHouse + run: | + docker pull yandex/clickhouse-server:latest + docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest + sleep 5 + - name: Run Tests + run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test - mariadb: - name: MariaDB Tests - runs-on: ubuntu-latest + cockroachdb: + name: DBMS Tests (CockroachDB) + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: @@ -78,21 +108,20 @@ jobs: java-version: 1.8 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Install MariaDB + - name: Set up CockroachDB run: | - sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 - sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main' - sudo apt update - sudo apt install mariadb-server - sudo systemctl start mariadb - - name: Create SQLancer User - run: sudo mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" + wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.2.linux-amd64.tgz | tar xvz + cd cockroach-v20.1.2.linux-amd64/ && ./cockroach start-single-node --insecure & + sleep 10 + - name: Create SQLancer user + run: cd cockroach-v20.1.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests - run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test + run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - mysql: - name: MySQL Tests + duckdb: + name: DBMS Tests (DuckDB) runs-on: ubuntu-latest + steps: - uses: actions/checkout@v2 with: @@ -101,26 +130,13 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build SQLancer + - name: Build run: mvn -B package -DskipTests=true - - name: Set up MySQL - run: | - sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 - sudo apt-get remove mysql-* - wget -q https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar - tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar - sudo dpkg -i *.deb - sudo systemctl start mysql - - name: Create SQLancer user - run: mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - - name: Run Tests - run: | - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP - + - name: DuckDB Tests + run: mvn -Dtest=TestDuckDB test - cockroachdb: - name: CockroachDB Tests + h2: + name: DBMS Tests (H2) runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -132,20 +148,21 @@ jobs: java-version: 1.8 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up CockroachDB + - name: Set up H2 run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.2.linux-amd64.tgz | tar xvz - cd cockroach-v20.1.2.linux-amd64/ && ./cockroach start-single-node --insecure & - sleep 10 - - name: Create SQLancer user - run: cd cockroach-v20.1.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + git clone https://github.com/h2database/h2database + cd h2database/h2 + ./build.sh jar + cp bin/h2-2.0.201.jar ../../target/lib/h2-1.4.200.jar + cp bin/h2-2.0.201.jar ~/.m2/repository/com/h2database/h2/1.4.200/h2-1.4.200.jar + ls -lh ../../target/lib - name: Run Tests - run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - + run: H2_AVAILABLE=true mvn -Dtest=TestH2 test - clickhouse: - name: ClickHouse Tests + mariadb: + name: DBMS Tests (MariaDB) runs-on: ubuntu-latest + steps: - uses: actions/checkout@v2 with: @@ -156,17 +173,20 @@ jobs: java-version: 1.8 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up ClickHouse + - name: Install MariaDB run: | - docker pull yandex/clickhouse-server:latest - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest - sleep 5 + sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 + sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main' + sudo apt update + sudo apt install mariadb-server + sudo systemctl start mariadb + - name: Create SQLancer User + run: sudo mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - name: Run Tests - run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test - + run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test - tidb: - name: TiDB Tests + mysql: + name: DBMS Tests (MySQL) runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -178,18 +198,23 @@ jobs: java-version: 1.8 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up TiDB + - name: Set up MySQL run: | - docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest - sleep 10 + sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 + sudo apt-get remove mysql-* + wget -q https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar + tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar + sudo dpkg -i *.deb + sudo systemctl start mysql - name: Create SQLancer user - run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + run: mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - name: Run Tests - run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test + run: | + MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS + MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP postgres: - name: PostgreSQL Tests + name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -211,9 +236,10 @@ jobs: - name: Run Tests run: POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test - h2: - name: H2 Tests + sqlite: + name: DBMS Tests (SQLite) runs-on: ubuntu-latest + steps: - uses: actions/checkout@v2 with: @@ -222,21 +248,16 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build SQLancer + - name: Build run: mvn -B package -DskipTests=true - - name: Set up H2 - run: | - git clone https://github.com/h2database/h2database - cd h2database/h2 - ./build.sh jar - cp bin/h2-2.0.201.jar ../../target/lib/h2-1.4.200.jar - cp bin/h2-2.0.201.jar ~/.m2/repository/com/h2database/h2/1.4.200/h2-1.4.200.jar - ls -lh ../../target/lib - - name: Run Tests - run: H2_AVAILABLE=true mvn -Dtest=TestH2 test + - name: SQLite Tests + run: | + mvn -Dtest=TestSQLitePQS test + mvn -Dtest=TestSQLite3 test - citus: - name: Citus Tests + + tidb: + name: DBMS Tests (TiDB) runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -248,34 +269,12 @@ jobs: java-version: 1.8 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up Citus + - name: Set up TiDB run: | - curl https://install.citusdata.com/community/deb.sh | sudo bash - sudo apt-get -y install postgresql-13-citus-9.5 - sudo chown -R $USER:$USER /var/run/postgresql - export PATH=/usr/lib/postgresql/13/bin:$PATH - cd ~ - mkdir -p citus/coordinator citus/worker1 citus/worker2 - initdb -D citus/coordinator - initdb -D citus/worker1 - initdb -D citus/worker2 - echo "shared_preload_libraries = 'citus'" >> citus/coordinator/postgresql.conf - echo "shared_preload_libraries = 'citus'" >> citus/worker1/postgresql.conf - echo "shared_preload_libraries = 'citus'" >> citus/worker2/postgresql.conf - pg_ctl -D citus/coordinator -o "-p 9700" -l coordinator_logfile start || cat coordinator_logfile || cat citus/coordinator/coordinator_logfile - pg_ctl -D citus/worker1 -o "-p 9701" -l worker1_logfile start - ls citus/worker1 - pg_ctl -D citus/worker2 -o "-p 9702" -l worker2_logfile start - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9700 -d postgres -U $USER - createdb test -p 9700 -U $USER - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9701 -d postgres -U $USER - createdb test -p 9701 -U $USER - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9702 -d postgres -U $USER - createdb test -p 9702 -U $USER - psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test - psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test - psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test - psql -c "SELECT * from master_add_node('localhost', 9701);" -p 9700 -U $USER -d test - psql -c "SELECT * from master_add_node('localhost', 9702);" -p 9700 -U $USER -d test + docker pull pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + sleep 10 + - name: Create SQLancer user + run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests - run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test + run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test From 5219f74d490bc3cd635a176063b9544761879d08 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 14 Dec 2020 21:59:03 +0100 Subject: [PATCH 0313/1284] Test SQLancer using Java 8, 13, and 15 (EA) --- .github/workflows/main.yml | 52 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 148b12092..7c0877268 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -278,3 +278,55 @@ jobs: run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test + + + java8: + name: Java 8 Compatibility (DuckDB) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build + run: mvn -B package -DskipTests=true + - name: Shortly run DuckDB + run: java -jar target/SQLancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb + + java13: + name: Java 13 Compatibility (DuckDB) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 13 + uses: actions/setup-java@v1 + with: + java-version: 13 + - name: Build + run: mvn -B package -DskipTests=true + - name: Shortly run DuckDB + run: java -jar target/SQLancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb + + java15: + name: Java 15 EA Compatibility (DuckDB) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 15 + uses: actions/setup-java@v1 + with: + java-version: 15-ea + - name: Build + run: mvn -B package -DskipTests=true + - name: Shortly run DuckDB + run: java -jar target/SQLancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb From 02a52bf2e73e9ab3d6cd77ca64b85a719236c3b2 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 14 Dec 2020 23:44:18 +0100 Subject: [PATCH 0314/1284] Fix the description of the quick-start instructions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 235d45d55..4c9208bb7 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Requirements: * [Maven](https://maven.apache.org/) (`sudo apt install maven` on Ubuntu) * The DBMS that you want to test (SQLite is an embedded DBMS and is included) -The following commands clone SQLancer, create a JAR, and start SQLancer to fuzz SQLite using Ternary Logic Query Partitioning (TLP): +The following commands clone SQLancer, create a JAR, and start SQLancer to test SQLite using Non-optimizing Reference Engine Construction (NoREC): ``` git clone https://github.com/sqlancer/sqlancer From 7b00f627b8ffdcbd059827ea26913eb86d63b858 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 17 Dec 2020 06:23:34 +0000 Subject: [PATCH 0315/1284] Bump ecj from 3.23.0 to 3.24.0 Bumps ecj from 3.23.0 to 3.24.0. Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9f6ba56f4..9a1f20a64 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ org.eclipse.jdt ecj - 3.23.0 + 3.24.0 From d8218d32d3e105979a252124bd08fb129b63df6f Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 12 Jan 2021 12:24:35 +0300 Subject: [PATCH 0316/1284] Fix ClickHouse Insert Generator error when empty column list was used --- .../clickhouse/gen/ClickHouseInsertGenerator.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index d92887541..9286185a7 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.gen; import java.sql.SQLException; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -36,8 +37,11 @@ public static SQLQueryAdapter getQuery(ClickHouseGlobalState globalState) throws private SQLQueryAdapter get() { ClickHouseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyColumnSubset().stream() - .filter(c -> !c.isAlias() && !c.isMaterialized()).collect(Collectors.toList()); + List columns = Collections.emptyList(); + while (columns.isEmpty()) { + columns = table.getRandomNonEmptyColumnSubset().stream().filter(c -> !c.isAlias() && !c.isMaterialized()) + .collect(Collectors.toList()); + } sb.append("INSERT INTO "); sb.append(table.getName()); sb.append("("); From 34b5524f83ce68c7e376d1bd263cf21823287e21 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 12 Jan 2021 11:07:34 +0000 Subject: [PATCH 0317/1284] Bump spotbugs-maven-plugin from 4.1.4 to 4.2.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.1.4 to 4.2.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.1.4...spotbugs-maven-plugin-4.2.0) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9a1f20a64..a941d74d0 100644 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.1.4 + 4.2.0 spotbugs From 4da080910ed4327a46102fd2e81b3f928c0d9144 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 12 Jan 2021 23:01:50 +0100 Subject: [PATCH 0318/1284] Fix new SpotBugs issues --- src/sqlancer/clickhouse/ast/ClickHouseCast.java | 2 +- src/sqlancer/sqlite3/ast/SQLite3Cast.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCast.java b/src/sqlancer/clickhouse/ast/ClickHouseCast.java index 24e179c4f..dc35f1fc7 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCast.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCast.java @@ -139,7 +139,7 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool for (int i = asString.length(); i >= 0; i--) { try { String substring = asString.substring(0, i); - double d = Double.valueOf(substring); + double d = Double.parseDouble(substring); BigDecimal first = new BigDecimal(substring); long longValue = first.longValue(); BigDecimal second = BigDecimal.valueOf(longValue); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Cast.java b/src/sqlancer/sqlite3/ast/SQLite3Cast.java index 32f83381c..5a33364a1 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Cast.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Cast.java @@ -161,7 +161,7 @@ private static SQLite3Constant convertInternal(SQLite3Constant originalValue, bo for (int i = asString.length(); i >= 0; i--) { try { String substring = asString.substring(0, i); - double d = Double.valueOf(substring); + double d = Double.parseDouble(substring); BigDecimal first = new BigDecimal(substring); long longValue = first.longValue(); BigDecimal second = BigDecimal.valueOf(longValue); From dd2d953c9d83ca4bce39586e2ee99a43b511d1da Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 18 Jan 2021 17:29:22 +0300 Subject: [PATCH 0319/1284] [ClickHouse] Fix aggregation results in TLP --- .../tlp/ClickHouseTLPAggregateOracle.java | 2 +- test/sqlancer/dbms/TestClickHouse.java | 37 ++++++++----------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index a7232d304..45878446b 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -63,7 +63,7 @@ public void check() throws SQLException { metamorphicText += ClickHouseVisitor.asString(leftSelect) + " UNION ALL " + ClickHouseVisitor.asString(middleSelect) + " UNION ALL " + ClickHouseVisitor.asString(rightSelect); metamorphicText += ")"; - + metamorphicText += " SETTINGS aggregate_functions_null_for_empty = 1"; List firstResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQuery, errors, state); List secondResult = ComparatorHelper.getResultSetFirstColumnAsString(metamorphicText, errors, state); diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 86d23a9b7..440deb1bc 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -14,9 +14,9 @@ public void testClickHouseTLPWhereGroupBy() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, - "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", - "TLPWhere", "--oracle", "TLPGroupBy" })); + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPWhere", + "--oracle", "TLPGroupBy")); } @Test @@ -24,10 +24,8 @@ public void testClickHouseTLPWhere() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, - "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", - "TLPWhere" })); + assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, + "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPWhere")); } @Test @@ -35,11 +33,10 @@ public void testClickHouseTLPHaving() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", "0", "--num-threads", "5", - "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPHaving" })); // Disabled - // in CI - // https://github.com/ClickHouse/ClickHouse/issues/12264 + assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", "0", "--num-threads", "5", + "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPHaving")); // Disabled + // in CI + // https://github.com/ClickHouse/ClickHouse/issues/12264 } @Test @@ -48,9 +45,8 @@ public void testClickHouseTLPGroupBy() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, - "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", - "TLPGroupBy" })); + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPGroupBy")); } @Test @@ -59,9 +55,8 @@ public void testClickHouseTLPDistinct() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, - "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", - "TLPDistinct" })); + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPDistinct")); } @Test @@ -69,9 +64,9 @@ public void testClickHouseTLPAggregate() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - assertEquals(0, Main.executeMain(new String[] { "--timeout-seconds", "0", "--num-queries", "0", "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPAggregate" })); // Disabled - // https://github.com/ClickHouse/ClickHouse/issues/13894 + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPAggregate")); } } From 4bd741275ed82c0cf475c88a4a4a2a3ce91e2f48 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 19 Jan 2021 22:01:07 +0100 Subject: [PATCH 0320/1284] Change the pom.xml to reflect the typical naming conventions --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a941d74d0..179370e8d 100644 --- a/pom.xml +++ b/pom.xml @@ -2,9 +2,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - SQLancer + com.sqlancer SQLancer - 0.0.1-SNAPSHOT + 1.0 UTF-8 From 244affaa736e1330b7f3ca61bb64209f768aa0e9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 19 Jan 2021 22:21:29 +0100 Subject: [PATCH 0321/1284] Support the TiDB varchar type in the schema --- src/sqlancer/tidb/TiDBSchema.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 9f426c42f..4d04369c2 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -181,7 +181,8 @@ private static TiDBCompositeDataType getColumnType(String typeString) { if (trimmedStringType.contains("decimal")) { return new TiDBCompositeDataType(TiDBDataType.DECIMAL); } - if (trimmedStringType.startsWith("var_string") || trimmedStringType.contains("binary")) { + if (trimmedStringType.startsWith("var_string") || trimmedStringType.contains("binary") + || trimmedStringType.startsWith("varchar")) { return new TiDBCompositeDataType(TiDBDataType.TEXT); } if (trimmedStringType.startsWith("char")) { From 50255ffe92ff3ef6cabbd54a08633ed302d1558e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 19 Jan 2021 21:54:40 +0000 Subject: [PATCH 0322/1284] Bump mysql-connector-java from 8.0.22 to 8.0.23 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.22 to 8.0.23. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.22...8.0.23) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 179370e8d..31ee8f5f3 100644 --- a/pom.xml +++ b/pom.xml @@ -204,7 +204,7 @@ mysql mysql-connector-java - 8.0.22 + 8.0.23 org.mariadb.jdbc From 80e234161a913236b1a759f56b005a77991b7b9d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 19 Jan 2021 23:08:47 +0100 Subject: [PATCH 0323/1284] Fix the pom.xml to work with IntelliJ Fixes https://github.com/sqlancer/sqlancer/issues/273. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 31ee8f5f3..dc5c26352 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ 8 8 - .settings/org.eclipse.jdt.core.prefs + ${project.basedir}/.settings/org.eclipse.jdt.core.prefs eclipse true From 6d5a0918535c4898ba90ca01f73ab1843317a1cb Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 19 Jan 2021 23:24:11 +0100 Subject: [PATCH 0324/1284] Change the JAR name from SQLancer-* to sqlancer-* This commit changes the JAR naming format from `SQLancer-*.jar` to `sqlancer-*.jar` to follow the typical naming conventions (see https://maven.apache.org/guides/mini/guide-naming-conventions.html). This is in preparation for creating proper releases (see https://github.com/sqlancer/sqlancer/issues/280). --- .github/workflows/main.yml | 6 +++--- README.md | 2 +- pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c0877268..16e054ca1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -295,7 +295,7 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: java -jar target/SQLancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb + run: java -jar target/sqlancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb java13: name: Java 13 Compatibility (DuckDB) @@ -312,7 +312,7 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: java -jar target/SQLancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb + run: java -jar target/sqlancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb java15: name: Java 15 EA Compatibility (DuckDB) @@ -329,4 +329,4 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: java -jar target/SQLancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb + run: java -jar target/sqlancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb diff --git a/README.md b/README.md index 4c9208bb7..2836ff423 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ git clone https://github.com/sqlancer/sqlancer cd sqlancer mvn package -DskipTests cd target -java -jar SQLancer-0.0.1-SNAPSHOT.jar --num-threads 4 sqlite3 --oracle NoREC +java -jar sqlancer-*.jar --num-threads 4 sqlite3 --oracle NoREC ``` If the execution prints progress information every five seconds, then the tool works as expected. Note that SQLancer might find bugs in SQLite. Before reporting these, be sure to check that they can still be reproduced when using the latest development version. The shortcut CTRL+C can be used to terminate SQLancer manually. If SQLancer does not find any bugs, it executes infinitely. The option `--num-tries` can be used to control after how many bugs SQLancer terminates. Alternatively, the option `--timeout-seconds` can be used to specify the maximum duration that SQLancer is allowed to run. diff --git a/pom.xml b/pom.xml index dc5c26352..e957845e5 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.sqlancer - SQLancer + sqlancer 1.0 UTF-8 From 3f8b027fa7596579c0a7e47bc2070a6df3de737a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 20 Jan 2021 21:13:23 +0100 Subject: [PATCH 0325/1284] Publish packages to Maven Central --- .github/workflows/maven-central.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/maven-central.yml diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml new file mode 100644 index 000000000..6346c9a73 --- /dev/null +++ b/.github/workflows/maven-central.yml @@ -0,0 +1,21 @@ +name: Publish package to the Maven Central Repository +on: + release: + types: [created] +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Maven Central Repository + uses: actions/setup-java@v1 + with: + java-version: 1.8 + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - name: Publish package + run: mvn --batch-mode deploy -DskipTests=true + env: + MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} From 9b7aa254a2c24d8ec1badff0c683fb5be8be1a01 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 20 Jan 2021 21:20:06 +0100 Subject: [PATCH 0326/1284] Add the OSSRH URL --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index e957845e5..8d5d1bc71 100644 --- a/pom.xml +++ b/pom.xml @@ -247,4 +247,11 @@ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + From 730e45b8c74eadd380dd8794be4e5e4fee1c1017 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 20 Jan 2021 22:04:31 +0100 Subject: [PATCH 0327/1284] Allow manually pushing a release to Maven Central --- .github/workflows/maven-central.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml index 6346c9a73..f78145bcc 100644 --- a/.github/workflows/maven-central.yml +++ b/.github/workflows/maven-central.yml @@ -1,5 +1,6 @@ name: Publish package to the Maven Central Repository on: + workflow_dispatch: release: types: [created] jobs: From 0fbac099dd4e88dca61dc2f5c3efab97ef84f148 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 20 Jan 2021 22:49:09 +0100 Subject: [PATCH 0328/1284] Generate a JavaDoc JAR This addresses the Sonatype "Missing: no javadoc jar found in folder '/com/sqlancer/sqlancer/1.0'" error. --- pom.xml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pom.xml b/pom.xml index 8d5d1bc71..e2e93edb1 100644 --- a/pom.xml +++ b/pom.xml @@ -12,6 +12,22 @@ src test + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + 8 + + + + attach-javadocs + + jar + + + + org.apache.maven.plugins maven-surefire-plugin From e11882413cd048be9b6189a4f8a54d09d7dcb9bf Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 20 Jan 2021 23:16:52 +0100 Subject: [PATCH 0329/1284] Fix JavaDoc issues --- pom.xml | 2 +- src/sqlancer/DatabaseProvider.java | 3 +++ src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java | 3 +++ src/sqlancer/mysql/ast/MySQLComputableFunction.java | 8 +++++--- src/sqlancer/mysql/ast/MySQLInOperation.java | 3 ++- src/sqlancer/mysql/gen/MySQLDropIndex.java | 2 +- .../mysql/gen/tblmaintenance/MySQLAnalyzeTable.java | 2 +- .../mysql/gen/tblmaintenance/MySQLCheckTable.java | 2 +- src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java | 2 +- src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java | 2 +- src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java | 2 +- src/sqlancer/postgres/ast/PostgresAggregate.java | 2 +- src/sqlancer/postgres/gen/PostgresCommentGenerator.java | 2 +- src/sqlancer/sqlite3/ast/SQLite3Aggregate.java | 2 +- src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java | 2 +- src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java | 2 +- src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java | 2 +- 17 files changed, 26 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index e2e93edb1..13e39945b 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ src test - + org.apache.maven.plugins maven-javadoc-plugin 3.2.0 diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 17f2a5ada..72acb30d8 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -24,6 +24,9 @@ public interface DatabaseProvider, O extends DBMS * @param globalState * the state created and is valid for this method call. * + * @throws Exception + * if creating the database fails. + * */ void generateAndTestDatabase(G globalState) throws Exception; diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index b790e6e99..cd2580b76 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -61,6 +61,7 @@ public final void check() throws Exception { * @return true if at least one row is contained, false otherwise * * @throws Exception + * if the query unexpectedly fails */ private boolean containsRows(Query query) throws Exception { try (SQLancerResultSet result = query.executeAndGet(globalState)) { @@ -108,6 +109,7 @@ protected void reportMissingPivotRow(Query query) { * @return a query that checks whether the pivot row is contained in pivotRowQuery * * @throws Exception + * if an unexpected error occurs */ protected abstract Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception; @@ -118,6 +120,7 @@ protected void reportMissingPivotRow(Query query) { * @return the rectified query * * @throws Exception + * if an unexpected error occurs */ protected abstract Query getRectifiedQuery() throws Exception; diff --git a/src/sqlancer/mysql/ast/MySQLComputableFunction.java b/src/sqlancer/mysql/ast/MySQLComputableFunction.java index 8e790b462..7ffc93000 100644 --- a/src/sqlancer/mysql/ast/MySQLComputableFunction.java +++ b/src/sqlancer/mysql/ast/MySQLComputableFunction.java @@ -38,7 +38,8 @@ public enum MySQLFunction { // } // }, /** - * @see https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#function_bit-count + * @see Bit Functions + * and Operators */ BIT_COUNT(1, "BIT_COUNT") { @@ -92,7 +93,8 @@ public boolean isVariadic() { }, /** - * @see https://dev.mysql.com/doc/refman/8.0/en/control-flow-functions.html#function_if + * @see Flow Control + * Functions */ IF(3, "IF") { @@ -113,7 +115,7 @@ public MySQLConstant apply(MySQLConstant[] args, MySQLExpression... origArgs) { }, /** - * @see https://dev.mysql.com/doc/refman/8.0/en/control-flow-functions.html#function_ifnull + * @see IFNULL */ IFNULL(2, "IFNULL") { diff --git a/src/sqlancer/mysql/ast/MySQLInOperation.java b/src/sqlancer/mysql/ast/MySQLInOperation.java index 93d340063..81dff62f2 100644 --- a/src/sqlancer/mysql/ast/MySQLInOperation.java +++ b/src/sqlancer/mysql/ast/MySQLInOperation.java @@ -5,7 +5,8 @@ import sqlancer.IgnoreMeException; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_in + * @see Comparison Functions and + * Operators */ public class MySQLInOperation implements MySQLExpression { diff --git a/src/sqlancer/mysql/gen/MySQLDropIndex.java b/src/sqlancer/mysql/gen/MySQLDropIndex.java index 4e56c9a30..569a5bfec 100644 --- a/src/sqlancer/mysql/gen/MySQLDropIndex.java +++ b/src/sqlancer/mysql/gen/MySQLDropIndex.java @@ -8,7 +8,7 @@ import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/drop-index.html + * @see DROP INDEX Statement */ public final class MySQLDropIndex { diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java index 7714031ff..c7533b0c7 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java @@ -10,7 +10,7 @@ import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/analyze-table.html + * @see ANALYZE TABLE Statement */ public class MySQLAnalyzeTable { diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java index 5858343bf..86e950cf0 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java @@ -9,7 +9,7 @@ import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/check-table.html + * @see CHECK TABLE Statement */ public class MySQLCheckTable { diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java index 4196405a5..a9442c003 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java @@ -9,7 +9,7 @@ import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/checksum-table.html + * @see CHECKSUM TABLE Statement */ public class MySQLChecksum { diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java index cbe57c46d..2ee0fe6d2 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java @@ -9,7 +9,7 @@ import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/optimize-table.html + * @see OPTIMIZE TABLE Statement */ public class MySQLOptimize { diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java index 4ce4db692..322fa353e 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java @@ -10,7 +10,7 @@ import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/repair-table.html + * @see REPAIR TABLE Statement */ public class MySQLRepair { diff --git a/src/sqlancer/postgres/ast/PostgresAggregate.java b/src/sqlancer/postgres/ast/PostgresAggregate.java index 087431756..c266e92ad 100644 --- a/src/sqlancer/postgres/ast/PostgresAggregate.java +++ b/src/sqlancer/postgres/ast/PostgresAggregate.java @@ -10,7 +10,7 @@ import sqlancer.postgres.ast.PostgresAggregate.PostgresAggregateFunction; /** - * @see https://www.sqlite.org/lang_aggfunc.html + * @see Built-in Aggregate Functions */ public class PostgresAggregate extends FunctionNode implements PostgresExpression { diff --git a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java index 343242ce9..adcce3d82 100644 --- a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java @@ -7,7 +7,7 @@ import sqlancer.postgres.PostgresSchema.PostgresTable; /** - * @see https://www.postgresql.org/docs/devel/sql-comment.html + * @see COMMENT */ public final class PostgresCommentGenerator { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Aggregate.java b/src/sqlancer/sqlite3/ast/SQLite3Aggregate.java index d1e036b52..a9129d489 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Aggregate.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Aggregate.java @@ -11,7 +11,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; /** - * @see https://www.sqlite.org/lang_aggfunc.html + * @see Built-in Aggregate Functions */ public class SQLite3Aggregate extends SQLite3Expression { diff --git a/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java b/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java index 2c4c16e12..0d5606c6a 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java +++ b/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java @@ -36,7 +36,7 @@ public SQLite3CollateSequence getImplicitCollateSequence() { /** * Supported unary prefix operators are these: -, +, ~, and NOT. * - * @see https://www.sqlite.org/lang_expr.html + * @see SQL Language Expressions * */ public enum UnaryOperator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java index df6fc3806..189b51cdf 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java @@ -7,7 +7,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema; /** - * @see https://www.sqlite.org/lang_reindex.html + * @see REINDEX */ public final class SQLite3ReindexGenerator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java index 24bed3333..c3c176980 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java @@ -6,7 +6,7 @@ import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; /** - * @see https://www.sqlite.org/lang_vacuum.html + * @see VACUUM */ public final class SQLite3VacuumGenerator { From cb9833d4163af6a8e5ee8962a4a1537c4465aad7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 20 Jan 2021 23:52:31 +0100 Subject: [PATCH 0330/1284] Extract the SQLite3GlobalState to a separate class to fix a JavaDoc issue --- src/sqlancer/sqlite3/SQLite3GlobalState.java | 15 +++++++++++++++ src/sqlancer/sqlite3/SQLite3Options.java | 1 - src/sqlancer/sqlite3/SQLite3Provider.java | 12 ------------ .../sqlite3/ast/SQLite3WindowFunction.java | 2 +- .../sqlite3/gen/SQLite3AnalyzeGenerator.java | 2 +- .../sqlite3/gen/SQLite3ColumnBuilder.java | 2 +- src/sqlancer/sqlite3/gen/SQLite3Common.java | 2 +- .../SQLite3CreateVirtualRtreeTabelGenerator.java | 2 +- .../sqlite3/gen/SQLite3ExplainGenerator.java | 2 +- .../sqlite3/gen/SQLite3ExpressionGenerator.java | 2 +- .../sqlite3/gen/SQLite3PragmaGenerator.java | 2 +- .../sqlite3/gen/SQLite3ReindexGenerator.java | 2 +- .../sqlite3/gen/SQLite3TransactionGenerator.java | 2 +- .../sqlite3/gen/SQLite3VacuumGenerator.java | 2 +- .../SQLite3VirtualFTSTableCommandGenerator.java | 2 +- .../sqlite3/gen/ddl/SQLite3AlterTable.java | 2 +- .../gen/ddl/SQLite3CreateTriggerGenerator.java | 2 +- .../gen/ddl/SQLite3DropIndexGenerator.java | 2 +- .../gen/ddl/SQLite3DropTableGenerator.java | 2 +- .../sqlite3/gen/ddl/SQLite3IndexGenerator.java | 2 +- .../sqlite3/gen/ddl/SQLite3TableGenerator.java | 2 +- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 2 +- .../sqlite3/gen/dml/SQLite3DeleteGenerator.java | 2 +- .../sqlite3/gen/dml/SQLite3InsertGenerator.java | 2 +- .../gen/dml/SQLite3StatTableGenerator.java | 2 +- .../sqlite3/gen/dml/SQLite3UpdateGenerator.java | 2 +- src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java | 2 +- .../sqlite3/oracle/SQLite3NoRECOracle.java | 2 +- .../SQLite3PivotedQuerySynthesisOracle.java | 2 +- .../oracle/SQLite3RandomQuerySynthesizer.java | 2 +- .../oracle/tlp/SQLite3TLPAggregateOracle.java | 2 +- .../sqlite3/oracle/tlp/SQLite3TLPBase.java | 2 +- .../oracle/tlp/SQLite3TLPDistinctOracle.java | 2 +- .../oracle/tlp/SQLite3TLPGroupByOracle.java | 2 +- .../oracle/tlp/SQLite3TLPHavingOracle.java | 2 +- .../sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java | 2 +- src/sqlancer/sqlite3/schema/SQLite3Schema.java | 2 +- 37 files changed, 49 insertions(+), 47 deletions(-) create mode 100644 src/sqlancer/sqlite3/SQLite3GlobalState.java diff --git a/src/sqlancer/sqlite3/SQLite3GlobalState.java b/src/sqlancer/sqlite3/SQLite3GlobalState.java new file mode 100644 index 000000000..ea8bd5f77 --- /dev/null +++ b/src/sqlancer/sqlite3/SQLite3GlobalState.java @@ -0,0 +1,15 @@ +package sqlancer.sqlite3; + +import java.sql.SQLException; + +import sqlancer.SQLGlobalState; +import sqlancer.sqlite3.schema.SQLite3Schema; + +public class SQLite3GlobalState extends SQLGlobalState { + + @Override + protected SQLite3Schema readSchema() throws SQLException { + return SQLite3Schema.fromConnection(this); + } + +} diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index fdf438672..dabc3ce13 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -13,7 +13,6 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.oracle.SQLite3Fuzzer; import sqlancer.sqlite3.oracle.SQLite3NoRECOracle; import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 77ceb3367..35e9b90cd 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -11,7 +11,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.DBMSCommon; @@ -19,7 +18,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; import sqlancer.sqlite3.gen.SQLite3CreateVirtualRtreeTabelGenerator; import sqlancer.sqlite3.gen.SQLite3ExplainGenerator; @@ -40,7 +38,6 @@ import sqlancer.sqlite3.gen.dml.SQLite3InsertGenerator; import sqlancer.sqlite3.gen.dml.SQLite3StatTableGenerator; import sqlancer.sqlite3.gen.dml.SQLite3UpdateGenerator; -import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; public class SQLite3Provider extends SQLProviderAdapter { @@ -111,15 +108,6 @@ public SQLQueryAdapter getQuery(SQLite3GlobalState state) throws Exception { } } - public static class SQLite3GlobalState extends SQLGlobalState { - - @Override - protected SQLite3Schema readSchema() throws SQLException { - return SQLite3Schema.fromConnection(this); - } - - } - private enum TableType { NORMAL, FTS, RTREE } diff --git a/src/sqlancer/sqlite3/ast/SQLite3WindowFunction.java b/src/sqlancer/sqlite3/ast/SQLite3WindowFunction.java index e48b9c42f..917949198 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3WindowFunction.java +++ b/src/sqlancer/sqlite3/ast/SQLite3WindowFunction.java @@ -4,8 +4,8 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Provider; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; diff --git a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java index 5d04d25e1..e795f610e 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; public final class SQLite3AnalyzeGenerator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index 1625c92a5..41a6fdcef 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -4,8 +4,8 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; diff --git a/src/sqlancer/sqlite3/gen/SQLite3Common.java b/src/sqlancer/sqlite3/gen/SQLite3Common.java index 830ab1c7f..e40e4d524 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3Common.java +++ b/src/sqlancer/sqlite3/gen/SQLite3Common.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index 7d9ff763f..08effcdc9 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -7,7 +7,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; public final class SQLite3CreateVirtualRtreeTabelGenerator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java index cfc7aa838..e9ede64d9 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java @@ -2,9 +2,9 @@ import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.sqlite3.SQLite3Provider.Action; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public final class SQLite3ExplainGenerator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 4d64ea664..edbbc29f6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; import sqlancer.sqlite3.ast.SQLite3Case.CasePair; diff --git a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java index fda897d61..c83cf4958 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; public class SQLite3PragmaGenerator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java index 189b51cdf..b3359b15d 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; /** diff --git a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java index f38f2a0d0..eff3a83af 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; public final class SQLite3TransactionGenerator { diff --git a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java index c3c176980..36b129cd6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; /** * @see VACUUM diff --git a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java index fafe50cc6..6efb6aeff 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java index 2e1b4d4e8..5cf4e9339 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.schema.SQLite3Schema; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java index a57f67445..610385501 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.gen.dml.SQLite3DeleteGenerator; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java index 1bfeb2753..3f4427bce 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; // see https://www.sqlite.org/lang_dropindex.html public final class SQLite3DropIndexGenerator { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java index 023cffe02..ed5eb16e4 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java @@ -4,7 +4,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; public final class SQLite3DropTableGenerator { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 184c9d9ab..0a33016fd 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -7,8 +7,8 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Provider; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.gen.SQLite3Common; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index a5f79b0c4..ef7d696cf 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -10,8 +10,8 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.schema.SQLite3Schema; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index ea8bf1e12..400c66864 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -7,8 +7,8 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Select; diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 44c5f0172..4c86417d0 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -6,7 +6,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java index 0561e28b2..0e01cdc5a 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java @@ -8,7 +8,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Constant; diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java index 5e37f79e5..c539ad648 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java @@ -10,7 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table.TableKind; diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index af762b846..791b7f5eb 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -7,7 +7,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index 500522ffb..429b22dd6 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -3,7 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; // tries to trigger a crash diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 609c76f70..db9f55531 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -12,7 +12,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Expression; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 5dc0bfd97..c19a1059d 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -15,7 +15,7 @@ import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java index 48c8aa391..4447ea2fb 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java @@ -4,7 +4,7 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Select; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 79d5bc09b..c0243425f 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -12,7 +12,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index a4518b6b6..de1f01c92 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -10,7 +10,7 @@ import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java index 41a2d0dc7..227db3b6a 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java @@ -5,7 +5,7 @@ import java.util.List; import sqlancer.ComparatorHelper; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Select.SelectType; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java index 72aec6bb0..91977d973 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java @@ -7,7 +7,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index d82666e48..2524d979b 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -11,7 +11,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.Join; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java index 91090e1db..95026de86 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java @@ -6,7 +6,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; public class SQLite3TLPWhereOracle extends SQLite3TLPBase { diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index ca616eed5..b4bf11002 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -20,7 +20,7 @@ import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; From f0b5a00f5c7a357ed56bd9a85b23e2de4ef00144 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 21 Jan 2021 10:13:22 +0100 Subject: [PATCH 0331/1284] Adapt the regex in the GitHub Actions workflow to not include the Javadoc --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 16e054ca1..6db7a4e72 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -295,7 +295,7 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: java -jar target/sqlancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 duckdb java13: name: Java 13 Compatibility (DuckDB) @@ -312,7 +312,7 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: java -jar target/sqlancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 duckdb java15: name: Java 15 EA Compatibility (DuckDB) @@ -329,4 +329,4 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: java -jar target/sqlancer-*.jar --num-threads 4 --timeout-seconds 30 duckdb + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 duckdb From 57dd1d746126221204fda62570fbaca88d5fa7cf Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 21 Jan 2021 13:18:32 +0100 Subject: [PATCH 0332/1284] Add additional project information to the pom.xml This addresses the currently failing POM Validation at https://oss.sonatype.org/: "Invalid POM: /com/sqlancer/sqlancer/1.0/sqlancer-1.0.pom: Project name missing, Project description missing, Project URL missing, License information missing, SCM URL missing, Developer information missing" --- pom.xml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pom.xml b/pom.xml index 13e39945b..ba85882f7 100644 --- a/pom.xml +++ b/pom.xml @@ -5,6 +5,35 @@ com.sqlancer sqlancer 1.0 + SQLancer + http://www.sqlancer.com/ + SQLancer finds logic bugs in Database Management Systems through automatic testing + + + MIT License + https://github.com/sqlancer/sqlancer/blob/master/LICENSE.md + repo + + + + + mrigger + Manuel Rigger + manuel.rigger@inf.ethz.ch + ETH Zurich + https://ethz.ch/ + https://www.manuelrigger.at/ + + architect + developer + + + + + git@github.com:sqlancer/sqlancer.git + scm:git:git@github.com:sqlancer/sqlancer.git + scm:git:git@github.com:sqlancer/sqlancer.git + UTF-8 From d8f5595cfe84fd6e5d49ad00bfe5d757cabe3f34 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 21 Jan 2021 16:58:25 +0100 Subject: [PATCH 0333/1284] Sign releases Based on the description titled "How to Sign and Release to The Central Repository with GitHub Actions" at https://gist.github.com/sualeh/ae78dc16123899d7942bc38baba5203c --- .github/workflows/maven-central.yml | 4 +++- pom.xml | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml index f78145bcc..ffa28cdd7 100644 --- a/.github/workflows/maven-central.yml +++ b/.github/workflows/maven-central.yml @@ -15,8 +15,10 @@ jobs: server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD + - name: Install gpg secret key + run: cat <(echo -e "${{ secrets.OSSRH_GPG_SECRET_KEY }}") | gpg --batch --import - name: Publish package - run: mvn --batch-mode deploy -DskipTests=true + run: mvn --batch-mode deploy -DskipTests=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} env: MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} diff --git a/pom.xml b/pom.xml index ba85882f7..4524ca698 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,27 @@ src test + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + true + + --pinentry-mode + loopback + + + + + sign-artifacts + deploy + + sign + + + + org.apache.maven.plugins maven-javadoc-plugin From ba03ed7fd6b68b2475b734496dd17fbb6621c99f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 21 Jan 2021 17:28:30 +0100 Subject: [PATCH 0334/1284] Fix a spurious Postgres error --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index d1b5058af..823cc916b 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -197,6 +197,7 @@ private void generateInherits() { errors.add("cannot inherit from temporary relation"); errors.add("cannot inherit from partitioned table"); errors.add("has a collation conflict"); + errors.add("inherits conflicting default values"); } } From 1dea00e1dca98a909ec61c732138c95d49bc10f8 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 21 Jan 2021 18:01:37 +0100 Subject: [PATCH 0335/1284] Fix a build issue introduced by the Javadoc changes Fix https://github.com/sqlancer/sqlancer/issues/288. The fix was adopted from https://stackoverflow.com/questions/49472783/maven-is-unable-to-find-javadoc-command and https://stackoverflow.com/questions/57081473/how-to-make-maven-javadoc-plugin-work-with-any-java-version. --- pom.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pom.xml b/pom.xml index 4524ca698..35edcd735 100644 --- a/pom.xml +++ b/pom.xml @@ -320,4 +320,24 @@ https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + jdk-8-config + + [1.3,1.9) + + + ${java.home}/../bin/javadoc + + + + jdk-11-config + + [11,) + + + ${java.home}/bin/javadoc + + + From e2787f0a880b219723fac2157569e0a7a76992dc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 21 Jan 2021 18:19:04 +0100 Subject: [PATCH 0336/1284] Test SQLancer also with JDK14 See https://github.com/sqlancer/sqlancer/issues/288. --- .github/workflows/main.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6db7a4e72..304fddbe2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -314,6 +314,24 @@ jobs: - name: Shortly run DuckDB run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 duckdb + java14: + name: Java 14 Compatibility (DuckDB) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 14 + uses: actions/setup-java@v1 + with: + java-version: 14 + - name: Build + run: mvn -B package -DskipTests=true + - name: Shortly run DuckDB + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 duckdb + + java15: name: Java 15 EA Compatibility (DuckDB) runs-on: ubuntu-latest From 491ea18c201bb2c2bc17002b9a257d688875614b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 21 Jan 2021 18:57:42 +0100 Subject: [PATCH 0337/1284] Fix the release GPG signing See https://stackoverflow.com/questions/14825039/suppressing-gpg-signing-for-maven-based-continuous-integration-builds-travis-ci. --- .github/workflows/maven-central.yml | 2 +- pom.xml | 48 ++++++++++++++++------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml index ffa28cdd7..be6b963d3 100644 --- a/.github/workflows/maven-central.yml +++ b/.github/workflows/maven-central.yml @@ -18,7 +18,7 @@ jobs: - name: Install gpg secret key run: cat <(echo -e "${{ secrets.OSSRH_GPG_SECRET_KEY }}") | gpg --batch --import - name: Publish package - run: mvn --batch-mode deploy -DskipTests=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} + run: mvn --batch-mode deploy -DskipTests=true -DreleaseBuild=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} env: MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} diff --git a/pom.xml b/pom.xml index 35edcd735..5ec8d0588 100644 --- a/pom.xml +++ b/pom.xml @@ -41,27 +41,6 @@ src test - - org.apache.maven.plugins - maven-gpg-plugin - 1.6 - - true - - --pinentry-mode - loopback - - - - - sign-artifacts - deploy - - sign - - - - org.apache.maven.plugins maven-javadoc-plugin @@ -339,5 +318,32 @@ ${java.home}/bin/javadoc + + release-sign-artifacts + + + releaseBuild + true + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + + From c48c3536cb87a660a0dee2e25b8e570d209e2f72 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 21 Jan 2021 20:14:06 +0100 Subject: [PATCH 0338/1284] Fix a signing issue Fixes the "gpg: signing failed: Inappropriate ioctl for device" error. --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 5ec8d0588..acb9c0ec3 100644 --- a/pom.xml +++ b/pom.xml @@ -339,6 +339,12 @@ sign + + + --pinentry-mode + loopback + + From b3ed42b95eddf9322bf0b56807a984b9d580a048 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 21 Jan 2021 21:18:08 +0100 Subject: [PATCH 0339/1284] Create a source JAR for IDEs This fixes a "Missing: no sources jar found in folder '/com/sqlancer/sqlancer/1.0'" error on https://oss.sonatype.org/. --- pom.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pom.xml b/pom.xml index acb9c0ec3..1f5fb90eb 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,19 @@ src test + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar-no-fork + + + + org.apache.maven.plugins maven-javadoc-plugin From f634ee63e17d8366677b927c4016939f0c7dc3ce Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 22 Jan 2021 18:15:03 +0300 Subject: [PATCH 0340/1284] [ClickHouse] Apply setting to original query also --- .../clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 45878446b..7bf1f01a7 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -48,6 +48,7 @@ public void check() throws SQLException { select.setOrderByExpressions(gen.generateOrderBys()); } String originalQuery = ClickHouseVisitor.asString(select); + originalQuery += " SETTINGS aggregate_functions_null_for_empty = 1"; ClickHouseExpression whereClause = gen .generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); From ee7c4eef45873ac6ca925ce76d5b9aefdf4049fd Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 22 Jan 2021 20:42:04 +0100 Subject: [PATCH 0341/1284] Build the javadoc and sources JARs only for release builds --- pom.xml | 60 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index 1f5fb90eb..b510f5b90 100644 --- a/pom.xml +++ b/pom.xml @@ -41,35 +41,6 @@ src test - - org.apache.maven.plugins - maven-source-plugin - 3.2.1 - - - attach-sources - - jar-no-fork - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.2.0 - - 8 - - - - attach-javadocs - - jar - - - - org.apache.maven.plugins maven-surefire-plugin @@ -332,7 +303,7 @@ - release-sign-artifacts + release-steps releaseBuild @@ -341,6 +312,35 @@ + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + 8 + + + + attach-javadocs + + jar + + + + org.apache.maven.plugins maven-gpg-plugin From 3d4da2bdceb7fad55a1ef9bb1620c31ca8da616a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 22 Jan 2021 20:47:05 +0100 Subject: [PATCH 0342/1284] Execute the Java tests with --num-queries 0 This is to prevent that DuckDB logic bugs affect this test. --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 304fddbe2..251d7e7cc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -295,7 +295,7 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 duckdb + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb java13: name: Java 13 Compatibility (DuckDB) @@ -312,7 +312,7 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 duckdb + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb java14: name: Java 14 Compatibility (DuckDB) @@ -329,7 +329,7 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 duckdb + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb java15: @@ -347,4 +347,4 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 duckdb + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb From ad2f3428afc379c67eb52f12114d6a70905f8bb1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 22 Jan 2021 21:27:50 +0100 Subject: [PATCH 0343/1284] Fix the SCM link in pom.xml --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index b510f5b90..d77d9c9cf 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.sqlancer sqlancer - 1.0 + 1.0.1 SQLancer http://www.sqlancer.com/ SQLancer finds logic bugs in Database Management Systems through automatic testing @@ -30,9 +30,9 @@ - git@github.com:sqlancer/sqlancer.git - scm:git:git@github.com:sqlancer/sqlancer.git - scm:git:git@github.com:sqlancer/sqlancer.git + https://github.com/sqlancer/sqlancer/ + scm:git:git://github.com/sqlancer/sqlancer.git + scm:git:ssh://github.com:sqlancer/sqlancer.git UTF-8 From ec1842bb1087ee218aa6ce36b3fe8f1cfaabac97 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 23 Jan 2021 10:53:44 +0100 Subject: [PATCH 0344/1284] Disable manually pushing releases to Maven Central --- .github/workflows/maven-central.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven-central.yml b/.github/workflows/maven-central.yml index be6b963d3..6eabf6e64 100644 --- a/.github/workflows/maven-central.yml +++ b/.github/workflows/maven-central.yml @@ -1,6 +1,5 @@ name: Publish package to the Maven Central Repository on: - workflow_dispatch: release: types: [created] jobs: From ada3c7f7cee967145c0c767f5df0bc3b8f1a0372 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 24 Jan 2021 10:08:05 +0100 Subject: [PATCH 0345/1284] Publish releases also at Docker Hub --- .../{maven-central.yml => release.yml} | 21 ++++++++++++++++++- Dockerfile | 9 ++++++++ pom.xml | 2 +- 3 files changed, 30 insertions(+), 2 deletions(-) rename .github/workflows/{maven-central.yml => release.yml} (53%) create mode 100644 Dockerfile diff --git a/.github/workflows/maven-central.yml b/.github/workflows/release.yml similarity index 53% rename from .github/workflows/maven-central.yml rename to .github/workflows/release.yml index 6eabf6e64..7777bdb6c 100644 --- a/.github/workflows/maven-central.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Publish package to the Maven Central Repository +name: Publish package to the Maven Central Repository and Docker Hub on: release: types: [created] @@ -21,3 +21,22 @@ jobs: env: MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Push to Docker Hub + uses: docker/build-push-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: mrigger/sqlancer + tag_with_ref: true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..54f3aacaf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:21.04 + +RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install openjdk-15-jdk maven --yes --no-install-recommends + +# assumes that the project has already been built +COPY target/sqlancer-*.jar sqlancer.jar +COPY target/lib/*.jar /lib/ + +ENTRYPOINT ["java", "-jar", "sqlancer.jar"] diff --git a/pom.xml b/pom.xml index d77d9c9cf..cad27b03c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.sqlancer sqlancer - 1.0.1 + 1.1.0 SQLancer http://www.sqlancer.com/ SQLancer finds logic bugs in Database Management Systems through automatic testing From 72f6a3910220bb454f2a9e3961a1e5cb16669639 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 24 Jan 2021 10:22:36 +0100 Subject: [PATCH 0346/1284] Fix a new expected Postgres error This fixes a "ERROR: child column "c0" specifies generation expression" error that occurred in the CI checks. --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 823cc916b..05e1e62ba 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -198,6 +198,7 @@ private void generateInherits() { errors.add("cannot inherit from partitioned table"); errors.add("has a collation conflict"); errors.add("inherits conflicting default values"); + errors.add("specifies generation expression"); } } From 06d681c6f0b37caf70a5e796d843c8a3deaf1603 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 24 Jan 2021 11:04:21 +0100 Subject: [PATCH 0347/1284] Document the available release build platforms --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 2836ff423..9559ac048 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,13 @@ We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_inv * [Contributing to SQLancer](CONTRIBUTING.md) * [Papers and .bib entries](docs/PAPERS.md) +# Releases + +Official release are available on: +* [GitHub](https://github.com/sqlancer/sqlancer/releases) +* [Maven Central](https://search.maven.org/artifact/com.sqlancer/sqlancer) +* [DockerHub](https://hub.docker.com/r/mrigger/sqlancer) + # Additional Resources * A talk on Ternary Logic Partitioning (TLP) and SQLancer is available on [YouTube](https://www.youtube.com/watch?v=Np46NQ6lqP8). From 3edd151de21ae6bf338431c37761a1b3634f1ed6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 06:46:43 +0000 Subject: [PATCH 0348/1284] Bump clickhouse-jdbc from 0.2.4 to 0.2.5 Bumps [clickhouse-jdbc](https://github.com/ClickHouse/clickhouse-jdbc) from 0.2.4 to 0.2.5. - [Release notes](https://github.com/ClickHouse/clickhouse-jdbc/releases) - [Changelog](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/ClickHouse/clickhouse-jdbc/compare/release_0.2.4...v0.2.5) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cad27b03c..1391ab0cc 100644 --- a/pom.xml +++ b/pom.xml @@ -259,7 +259,7 @@ ru.yandex.clickhouse clickhouse-jdbc - 0.2.4 + 0.2.5 com.h2database From 61d3f411b60ebdfdda0b0da6f3528678812fdfd3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 09:36:23 +0000 Subject: [PATCH 0349/1284] Bump mariadb-java-client from 2.7.1 to 2.7.2 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.7.1 to 2.7.2. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/2.7.1...2.7.2) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1391ab0cc..4e76eb144 100644 --- a/pom.xml +++ b/pom.xml @@ -238,7 +238,7 @@ org.mariadb.jdbc mariadb-java-client - 2.7.1 + 2.7.2 org.duckdb From a18516d9ef53c458ae451556ccf28f0064643239 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 14:21:15 +0000 Subject: [PATCH 0350/1284] Bump maven-checkstyle-plugin from 3.1.1 to 3.1.2 Bumps [maven-checkstyle-plugin](https://github.com/apache/maven-checkstyle-plugin) from 3.1.1 to 3.1.2. - [Release notes](https://github.com/apache/maven-checkstyle-plugin/releases) - [Commits](https://github.com/apache/maven-checkstyle-plugin/compare/maven-checkstyle-plugin-3.1.1...maven-checkstyle-plugin-3.1.2) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4e76eb144..298f2e752 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.1.1 + 3.1.2 configs/checkstyle.xml From b6d99d470757993cd95a545c11cb88cd0de06f68 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 3 Feb 2021 07:17:36 +0000 Subject: [PATCH 0351/1284] Bump duckdb_jdbc from 0.2.3 to 0.2.4 Bumps [duckdb_jdbc](https://github.com/cwida/duckdb) from 0.2.3 to 0.2.4. - [Release notes](https://github.com/cwida/duckdb/releases) - [Commits](https://github.com/cwida/duckdb/compare/v0.2.3...v0.2.4) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 298f2e752..3dea1ef15 100644 --- a/pom.xml +++ b/pom.xml @@ -243,7 +243,7 @@ org.duckdb duckdb_jdbc - 0.2.3 + 0.2.4 org.junit.jupiter From c9d46e3cb438c4e64dea62ee10f8fde827046ff0 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sat, 6 Feb 2021 17:36:36 +0100 Subject: [PATCH 0352/1284] Add intellij files to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 3d3378a71..dcec833f6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ target/ .project .checkstyle *.DS_Store +.idea +SQLancer.iml From b7501c7a86066d430c87bcdad79a1a50452dcad5 Mon Sep 17 00:00:00 2001 From: Ethan-DB Date: Tue, 9 Mar 2021 12:15:04 +0800 Subject: [PATCH 0353/1284] suuport host/port arguments for postgres --- src/sqlancer/MainOptions.java | 15 +++++++++++++++ src/sqlancer/postgres/PostgresProvider.java | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 144826298..122bc595f 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -49,6 +49,13 @@ public class MainOptions { @Parameter(names = "--password", description = "The password used to log into the DBMS") private String password = "sqlancer"; // NOPMD + @Parameter(names = "--host", description = "The host used to log into the DBMS") + private String host = "sqlancer"; // NOPMD + + @Parameter(names = "--port", description = "The port used to log into the DBMS") + private String port = "sqlancer"; // NOPMD + + @Parameter(names = "--print-progress-information", description = "Whether to print progress information such as the number of databases generated or queries issued", arity = 1) private boolean printProgressInformation = true; // NOPMD @@ -151,6 +158,14 @@ public String getPassword() { return password; } + public String getHost(){ + return host; + } + + public String getPort(){ + return port; + } + public boolean printProgressInformation() { return printProgressInformation; } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index c5d295611..d134866b0 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -54,6 +54,8 @@ public class PostgresProvider extends SQLProviderAdapter Date: Tue, 9 Mar 2021 14:04:53 +0800 Subject: [PATCH 0354/1284] support host/port arguments for MySQL --- src/sqlancer/mysql/MySQLProvider.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index dd51f4982..19dc5db70 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -31,6 +31,10 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; public class MySQLProvider extends SQLProviderAdapter { + protected String username; + protected String password; + protected String host; + protected String port; public MySQLProvider() { super(MySQLGlobalState.class, MySQLOptions.class); @@ -149,12 +153,21 @@ public void generateDatabase(MySQLGlobalState globalState) throws Exception { @Override public SQLConnection createDatabase(MySQLGlobalState globalState) throws SQLException { + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + if("sqlancer".equals(host)){ + host = "localhost"; + } + if("sqlancer".equals(port)){ + port = "3306"; + } String databaseName = globalState.getDatabaseName(); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); globalState.getState().logStatement("CREATE DATABASE " + databaseName); globalState.getState().logStatement("USE " + databaseName); - String url = "jdbc:mysql://localhost:3306/?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true"; - Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), + String url = "jdbc:mysql://" + host + ":" + port + "?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true"; + System.out.println("url: "+ url); + Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); From 96f22a058cecdac44c18137393df202f98fe3a63 Mon Sep 17 00:00:00 2001 From: Ethan-DB Date: Tue, 9 Mar 2021 15:05:09 +0800 Subject: [PATCH 0355/1284] support host/port arguments for mariadb --- src/sqlancer/mariadb/MariaDBProvider.java | 20 +++++++++++++++++--- src/sqlancer/mysql/MySQLProvider.java | 6 +++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 602eaccae..2884d880f 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -27,6 +27,10 @@ public class MariaDBProvider extends SQLProviderAdapter { public static final int MAX_EXPRESSION_DEPTH = 3; + protected String username; + protected String password; + protected String host; + protected String port; public MariaDBProvider() { super(MariaDBGlobalState.class, MariaDBOptions.class); @@ -168,9 +172,19 @@ public SQLConnection createDatabase(MariaDBGlobalState globalState) throws SQLEx globalState.getState().logStatement("CREATE DATABASE " + globalState.getDatabaseName()); globalState.getState().logStatement("USE " + globalState.getDatabaseName()); // /?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true - String url = "jdbc:mariadb://localhost:3306"; - Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); + // String url = "jdbc:mariadb://localhost:3306"; + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + if("sqlancer".equals(host)){ + host = "localhost"; + } + if("sqlancer".equals(port)){ + port = "3306"; + } + String url = "jdbc:mariadb://" + host + ":" + port; + Connection con = DriverManager.getConnection(url, username, password); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + globalState.getDatabaseName()); } diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 19dc5db70..5b094a3ef 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -153,6 +153,8 @@ public void generateDatabase(MySQLGlobalState globalState) throws Exception { @Override public SQLConnection createDatabase(MySQLGlobalState globalState) throws SQLException { + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); if("sqlancer".equals(host)){ @@ -166,9 +168,7 @@ public SQLConnection createDatabase(MySQLGlobalState globalState) throws SQLExce globalState.getState().logStatement("CREATE DATABASE " + databaseName); globalState.getState().logStatement("USE " + databaseName); String url = "jdbc:mysql://" + host + ":" + port + "?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true"; - System.out.println("url: "+ url); - Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); + Connection con = DriverManager.getConnection(url, username, password); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); } From 3051257cdeef232ef18edbb6fc2161212626d482 Mon Sep 17 00:00:00 2001 From: Ethan-DB Date: Tue, 9 Mar 2021 15:16:47 +0800 Subject: [PATCH 0356/1284] support host/port arguments for clickhouse --- src/sqlancer/clickhouse/ClickHouseProvider.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index c45db308c..6cc75425d 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -21,6 +21,8 @@ import sqlancer.common.query.SQLQueryProvider; public class ClickHouseProvider extends SQLProviderAdapter { + protected String host; + protected String port; public ClickHouseProvider() { super(ClickHouseGlobalState.class, ClickHouseOptions.class); @@ -102,9 +104,19 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws Exception @Override public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQLException { + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + if("sqlancer".equals(host)){ + host = "localhost"; + } + if("sqlancer".equals(port)){ + port = "8123"; + } + ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); - String url = "jdbc:clickhouse://localhost:8123/default"; + //String url = "jdbc:clickhouse://localhost:8123/default"; + String url = "jdbc:clickhouse://" + host + ":" + port + "/defult"; String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); From 33b9ef957e87a84ecc02da31bff41dc336a7a27d Mon Sep 17 00:00:00 2001 From: Ethan-DB Date: Tue, 9 Mar 2021 15:38:13 +0800 Subject: [PATCH 0357/1284] support host/port arguments for cockroachdb --- src/sqlancer/cockroachdb/CockroachDBProvider.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index ccfa4b13a..f3ee7ce70 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -35,6 +35,8 @@ import sqlancer.common.query.SQLQueryProvider; public class CockroachDBProvider extends SQLProviderAdapter { + protected String host; + protected String port; public CockroachDBProvider() { super(CockroachDBGlobalState.class, CockroachDBOptions.class); @@ -248,8 +250,17 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio @Override public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws SQLException { + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + if("sqlancer".equals(host)){ + host = "localhost"; + } + if("sqlancer".equals(port)){ + port = "26257"; + } String databaseName = globalState.getDatabaseName(); - String url = "jdbc:postgresql://localhost:26257/test"; + //String url = "jdbc:postgresql://localhost:26257/test"; + String url = "jdbc:postgresql://" + host + ":" + port + "/test"; Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); globalState.getState().logStatement("USE test"); From fceb0ac62a15873465e2a1342bb3fef893db83a2 Mon Sep 17 00:00:00 2001 From: Ethan-DB Date: Tue, 9 Mar 2021 15:51:52 +0800 Subject: [PATCH 0358/1284] support host/port arguments for TiDB --- src/sqlancer/tidb/TiDBProvider.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 1e1abf2a9..a578fc0ff 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -28,6 +28,8 @@ import sqlancer.tidb.gen.TiDBViewGenerator; public class TiDBProvider extends SQLProviderAdapter { + protected String host; + protected String port; public TiDBProvider() { super(TiDBGlobalState.class, TiDBOptions.class); @@ -132,8 +134,18 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { @Override public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLException { + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + if("sqlancer".equals(host)){ + host = "127.0.0.1"; + } + if("sqlancer".equals(port)){ + port = "4000"; + } + String databaseName = globalState.getDatabaseName(); - String url = "jdbc:mysql://127.0.0.1:4000/"; + //String url = "jdbc:mysql://127.0.0.1:4000/"; + String url = "jdbc:mysql://" + host + ":" + port + "/"; Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); globalState.getState().logStatement("USE test"); From decb8ff044532b9ee1d8fe927bc65dce1aee253d Mon Sep 17 00:00:00 2001 From: Ethan-DB Date: Tue, 9 Mar 2021 16:46:45 +0800 Subject: [PATCH 0359/1284] format --- src/sqlancer/MainOptions.java | 5 ++-- .../clickhouse/ClickHouseProvider.java | 20 +++++++-------- .../cockroachdb/CockroachDBProvider.java | 20 +++++++-------- src/sqlancer/mariadb/MariaDBProvider.java | 24 +++++++++--------- src/sqlancer/mysql/MySQLProvider.java | 25 ++++++++++--------- src/sqlancer/postgres/PostgresProvider.java | 18 ++++++------- src/sqlancer/tidb/TiDBProvider.java | 20 +++++++-------- 7 files changed, 66 insertions(+), 66 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 122bc595f..43e7e5e5b 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -55,7 +55,6 @@ public class MainOptions { @Parameter(names = "--port", description = "The port used to log into the DBMS") private String port = "sqlancer"; // NOPMD - @Parameter(names = "--print-progress-information", description = "Whether to print progress information such as the number of databases generated or queries issued", arity = 1) private boolean printProgressInformation = true; // NOPMD @@ -158,11 +157,11 @@ public String getPassword() { return password; } - public String getHost(){ + public String getHost() { return host; } - public String getPort(){ + public String getPort() { return port; } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 6cc75425d..5aecc8e90 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -104,19 +104,19 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws Exception @Override public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQLException { - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); - if("sqlancer".equals(host)){ - host = "localhost"; - } - if("sqlancer".equals(port)){ - port = "8123"; - } + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + if ("sqlancer".equals(host)) { + host = "localhost"; + } + if ("sqlancer".equals(port)) { + port = "8123"; + } ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); - //String url = "jdbc:clickhouse://localhost:8123/default"; - String url = "jdbc:clickhouse://" + host + ":" + port + "/defult"; + // String url = "jdbc:clickhouse://localhost:8123/default"; + String url = "jdbc:clickhouse://" + host + ":" + port + "/defult"; String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index f3ee7ce70..579a3b67e 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -250,17 +250,17 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio @Override public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws SQLException { - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); - if("sqlancer".equals(host)){ - host = "localhost"; - } - if("sqlancer".equals(port)){ - port = "26257"; - } + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + if ("sqlancer".equals(host)) { + host = "localhost"; + } + if ("sqlancer".equals(port)) { + port = "26257"; + } String databaseName = globalState.getDatabaseName(); - //String url = "jdbc:postgresql://localhost:26257/test"; - String url = "jdbc:postgresql://" + host + ":" + port + "/test"; + // String url = "jdbc:postgresql://localhost:26257/test"; + String url = "jdbc:postgresql://" + host + ":" + port + "/test"; Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); globalState.getState().logStatement("USE test"); diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 2884d880f..3516e2b1b 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -30,7 +30,7 @@ public class MariaDBProvider extends SQLProviderAdapter Date: Tue, 9 Mar 2021 17:27:32 +0800 Subject: [PATCH 0360/1284] modify format --- src/sqlancer/postgres/PostgresProvider.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 0be9609d4..237aa1588 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -55,7 +55,6 @@ public class PostgresProvider extends SQLProviderAdapter Date: Tue, 9 Mar 2021 18:53:02 +0800 Subject: [PATCH 0361/1284] Update TiDBProvider.java --- src/sqlancer/tidb/TiDBProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 2d0052748..40ef93868 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -137,7 +137,7 @@ public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLExcep host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); if ("sqlancer".equals(host)) { - host = "127.0.0.1"; + host = "localhost"; } if ("sqlancer".equals(port)) { port = "4000"; From 7a3ebf4795cd15373e2305bb57c052e9a190ca5c Mon Sep 17 00:00:00 2001 From: Ethan-DB <13810907+Ethan-DB@users.noreply.github.com> Date: Fri, 12 Mar 2021 15:05:26 +0800 Subject: [PATCH 0362/1284] Update Main.java --- src/sqlancer/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 755ce36a8..3f5b92f3a 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -453,7 +453,7 @@ private String formatInteger(long intValue) { .testConnection(); } catch (Exception e) { System.err.println( - "SQLancer failed creating a test database, indicating that SQLancer might have failed connecting to the DBMS. In order to change the username and password, you can use the --username and --password options. Currently, SQLancer does not yet support passing a host and port (see https://github.com/sqlancer/sqlancer/issues/95).\n\n"); + "SQLancer failed creating a test database, indicating that SQLancer might have failed connecting to the DBMS. In order to change the username, password, host and port, you can use the --username, --password, --host and --port options.\n\n"); e.printStackTrace(); return options.getErrorExitCode(); } From 03f8d6fc68d3799cf67249017d9ab6fc96351401 Mon Sep 17 00:00:00 2001 From: Ethan-DB <13810907+Ethan-DB@users.noreply.github.com> Date: Wed, 17 Mar 2021 11:11:01 +0800 Subject: [PATCH 0363/1284] modify typo error --- src/sqlancer/clickhouse/ClickHouseProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 5aecc8e90..2c3d910c0 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -116,7 +116,7 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); // String url = "jdbc:clickhouse://localhost:8123/default"; - String url = "jdbc:clickhouse://" + host + ":" + port + "/defult"; + String url = "jdbc:clickhouse://" + host + ":" + port + "/default"; String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); From 17e79ae4d0e57f3b1838b734e1ccc4875372c54c Mon Sep 17 00:00:00 2001 From: Ethan-DB Date: Thu, 18 Mar 2021 11:53:49 +0800 Subject: [PATCH 0364/1284] using String.format replace + in String --- src/sqlancer/clickhouse/ClickHouseProvider.java | 3 ++- src/sqlancer/cockroachdb/CockroachDBProvider.java | 3 ++- src/sqlancer/mariadb/MariaDBProvider.java | 3 ++- src/sqlancer/mysql/MySQLProvider.java | 6 ++++-- src/sqlancer/postgres/PostgresProvider.java | 2 +- src/sqlancer/tidb/TiDBProvider.java | 3 ++- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 2c3d910c0..85f2361fa 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -116,7 +116,8 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); // String url = "jdbc:clickhouse://localhost:8123/default"; - String url = "jdbc:clickhouse://" + host + ":" + port + "/default"; + String url = String.format("jdbc:clickhouse://%s:%s/default", host, port); + // String url = "jdbc:clickhouse://" + host + ":" + port + "/default"; String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 579a3b67e..3e17a3c8d 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -260,7 +260,8 @@ public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws S } String databaseName = globalState.getDatabaseName(); // String url = "jdbc:postgresql://localhost:26257/test"; - String url = "jdbc:postgresql://" + host + ":" + port + "/test"; + // String url = "jdbc:postgresql://" + host + ":" + port + "/test"; + String url = String.format("jdbc:postgresql://%s:%s/test", host, port); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); globalState.getState().logStatement("USE test"); diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 3516e2b1b..9f87f5e32 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -183,7 +183,8 @@ public SQLConnection createDatabase(MariaDBGlobalState globalState) throws SQLEx if ("sqlancer".equals(port)) { port = "3306"; } - String url = "jdbc:mariadb://" + host + ":" + port; + // String url = "jdbc:mariadb://" + host + ":" + port; + String url = String.format("jdbc:mariadb://%s:%s", host, port); Connection con = DriverManager.getConnection(url, username, password); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + globalState.getDatabaseName()); diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 8e264bb46..a173eb55b 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -167,8 +167,10 @@ public SQLConnection createDatabase(MySQLGlobalState globalState) throws SQLExce globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); globalState.getState().logStatement("CREATE DATABASE " + databaseName); globalState.getState().logStatement("USE " + databaseName); - String url = "jdbc:mysql://" + host + ":" + port - + "?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true"; + // String url = "jdbc:mysql://" + host + ":" + port + // + "?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true"; + String url = String.format("jdbc:mysql://%s:%s?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); Connection con = DriverManager.getConnection(url, username, password); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 237aa1588..5e3872dc5 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -240,7 +240,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE if ("sqlancer".equals(port)) { port = Integer.toString(uri.getPort()); } - entryURL = uri.getScheme() + "://" + host + ":" + port + "/" + entryDatabaseName; + entryURL = String.format("%s://%s:%s/%s", uri.getScheme(), host, port, entryDatabaseName); } catch (URISyntaxException e) { throw new AssertionError(e); } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 40ef93868..20ba6223c 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -145,7 +145,8 @@ public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLExcep String databaseName = globalState.getDatabaseName(); // String url = "jdbc:mysql://127.0.0.1:4000/"; - String url = "jdbc:mysql://" + host + ":" + port + "/"; + // String url = "jdbc:mysql://" + host + ":" + port + "/"; + String url = String.format("jdbc:mysql://%s:%s/", host, port); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); globalState.getState().logStatement("USE test"); From 9009a50f5b0fc00fa25aaf016f3d6be1f577267b Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 15 Mar 2021 17:39:54 -0700 Subject: [PATCH 0365/1284] Update CockroachDB to v20.2.6 This commit updates the version of CockroachDB to v20.2.6. Cluster and session settings for the CockroachDB adapter have also been updated so that they are compatible with the new version. Specifically: - Cluster setting `sql.distsql.merge_joins.enabled` was removed in 20.2. - Cluster setting `sql.parallel_scans.enabled` was removed in 20.2. - Session setting `experimental_serial_normalization` was renamed to `serial_normalization` in 20.2. - Session setting `vectorize` no longer accepts `auto` as of 20.2. Additionally, this commit sets the session setting `sql_safe_updates` to `off` for every generated session. When this is on, mutations without `WHERE` clauses caused errors. I don't think there is any reason to keep this setting enabled in the context of SQLancer. --- .github/workflows/main.yml | 6 ++-- .../gen/CockroachDBIndexGenerator.java | 2 +- ...CockroachDBSetClusterSettingGenerator.java | 12 ++++---- .../gen/CockroachDBSetSessionGenerator.java | 28 ++++++++++--------- .../gen/CockroachDBTableGenerator.java | 3 +- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 251d7e7cc..8059af59b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,11 +110,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.2.linux-amd64.tgz | tar xvz - cd cockroach-v20.1.2.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v20.2.6.linux-amd64.tgz | tar xvz + cd cockroach-v20.2.6.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v20.1.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v20.2.6.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 004c8be63..fd4dfd70d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -26,7 +26,7 @@ public void buildStatement() { errors.add("already contains column"); errors.add("violates unique constraint"); errors.add("schema change statement cannot follow a statement that has written in the same transaction"); - errors.add("https://github.com/cockroachdb/cockroach/issues/35730"); // some array types are not indexable + errors.add("and thus is not indexable"); // array types are not indexable errors.add("cannot determine type of empty array. Consider annotating with the desired type"); errors.add("incompatible IF expression"); // TODO: investigate; seems to be a bug CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 5b39c5b1f..4fb36a5d1 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -15,18 +15,16 @@ private CockroachDBSetClusterSettingGenerator() { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBClusterSetting { - COMPATOR_ENABLED("compactor.enabled", CockroachDBSetSessionGenerator::onOff), // - BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), // + COMPATOR_ENABLED("compactor.enabled", CockroachDBSetSessionGenerator::onOff), + BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", - (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), // - RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), // + (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), + RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), SQL_DEFAULTS_VECTORIZE_ROW_COUNT_THRESHOLD("sql.defaults.vectorize_row_count_threshold", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), // SQL_DEFAULTS_EXPERIMENTAL_OPTIMIZER_FOREIGN_KEYS_ENABLED("sql.defaults.experimental_optimizer_foreign_keys.enabled", // CockroachDBSetSessionGenerator::onOff), - MERGE_JOINS_ENABLED("sql.distsql.merge_joins.enabled", CockroachDBSetSessionGenerator::onOff), // - PARALLEL_SCANS_ENABLED("sql.parallel_scans.enabled", CockroachDBSetSessionGenerator::onOff), // - SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), // + SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), SQL_STATS_HISTOGRAM_COLLECTION_ENABLED("sql.stats.histogram_collection.enabled", CockroachDBSetSessionGenerator::onOff); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index e3d9bd147..8aa26e88c 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -19,23 +19,25 @@ public static String onOff(CockroachDBGlobalState globalState) { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBSetting { - BYTEA_OUTPUT((g) -> Randomly.fromOptions("hex", "escape", "base64")), // - DEFAULT_INT_SIZE((g) -> Randomly.fromOptions(4, 8)), // - DISTSQL((g) -> Randomly.fromOptions("on", "off", "auto", "always")), // - ENABLE_IMPLICIT_SELECT_FOR_UPDATE(CockroachDBSetSessionGenerator::onOff), // - ENABLE_INSERT_FAST_PATH(CockroachDBSetSessionGenerator::onOff), // + BYTEA_OUTPUT((g) -> Randomly.fromOptions("hex", "escape", "base64")), + DEFAULT_INT_SIZE((g) -> Randomly.fromOptions(4, 8)), + DISTSQL((g) -> Randomly.fromOptions("on", "off", "auto", "always")), + ENABLE_IMPLICIT_SELECT_FOR_UPDATE(CockroachDBSetSessionGenerator::onOff), + ENABLE_INSERT_FAST_PATH(CockroachDBSetSessionGenerator::onOff), ENABLE_ZIGZAG_JOIN(CockroachDBSetSessionGenerator::onOff), // EXPERIMENTAL_ENABLE_HASH_SHARDED_INDEXES(CockroachDBSetSessionGenerator::onOff), - EXPERIMENTAL_SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), // - EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), // + SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), + EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), REORDER_JOINS_LIMIT((g) -> g.getRandomly().getInteger(0, Integer.MAX_VALUE)), // - SQL_SAFE_UPDATES(CockroachDBSetSessionGenerator::onOff), + SQL_SAFE_UPDATES((g) -> "off"), // TRACING(CockroachDBSetSessionGenerator::onOff) - VECTORIZE((g) -> Randomly.fromOptions("auto", "on", - "off")); /* - * see https://github.com/cockroachdb/cockroach/issues/44133, - * https://github.com/cockroachdb/cockroach/issues/44207 - */ + /* + * CockroachDB enables vectorized (column-oriented) execution by default. Row-oriented execution can be enforced + * by setting vectorized to "off". Some examples of bugs found in the vectorized execution engine are: + * https://github.com/cockroachdb/cockroach/issues/44133 https://github.com/cockroachdb/cockroach/issues/44207 + * + */ + VECTORIZE((g) -> Randomly.fromOptions("on", "off")); private Function f; diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index 22a863b78..a33b7521f 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -34,7 +34,7 @@ public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { @Override public void buildStatement() { - errors.add("https://github.com/cockroachdb/cockroach/issues/35730"); // not indexable array types + errors.add("and thus is not indexable"); // array types are not indexable if (globalState.getDmbsSpecificOptions().testTempTables) { errors.add("constraints on temporary tables may reference only temporary tables"); errors.add("constraints on permanent tables may reference only permanent tables"); @@ -75,6 +75,7 @@ public void buildStatement() { sb.append(CockroachDBVisitor.asString(gen.generateExpression(cockroachDBColumn.getType()))); sb.append(") STORED"); errors.add("computed columns cannot reference other computed columns"); + errors.add("context-dependent operators are not allowed in computed column"); errors.add("has type unknown"); } if (Randomly.getBooleanWithRatherLowProbability()) { From 4b4562996d152ff17d9443eac2b77faaa7796e8b Mon Sep 17 00:00:00 2001 From: Ethan-DB Date: Sat, 20 Mar 2021 15:50:21 +0800 Subject: [PATCH 0366/1284] adding the help text and modifing the default values of host and port --- src/sqlancer/MainOptions.java | 6 +++--- src/sqlancer/clickhouse/ClickHouseOptions.java | 2 +- src/sqlancer/clickhouse/ClickHouseProvider.java | 10 +++++----- src/sqlancer/cockroachdb/CockroachDBOptions.java | 2 +- src/sqlancer/cockroachdb/CockroachDBProvider.java | 10 +++++----- src/sqlancer/mariadb/MariaDBOptions.java | 2 +- src/sqlancer/mariadb/MariaDBProvider.java | 10 +++++----- src/sqlancer/mysql/MySQLOptions.java | 2 +- src/sqlancer/mysql/MySQLProvider.java | 10 +++++----- src/sqlancer/postgres/PostgresOptions.java | 2 +- src/sqlancer/postgres/PostgresProvider.java | 10 +++++----- src/sqlancer/tidb/TiDBOptions.java | 2 +- src/sqlancer/tidb/TiDBProvider.java | 10 +++++----- 13 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 43e7e5e5b..73312f736 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -50,10 +50,10 @@ public class MainOptions { private String password = "sqlancer"; // NOPMD @Parameter(names = "--host", description = "The host used to log into the DBMS") - private String host = "sqlancer"; // NOPMD + private String host = null; // NOPMD @Parameter(names = "--port", description = "The port used to log into the DBMS") - private String port = "sqlancer"; // NOPMD + private int port = -1; // NOPMD @Parameter(names = "--print-progress-information", description = "Whether to print progress information such as the number of databases generated or queries issued", arity = 1) private boolean printProgressInformation = true; // NOPMD @@ -161,7 +161,7 @@ public String getHost() { return host; } - public String getPort() { + public int getPort() { return port; } diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 5ca84871e..ed983f787 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -18,7 +18,7 @@ import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; import sqlancer.common.oracle.TestOracle; -@Parameters(separators = "=", commandDescription = "ClickHouse") +@Parameters(separators = "=", commandDescription = "ClickHouse (default port: 8123, default host: localhost)") public class ClickHouseOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 85f2361fa..255a6ff39 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -22,7 +22,7 @@ public class ClickHouseProvider extends SQLProviderAdapter { protected String host; - protected String port; + protected int port; public ClickHouseProvider() { super(ClickHouseGlobalState.class, ClickHouseOptions.class); @@ -106,17 +106,17 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws Exception public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQLException { host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); - if ("sqlancer".equals(host)) { + if (host == null) { host = "localhost"; } - if ("sqlancer".equals(port)) { - port = "8123"; + if (port == -1) { + port = 8123; } ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); // String url = "jdbc:clickhouse://localhost:8123/default"; - String url = String.format("jdbc:clickhouse://%s:%s/default", host, port); + String url = String.format("jdbc:clickhouse://%s:%d/default", host, port); // String url = "jdbc:clickhouse://" + host + ":" + port + "/default"; String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 7772117b8..36184f1fb 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -23,7 +23,7 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; -@Parameters(separators = "=", commandDescription = "Test CockroachDB") +@Parameters(separators = "=", commandDescription = "CockroachDB (default port: 26257, default host: localhost)") public class CockroachDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 3e17a3c8d..5003e40c3 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -36,7 +36,7 @@ public class CockroachDBProvider extends SQLProviderAdapter { protected String host; - protected String port; + protected int port; public CockroachDBProvider() { super(CockroachDBGlobalState.class, CockroachDBOptions.class); @@ -252,16 +252,16 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws SQLException { host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); - if ("sqlancer".equals(host)) { + if (host == null) { host = "localhost"; } - if ("sqlancer".equals(port)) { - port = "26257"; + if (port == -1) { + port = 26257; } String databaseName = globalState.getDatabaseName(); // String url = "jdbc:postgresql://localhost:26257/test"; // String url = "jdbc:postgresql://" + host + ":" + port + "/test"; - String url = String.format("jdbc:postgresql://%s:%s/test", host, port); + String url = String.format("jdbc:postgresql://%s:%d/test", host, port); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); globalState.getState().logStatement("USE test"); diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 6037bec86..e4893708b 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -14,7 +14,7 @@ import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.oracle.MariaDBNoRECOracle; -@Parameters +@Parameters(separators = "=", commandDescription = "MariaDB (default port: 3306, default host: localhost)") public class MariaDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 9f87f5e32..6f462b744 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -30,7 +30,7 @@ public class MariaDBProvider extends SQLProviderAdapter { @Parameter(names = "--oracle") diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index a173eb55b..b9084b3d9 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -34,7 +34,7 @@ public class MySQLProvider extends SQLProviderAdapter { @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 5e3872dc5..f770017e9 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -54,7 +54,7 @@ public class PostgresProvider extends SQLProviderAdapter { @Parameter(names = "--oracle") diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 20ba6223c..d6bc786a8 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -29,7 +29,7 @@ public class TiDBProvider extends SQLProviderAdapter { protected String host; - protected String port; + protected int port; public TiDBProvider() { super(TiDBGlobalState.class, TiDBOptions.class); @@ -136,17 +136,17 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLException { host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); - if ("sqlancer".equals(host)) { + if (host == null) { host = "localhost"; } - if ("sqlancer".equals(port)) { - port = "4000"; + if (port == -1) { + port = 4000; } String databaseName = globalState.getDatabaseName(); // String url = "jdbc:mysql://127.0.0.1:4000/"; // String url = "jdbc:mysql://" + host + ":" + port + "/"; - String url = String.format("jdbc:mysql://%s:%s/", host, port); + String url = String.format("jdbc:mysql://%s:%d/", host, port); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); globalState.getState().logStatement("USE test"); From f201549848ac6764031918743ec926f5356c7c2c Mon Sep 17 00:00:00 2001 From: EthanDBer Date: Sat, 20 Mar 2021 21:24:06 +0800 Subject: [PATCH 0367/1284] constant representation for the default value of host and port --- src/sqlancer/MainOptions.java | 3 ++- src/sqlancer/clickhouse/ClickHouseOptions.java | 2 ++ src/sqlancer/clickhouse/ClickHouseProvider.java | 7 ++++--- src/sqlancer/cockroachdb/CockroachDBOptions.java | 2 ++ src/sqlancer/cockroachdb/CockroachDBProvider.java | 6 +++--- src/sqlancer/mariadb/MariaDBOptions.java | 2 ++ src/sqlancer/mariadb/MariaDBProvider.java | 6 +++--- src/sqlancer/mysql/MySQLOptions.java | 2 ++ src/sqlancer/mysql/MySQLProvider.java | 7 ++++--- src/sqlancer/postgres/PostgresOptions.java | 5 ++++- src/sqlancer/postgres/PostgresProvider.java | 3 ++- src/sqlancer/tidb/TiDBOptions.java | 2 ++ src/sqlancer/tidb/TiDBProvider.java | 7 ++++--- 13 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 73312f736..ff588de5f 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -7,6 +7,7 @@ @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { + public static final int NO_SET_PORT = -1; @Parameter(names = { "--help", "-h" }, description = "Lists all supported options and commands", help = true) private boolean help; // NOPMD @@ -53,7 +54,7 @@ public class MainOptions { private String host = null; // NOPMD @Parameter(names = "--port", description = "The port used to log into the DBMS") - private int port = -1; // NOPMD + private int port = MainOptions.NO_SET_PORT; // NOPMD @Parameter(names = "--print-progress-information", description = "Whether to print progress information such as the number of databases generated or queries issued", arity = 1) private boolean printProgressInformation = true; // NOPMD diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index ed983f787..e40488fc9 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -20,6 +20,8 @@ @Parameters(separators = "=", commandDescription = "ClickHouse (default port: 8123, default host: localhost)") public class ClickHouseOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 8123; @Parameter(names = "--oracle") public List oracle = Arrays.asList(ClickHouseOracleFactory.TLPWhere); diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 255a6ff39..ed49a3e47 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -8,6 +8,7 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; @@ -107,10 +108,10 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); if (host == null) { - host = "localhost"; + host = ClickHouseOptions.DEFAULT_HOST; } - if (port == -1) { - port = 8123; + if (port == MainOptions.NO_SET_PORT) { + port = ClickHouseOptions.DEFAULT_PORT; } ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 36184f1fb..149df7f60 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -25,6 +25,8 @@ @Parameters(separators = "=", commandDescription = "CockroachDB (default port: 26257, default host: localhost)") public class CockroachDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 26257; @Parameter(names = "--oracle") public CockroachDBOracleFactory oracle = CockroachDBOracleFactory.NOREC; diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 5003e40c3..dcfb2eb16 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -253,10 +253,10 @@ public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws S host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); if (host == null) { - host = "localhost"; + host = CockroachDBOptions.DEFAULT_HOST; } - if (port == -1) { - port = 26257; + if (port == MainOptions.NO_SET_PORT) { + port = CockroachDBOptions.DEFAULT_PORT; } String databaseName = globalState.getDatabaseName(); // String url = "jdbc:postgresql://localhost:26257/test"; diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index e4893708b..ba9d7d8b4 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -16,6 +16,8 @@ @Parameters(separators = "=", commandDescription = "MariaDB (default port: 3306, default host: localhost)") public class MariaDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 3306; @Parameter(names = "--oracle") public List oracles = Arrays.asList(MariaDBOracleFactory.NOREC); diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 6f462b744..2e9f63bf9 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -178,10 +178,10 @@ public SQLConnection createDatabase(MariaDBGlobalState globalState) throws SQLEx host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); if (host == null) { - host = "localhost"; + host = MariaDBOptions.DEFAULT_HOST; } - if (port == -1) { - port = 3306; + if (port == MainOptions.NO_SET_PORT) { + port = MariaDBOptions.DEFAULT_PORT; } // String url = "jdbc:mariadb://" + host + ":" + port; String url = String.format("jdbc:mariadb://%s:%d", host, port); diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index ca06ba6e7..377926d0f 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -16,6 +16,8 @@ @Parameters(separators = "=", commandDescription = "MySQL (default port: 3306, default host: localhost)") public class MySQLOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 3306; @Parameter(names = "--oracle") public List oracles = Arrays.asList(MySQLOracleFactory.TLP_WHERE); diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index b9084b3d9..f6e862cb2 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -7,6 +7,7 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLProviderAdapter; @@ -158,10 +159,10 @@ public SQLConnection createDatabase(MySQLGlobalState globalState) throws SQLExce host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); if (host == null) { - host = "localhost"; + host = MySQLOptions.DEFAULT_HOST; } - if (port == -1) { - port = 3306; + if (port == MainOptions.NO_SET_PORT) { + port = MySQLOptions.DEFAULT_PORT; } String databaseName = globalState.getDatabaseName(); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index d5a32e319..1614ef516 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -21,6 +21,8 @@ @Parameters(separators = "=", commandDescription = "PostgreSQL (default port: 5432, default host: localhost)") public class PostgresOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 5432; @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) public boolean allowBulkInsert; @@ -32,7 +34,8 @@ public class PostgresOptions implements DBMSSpecificOptions { NOREC { diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index f770017e9..683ba95d1 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -10,6 +10,7 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLProviderAdapter; @@ -237,7 +238,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE if (host == null) { host = uri.getHost(); } - if (port == -1) { + if (port == MainOptions.NO_SET_PORT) { port = uri.getPort(); } entryURL = String.format("%s://%s:%d/%s", uri.getScheme(), host, port, entryDatabaseName); diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index c5892c74f..b405e525d 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -19,6 +19,8 @@ @Parameters(separators = "=", commandDescription = "PostgreSQL (default port: 4000, default host: localhost)") public class TiDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 4000; @Parameter(names = "--oracle") public List oracle = Arrays.asList(TiDBOracleFactory.QUERY_PARTITIONING); diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index d6bc786a8..9726f3a38 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -7,6 +7,7 @@ import sqlancer.AbstractAction; import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; @@ -137,10 +138,10 @@ public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLExcep host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); if (host == null) { - host = "localhost"; + host = TiDBOptions.DEFAULT_HOST; } - if (port == -1) { - port = 4000; + if (port == MainOptions.NO_SET_PORT) { + port = TiDBOptions.DEFAULT_PORT; } String databaseName = globalState.getDatabaseName(); From dc2bc4485ee5cb34cc712e246d492a2e558b2f0f Mon Sep 17 00:00:00 2001 From: EthanDBer Date: Sat, 20 Mar 2021 23:11:58 +0800 Subject: [PATCH 0368/1284] delete comments --- src/sqlancer/clickhouse/ClickHouseProvider.java | 2 -- src/sqlancer/cockroachdb/CockroachDBProvider.java | 2 -- src/sqlancer/mariadb/MariaDBProvider.java | 3 --- src/sqlancer/mysql/MySQLProvider.java | 2 -- src/sqlancer/tidb/TiDBProvider.java | 2 -- 5 files changed, 11 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index ed49a3e47..eba287ac6 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -116,9 +116,7 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); - // String url = "jdbc:clickhouse://localhost:8123/default"; String url = String.format("jdbc:clickhouse://%s:%d/default", host, port); - // String url = "jdbc:clickhouse://" + host + ":" + port + "/default"; String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index dcfb2eb16..0093ccaac 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -259,8 +259,6 @@ public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws S port = CockroachDBOptions.DEFAULT_PORT; } String databaseName = globalState.getDatabaseName(); - // String url = "jdbc:postgresql://localhost:26257/test"; - // String url = "jdbc:postgresql://" + host + ":" + port + "/test"; String url = String.format("jdbc:postgresql://%s:%d/test", host, port); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 2e9f63bf9..b95e7d63e 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -171,8 +171,6 @@ public SQLConnection createDatabase(MariaDBGlobalState globalState) throws SQLEx globalState.getState().logStatement("DROP DATABASE IF EXISTS " + globalState.getDatabaseName()); globalState.getState().logStatement("CREATE DATABASE " + globalState.getDatabaseName()); globalState.getState().logStatement("USE " + globalState.getDatabaseName()); - // /?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true - // String url = "jdbc:mariadb://localhost:3306"; username = globalState.getOptions().getUserName(); password = globalState.getOptions().getPassword(); host = globalState.getOptions().getHost(); @@ -183,7 +181,6 @@ public SQLConnection createDatabase(MariaDBGlobalState globalState) throws SQLEx if (port == MainOptions.NO_SET_PORT) { port = MariaDBOptions.DEFAULT_PORT; } - // String url = "jdbc:mariadb://" + host + ":" + port; String url = String.format("jdbc:mariadb://%s:%d", host, port); Connection con = DriverManager.getConnection(url, username, password); try (Statement s = con.createStatement()) { diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index f6e862cb2..92b13839d 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -168,8 +168,6 @@ public SQLConnection createDatabase(MySQLGlobalState globalState) throws SQLExce globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); globalState.getState().logStatement("CREATE DATABASE " + databaseName); globalState.getState().logStatement("USE " + databaseName); - // String url = "jdbc:mysql://" + host + ":" + port - // + "?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true"; String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", host, port); Connection con = DriverManager.getConnection(url, username, password); diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 9726f3a38..32d5c684c 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -145,8 +145,6 @@ public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLExcep } String databaseName = globalState.getDatabaseName(); - // String url = "jdbc:mysql://127.0.0.1:4000/"; - // String url = "jdbc:mysql://" + host + ":" + port + "/"; String url = String.format("jdbc:mysql://%s:%d/", host, port); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); From 38f1972d41e68d254774b8d1f42b8c26c1920ad0 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sun, 3 Jan 2021 11:39:42 +0100 Subject: [PATCH 0369/1284] Introduce MongoDB to SQLancer and support creating collections Adding support for mongodb database version 4.4. With this commit random collections are created with optional validation options which can enforce the schema on inserts. Additionally the execution is logged in mongo shell code. --- pom.xml | 5 + src/sqlancer/GlobalState.java | 6 +- src/sqlancer/Main.java | 4 +- .../TernaryLogicPartitioningOracleBase.java | 4 +- src/sqlancer/mongodb/MongoDBConnection.java | 31 +++++ .../mongodb/MongoDBLoggableFactory.java | 40 ++++++ src/sqlancer/mongodb/MongoDBOptions.java | 40 ++++++ src/sqlancer/mongodb/MongoDBProvider.java | 107 ++++++++++++++++ src/sqlancer/mongodb/MongoDBQueryAdapter.java | 21 ++++ .../mongodb/MongoDBQueryProvider.java | 6 + src/sqlancer/mongodb/MongoDBSchema.java | 117 ++++++++++++++++++ .../mongodb/ast/MongoDBExpression.java | 4 + .../mongodb/gen/MongoDBTableGenerator.java | 49 ++++++++ .../query/MongoDBCreateTableQuery.java | 109 ++++++++++++++++ .../test/MongoDBQueryPartitioningBase.java | 26 ++++ .../MongoDBQueryPartitioningWhereTester.java | 9 ++ 16 files changed, 572 insertions(+), 6 deletions(-) create mode 100644 src/sqlancer/mongodb/MongoDBConnection.java create mode 100644 src/sqlancer/mongodb/MongoDBLoggableFactory.java create mode 100644 src/sqlancer/mongodb/MongoDBOptions.java create mode 100644 src/sqlancer/mongodb/MongoDBProvider.java create mode 100644 src/sqlancer/mongodb/MongoDBQueryAdapter.java create mode 100644 src/sqlancer/mongodb/MongoDBQueryProvider.java create mode 100644 src/sqlancer/mongodb/MongoDBSchema.java create mode 100644 src/sqlancer/mongodb/ast/MongoDBExpression.java create mode 100644 src/sqlancer/mongodb/gen/MongoDBTableGenerator.java create mode 100644 src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java create mode 100644 src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java create mode 100644 src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java diff --git a/pom.xml b/pom.xml index 3dea1ef15..c0f0bbd9b 100644 --- a/pom.xml +++ b/pom.xml @@ -266,6 +266,11 @@ h2 1.4.200 + + org.mongodb + mongodb-driver-sync + 4.1.1 + diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 97c5a7a9d..642c9da6a 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -89,13 +89,13 @@ private ExecutionTimer executePrologue(Query q) throws Exception { timer = new ExecutionTimer().start(); } if (getOptions().printAllStatements()) { - System.out.println(q.getQueryString()); + System.out.println(q.getLogString()); } if (getOptions().logEachSelect()) { if (logExecutionTime) { - getLogger().writeCurrentNoLineBreak(q.getQueryString()); + getLogger().writeCurrentNoLineBreak(q.getLogString()); } else { - getLogger().writeCurrent(q.getQueryString()); + getLogger().writeCurrent(q.getLogString()); } } return timer; diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 755ce36a8..d606ee0f5 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -30,6 +30,7 @@ import sqlancer.duckdb.DuckDBProvider; import sqlancer.h2.H2Provider; import sqlancer.mariadb.MariaDBProvider; +import sqlancer.mongodb.MongoDBProvider; import sqlancer.mysql.MySQLProvider; import sqlancer.postgres.PostgresProvider; import sqlancer.sqlite3.SQLite3Provider; @@ -209,7 +210,7 @@ private void printState(FileWriter writer, StateToReproduce state) { .getInfo(state.getDatabaseName(), state.getDatabaseVersion(), state.getSeedValue()).getLogString()); for (Query s : state.getStatements()) { - sb.append(s.getQueryString()); + sb.append(s.getLogString()); sb.append('\n'); } try { @@ -554,6 +555,7 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new ClickHouseProvider()); providers.add(new DuckDBProvider()); providers.add(new H2Provider()); + providers.add(new MongoDBProvider()); return providers; } diff --git a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java index 991824628..3b5d87814 100644 --- a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java @@ -1,6 +1,6 @@ package sqlancer.common.oracle; -import sqlancer.SQLGlobalState; +import sqlancer.GlobalState; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.query.ExpectedErrors; @@ -14,7 +14,7 @@ * @param * the global state type */ -public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { +public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { protected E predicate; protected E negatedPredicate; diff --git a/src/sqlancer/mongodb/MongoDBConnection.java b/src/sqlancer/mongodb/MongoDBConnection.java new file mode 100644 index 000000000..c15a46be5 --- /dev/null +++ b/src/sqlancer/mongodb/MongoDBConnection.java @@ -0,0 +1,31 @@ +package sqlancer.mongodb; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoDatabase; + +import sqlancer.SQLancerDBConnection; + +public class MongoDBConnection implements SQLancerDBConnection { + + private final MongoClient client; + private final MongoDatabase database; + + public MongoDBConnection(MongoClient client, MongoDatabase database) { + this.client = client; + this.database = database; + } + + @Override + public String getDatabaseVersion() throws Exception { + return "4.4, Java API 4.1"; + } + + @Override + public void close() throws Exception { + client.close(); + } + + public MongoDatabase getDatabase() { + return database; + } +} diff --git a/src/sqlancer/mongodb/MongoDBLoggableFactory.java b/src/sqlancer/mongodb/MongoDBLoggableFactory.java new file mode 100644 index 000000000..b668301b3 --- /dev/null +++ b/src/sqlancer/mongodb/MongoDBLoggableFactory.java @@ -0,0 +1,40 @@ +package sqlancer.mongodb; + +import java.util.Arrays; + +import sqlancer.common.log.Loggable; +import sqlancer.common.log.LoggableFactory; +import sqlancer.common.log.LoggedString; +import sqlancer.common.query.Query; + +public class MongoDBLoggableFactory extends LoggableFactory { + @Override + protected Loggable createLoggable(String input, String suffix) { + return new LoggedString(input + suffix); + } + + @Override + public Query getQueryForStateToReproduce(String queryString) { + throw new UnsupportedOperationException(); + } + + @Override + public Query commentOutQuery(Query query) { + throw new UnsupportedOperationException(); + } + + @Override + protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { + StringBuilder sb = new StringBuilder(); + sb.append("// Time: ").append(time).append("\n"); + sb.append("// Database: ").append(databaseName).append("\n"); + sb.append("// Database version: ").append(databaseVersion).append("\n"); + sb.append("// seed value: ").append(seedValue).append("\n"); + return new LoggedString(sb.toString()); + } + + @Override + public Loggable convertStacktraceToLoggable(Throwable throwable) { + return new LoggedString(Arrays.toString(throwable.getStackTrace()) + "\n" + throwable.getMessage()); + } +} diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java new file mode 100644 index 000000000..58086ac2f --- /dev/null +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -0,0 +1,40 @@ +package sqlancer.mongodb; + +import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.QUERY_PARTITIONING; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.mongodb.test.MongoDBQueryPartitioningWhereTester; + +public class MongoDBOptions implements DBMSSpecificOptions { + + @Parameter(names = "--test-validation", description = "Enable/Disable validation of schema with Schema Validation", arity = 1) + public boolean testValidation = true; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(QUERY_PARTITIONING); + + @Override + public List getTestOracleFactory() { + return oracles; + } + + public enum MongoDBOracleFactory implements OracleFactory { + QUERY_PARTITIONING { + @Override + public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws Exception { + List oracles = new ArrayList<>(); + oracles.add(new MongoDBQueryPartitioningWhereTester(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + } +} diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java new file mode 100644 index 000000000..5fd4801a0 --- /dev/null +++ b/src/sqlancer/mongodb/MongoDBProvider.java @@ -0,0 +1,107 @@ +package sqlancer.mongodb; + +import java.util.ArrayList; +import java.util.List; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; + +import sqlancer.ExecutionTimer; +import sqlancer.GlobalState; +import sqlancer.ProviderAdapter; +import sqlancer.Randomly; +import sqlancer.common.log.LoggableFactory; +import sqlancer.common.query.Query; +import sqlancer.mongodb.MongoDBSchema.MongoDBTable; +import sqlancer.mongodb.gen.MongoDBTableGenerator; + +public class MongoDBProvider + extends ProviderAdapter { + + public MongoDBProvider() { + super(MongoDBGlobalState.class, MongoDBOptions.class); + } + + /* + * enum Action implements AbstractAction { ; INSERT(MongoDBInsertGenerator::getQuery); + * CREATE_INDEX(MongoDBIndexGenerator::getQuery); + * + * private final MongoDBQueryProvider queryProvider; + * + * Action(MongoDBQueryProvider queryProvider) { this.queryProvider = queryProvider; } + * + * @Override public Query getQuery(MongoDBGlobalState globalState) throws Exception { return + * queryProvider.getQuery(globalState); } } + * + * private static int mapActions(MongoDBGlobalState globalState, Action a) { Randomly r = globalState.getRandomly(); + * switch (a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case + * CREATE_INDEX: return r.getInteger(0, 2); default: throw new AssertionError(a); } return -1; } + */ + + public static class MongoDBGlobalState extends GlobalState { + + private final List schemaTables = new ArrayList<>(); + + public synchronized void addTable(MongoDBTable table) { + schemaTables.add(table); + } + + @Override + protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { + boolean logExecutionTime = getOptions().logExecutionTime(); + if (success && getOptions().printSucceedingStatements()) { + System.out.println(q.getLogString()); + } + if (logExecutionTime) { + getLogger().writeCurrent("// " + timer.end().asString()); + } + if (q.couldAffectSchema()) { + updateSchema(); + } + } + + @Override + protected MongoDBSchema readSchema() throws Exception { + return new MongoDBSchema(schemaTables); + } + } + + @Override + public void generateDatabase(MongoDBGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { + boolean success; + do { + MongoDBQueryAdapter query = new MongoDBTableGenerator().getQuery(globalState); + success = globalState.executeStatement(query); + } while (!success); + } + /* + * StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + * MongoDBProvider::mapActions, (q) -> { if(globalState.getSchema().getDatabaseTables().isEmpty()) { throw new + * IgnoreMeException(); } }); se.executeStatements(); + */ + } + + @Override + public MongoDBConnection createDatabase(MongoDBGlobalState globalState) throws Exception { + MongoClient mongoClient = MongoClients.create(); + MongoDatabase database = mongoClient.getDatabase(globalState.getDatabaseName()); + database.drop(); + return new MongoDBConnection(mongoClient, database); + } + + @Override + public String getDBMSName() { + return "mongodb"; + } + + @Override + public LoggableFactory getLoggableFactory() { + return new MongoDBLoggableFactory(); + } + + @Override + protected void checkViewsAreValid(MongoDBGlobalState globalState) { + } +} diff --git a/src/sqlancer/mongodb/MongoDBQueryAdapter.java b/src/sqlancer/mongodb/MongoDBQueryAdapter.java new file mode 100644 index 000000000..438163486 --- /dev/null +++ b/src/sqlancer/mongodb/MongoDBQueryAdapter.java @@ -0,0 +1,21 @@ +package sqlancer.mongodb; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; + +public abstract class MongoDBQueryAdapter extends Query { + @Override + public String getQueryString() { + throw new UnsupportedOperationException(); + } + + @Override + public String getUnterminatedQueryString() { + throw new UnsupportedOperationException(); + } + + @Override + public ExpectedErrors getExpectedErrors() { + return new ExpectedErrors(); + } +} diff --git a/src/sqlancer/mongodb/MongoDBQueryProvider.java b/src/sqlancer/mongodb/MongoDBQueryProvider.java new file mode 100644 index 000000000..970c90cea --- /dev/null +++ b/src/sqlancer/mongodb/MongoDBQueryProvider.java @@ -0,0 +1,6 @@ +package sqlancer.mongodb; + +@FunctionalInterface +public interface MongoDBQueryProvider { + MongoDBQueryAdapter getQuery(S globalState) throws Exception; +} diff --git a/src/sqlancer/mongodb/MongoDBSchema.java b/src/sqlancer/mongodb/MongoDBSchema.java new file mode 100644 index 000000000..5bf3cf2b5 --- /dev/null +++ b/src/sqlancer/mongodb/MongoDBSchema.java @@ -0,0 +1,117 @@ +package sqlancer.mongodb; + +import java.util.Collections; +import java.util.List; + +import org.bson.BsonType; + +import com.mongodb.client.MongoDatabase; + +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; + +public class MongoDBSchema extends AbstractSchema { + + public enum MongoDBDataType implements HasBsonType { + INTEGER { + @Override + public BsonType getBsonType() { + return BsonType.INT32; + } + }, + STRING { + @Override + public BsonType getBsonType() { + return BsonType.STRING; + } + }, + BOOLEAN { + @Override + public BsonType getBsonType() { + return BsonType.BOOLEAN; + } + }, + DOUBLE { + @Override + public BsonType getBsonType() { + return BsonType.DOUBLE; + } + }, + DATE_TIME { + @Override + public BsonType getBsonType() { + return BsonType.DATE_TIME; + } + }, + TIMESTAMP { + @Override + public BsonType getBsonType() { + return BsonType.TIMESTAMP; + } + }; + + public static MongoDBDataType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public interface HasBsonType { + BsonType getBsonType(); + } + + public static class MongoDBColumn extends AbstractTableColumn { + + private final boolean isId; + private final boolean isNullable; + + public MongoDBColumn(String name, MongoDBDataType type, boolean isId, boolean isNullable) { + super(name, null, type); + this.isId = isId; + this.isNullable = isNullable; + } + + public boolean isId() { + return isId; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class MongoDBTables extends AbstractTables { + + public MongoDBTables(List tables) { + super(tables); + } + } + + public MongoDBSchema(List databaseTables) { + super(databaseTables); + } + + public static class MongoDBTable extends AbstractTable { + public MongoDBTable(String name, List columns, boolean isView) { + super(name, columns, Collections.emptyList(), isView); + } + + @Override + public long getNrRows(MongoDBGlobalState globalState) { + throw new UnsupportedOperationException(); + } + } + + public static MongoDBSchema fromConnection(MongoDatabase connection, String databaseName) { + throw new UnsupportedOperationException(); + } + + public MongoDBTables getRandomTableNonEmptyTables() { + return new MongoDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } +} diff --git a/src/sqlancer/mongodb/ast/MongoDBExpression.java b/src/sqlancer/mongodb/ast/MongoDBExpression.java new file mode 100644 index 000000000..1235a1fbc --- /dev/null +++ b/src/sqlancer/mongodb/ast/MongoDBExpression.java @@ -0,0 +1,4 @@ +package sqlancer.mongodb.ast; + +public interface MongoDBExpression { +} diff --git a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java new file mode 100644 index 000000000..f71ea6411 --- /dev/null +++ b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.mongodb.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; +import sqlancer.mongodb.MongoDBQueryAdapter; +import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; +import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; +import sqlancer.mongodb.MongoDBSchema.MongoDBTable; +import sqlancer.mongodb.query.MongoDBCreateTableQuery; + +public class MongoDBTableGenerator { + + private MongoDBTable table; + private final List columnsToBeAdded = new ArrayList<>(); + + public MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { + String tableName = globalState.getSchema().getFreeTableName(); + MongoDBCreateTableQuery createTableQuery = new MongoDBCreateTableQuery(tableName); + table = new MongoDBTable(tableName, columnsToBeAdded, false); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + MongoDBDataType type = createColumn(columnName); + if (globalState.getDmbsSpecificOptions().testValidation) { + createTableQuery.addValidation(columnName, type.getBsonType()); + } + } + globalState.addTable(table); + return createTableQuery; + } + + private MongoDBDataType createColumn(String columnName) { + MongoDBDataType columnType = MongoDBDataType.getRandom(); + MongoDBColumn newColumn = new MongoDBColumn(columnName, columnType, false, false); + newColumn.setTable(table); + columnsToBeAdded.add(newColumn); + return columnType; + } + + public String getTableName() { + return table.getName(); + } + + public MongoDBTable getGeneratedTable() { + return table; + } +} diff --git a/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java b/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java new file mode 100644 index 000000000..da4381079 --- /dev/null +++ b/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java @@ -0,0 +1,109 @@ +package sqlancer.mongodb.query; + +import java.util.ArrayList; +import java.util.List; + +import org.bson.BsonType; +import org.bson.conversions.Bson; + +import com.mongodb.client.model.CreateCollectionOptions; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.ValidationOptions; + +import sqlancer.GlobalState; +import sqlancer.Main; +import sqlancer.mongodb.MongoDBConnection; +import sqlancer.mongodb.MongoDBQueryAdapter; + +public class MongoDBCreateTableQuery extends MongoDBQueryAdapter { + + private final String tableName; + private Bson validationFilter; + private final List logRequiredList; + private final List logPropertiesList; + + public MongoDBCreateTableQuery(String tableName) { + this.tableName = tableName; + this.validationFilter = null; + logRequiredList = new ArrayList<>(); + logPropertiesList = new ArrayList<>(); + } + + @Override + public boolean couldAffectSchema() { + return true; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + ValidationOptions collOptions = new ValidationOptions().validator(this.validationFilter); + Main.nrSuccessfulActions.addAndGet(1); + globalState.getConnection().getDatabase().createCollection(tableName, + new CreateCollectionOptions().validationOptions(collOptions)); + return true; + } + + @Override + public String getLogString() { + String helper = ""; + StringBuilder sb = new StringBuilder(); + sb.append("db.createCollection(\"").append(tableName).append("\", {\n"); + + if (!logPropertiesList.isEmpty()) { + sb.append("validator: {"); + sb.append("$jsonSchema: {"); + sb.append("bsonType:\"object\","); + sb.append("required: [\n"); + for (String req : logRequiredList) { + sb.append(helper); + helper = ","; + sb.append(req); + } + sb.append("],"); + sb.append("properties: {\n"); + for (String prop : logPropertiesList) { + sb.append(prop); + } + sb.append("}}}})"); + } else { + sb.append("})"); + } + + return sb.toString(); + } + + public void addValidation(String columnName, BsonType type) { + Bson nameFilter = Filters.exists(columnName); + Bson typeFilter = Filters.type(columnName, type); + + if (validationFilter == null) { + validationFilter = Filters.and(nameFilter, typeFilter); + } else { + validationFilter = Filters.and(validationFilter, Filters.and(nameFilter, typeFilter)); + } + + logRequiredList.add("\"" + columnName + "\""); + logPropertiesList.add(columnName + ": { bsonType:\"" + bsonTypeToString(type) + "\"},\n"); + } + + public String bsonTypeToString(BsonType type) { + switch (type) { + case DOUBLE: + return "double"; + case STRING: + return "string"; + case BOOLEAN: + return "bool"; + case INT32: + case INT64: + return "int"; + case DATE_TIME: + return "date"; + case TIMESTAMP: + return "timestamp"; + default: + throw new IllegalStateException(); + } + } +} diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java new file mode 100644 index 000000000..1f59c1909 --- /dev/null +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -0,0 +1,26 @@ +package sqlancer.mongodb.test; + +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; +import sqlancer.mongodb.ast.MongoDBExpression; + +public class MongoDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, MongoDBGlobalState> implements TestOracle { + + public MongoDBQueryPartitioningBase(MongoDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + + } + + @Override + protected ExpressionGenerator> getGen() { + throw new UnsupportedOperationException(); + } +} diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java new file mode 100644 index 000000000..5ee795011 --- /dev/null +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java @@ -0,0 +1,9 @@ +package sqlancer.mongodb.test; + +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; + +public class MongoDBQueryPartitioningWhereTester extends MongoDBQueryPartitioningBase { + public MongoDBQueryPartitioningWhereTester(MongoDBGlobalState state) { + super(state); + } +} From 24e8897da646d1bf3ed5d197fb7835e5bdcf32bf Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 6 Jan 2021 16:13:06 +0100 Subject: [PATCH 0370/1284] Add data inserts and index creation for MongoDB Random Document insertion with the option to follow the schema validation or set random types is now supported. Also an option has been added to randomly set an insert to null. Validation has to be turned off for these options though. Additionally, random indexes are created over random columns, either ascending or descending and when there are multiple indexes added for a single collection it creates a composite index out of the other ones. --- src/sqlancer/mongodb/MongoDBOptions.java | 9 + src/sqlancer/mongodb/MongoDBProvider.java | 58 ++++-- src/sqlancer/mongodb/ast/MongoDBConstant.java | 175 ++++++++++++++++++ .../mongodb/gen/MongoDBConstantGenerator.java | 66 +++++++ .../mongodb/gen/MongoDBIndexGenerator.java | 25 +++ .../mongodb/gen/MongoDBInsertGenerator.java | 38 ++++ .../query/MongoDBCreateIndexQuery.java | 71 +++++++ .../mongodb/query/MongoDBInsertQuery.java | 79 ++++++++ 8 files changed, 501 insertions(+), 20 deletions(-) create mode 100644 src/sqlancer/mongodb/ast/MongoDBConstant.java create mode 100644 src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java create mode 100644 src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java create mode 100644 src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java create mode 100644 src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java create mode 100644 src/sqlancer/mongodb/query/MongoDBInsertQuery.java diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 58086ac2f..a6306b515 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -19,6 +19,15 @@ public class MongoDBOptions implements DBMSSpecificOptions oracles = Arrays.asList(QUERY_PARTITIONING); diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java index 5fd4801a0..cf4375d01 100644 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ b/src/sqlancer/mongodb/MongoDBProvider.java @@ -7,13 +7,18 @@ import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; +import sqlancer.AbstractAction; import sqlancer.ExecutionTimer; import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Randomly; +import sqlancer.StatementExecutor; import sqlancer.common.log.LoggableFactory; import sqlancer.common.query.Query; import sqlancer.mongodb.MongoDBSchema.MongoDBTable; +import sqlancer.mongodb.gen.MongoDBIndexGenerator; +import sqlancer.mongodb.gen.MongoDBInsertGenerator; import sqlancer.mongodb.gen.MongoDBTableGenerator; public class MongoDBProvider @@ -23,21 +28,32 @@ public MongoDBProvider() { super(MongoDBGlobalState.class, MongoDBOptions.class); } - /* - * enum Action implements AbstractAction { ; INSERT(MongoDBInsertGenerator::getQuery); - * CREATE_INDEX(MongoDBIndexGenerator::getQuery); - * - * private final MongoDBQueryProvider queryProvider; - * - * Action(MongoDBQueryProvider queryProvider) { this.queryProvider = queryProvider; } - * - * @Override public Query getQuery(MongoDBGlobalState globalState) throws Exception { return - * queryProvider.getQuery(globalState); } } - * - * private static int mapActions(MongoDBGlobalState globalState, Action a) { Randomly r = globalState.getRandomly(); - * switch (a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case - * CREATE_INDEX: return r.getInteger(0, 2); default: throw new AssertionError(a); } return -1; } - */ + enum Action implements AbstractAction { + INSERT(MongoDBInsertGenerator::getQuery), CREATE_INDEX(MongoDBIndexGenerator::getQuery); + + private final MongoDBQueryProvider queryProvider; + + Action(MongoDBQueryProvider queryProvider) { + this.queryProvider = queryProvider; + } + + @Override + public Query getQuery(MongoDBGlobalState globalState) throws Exception { + return queryProvider.getQuery(globalState); + } + } + + private static int mapActions(MongoDBGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case CREATE_INDEX: + return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumberIndexes); + default: + throw new AssertionError(a); + } + } public static class MongoDBGlobalState extends GlobalState { @@ -76,11 +92,13 @@ public void generateDatabase(MongoDBGlobalState globalState) throws Exception { success = globalState.executeStatement(query); } while (!success); } - /* - * StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - * MongoDBProvider::mapActions, (q) -> { if(globalState.getSchema().getDatabaseTables().isEmpty()) { throw new - * IgnoreMeException(); } }); se.executeStatements(); - */ + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + MongoDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); } @Override diff --git a/src/sqlancer/mongodb/ast/MongoDBConstant.java b/src/sqlancer/mongodb/ast/MongoDBConstant.java new file mode 100644 index 000000000..75caa0efd --- /dev/null +++ b/src/sqlancer/mongodb/ast/MongoDBConstant.java @@ -0,0 +1,175 @@ +package sqlancer.mongodb.ast; + +import org.bson.BsonDateTime; +import org.bson.BsonTimestamp; +import org.bson.Document; + +import sqlancer.common.ast.newast.Node; + +public abstract class MongoDBConstant implements Node { + private MongoDBConstant() { + } + + public abstract void setValueInDocument(Document document, String key); + + public abstract String getLogValue(); + + public static class MongoDBNullConstant extends MongoDBConstant { + + @Override + public void setValueInDocument(Document document, String key) { + document.append(key, null); + } + + @Override + public String getLogValue() { + return "null"; + } + } + + public static class MongoDBIntegerConstant extends MongoDBConstant { + + private final int value; + + public MongoDBIntegerConstant(int value) { + this.value = value; + } + + @Override + public void setValueInDocument(Document document, String key) { + document.append(key, value); + } + + @Override + public String getLogValue() { + return String.valueOf(value); + } + } + + public static Node createIntegerConstant(int value) { + return new MongoDBIntegerConstant(value); + } + + public static class MongoDBStringConstant extends MongoDBConstant { + + private final String value; + + public MongoDBStringConstant(String value) { + this.value = value; + } + + public String getStringValue() { + return value; + } + + @Override + public void setValueInDocument(Document document, String key) { + document.append(key, value); + } + + @Override + public String getLogValue() { + return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; + } + } + + public static Node createStringConstant(String value) { + return new MongoDBStringConstant(value); + } + + public static class MongoDBBooleanConstant extends MongoDBConstant { + + private final boolean value; + + public MongoDBBooleanConstant(boolean value) { + this.value = value; + } + + @Override + public void setValueInDocument(Document document, String key) { + document.append(key, value); + } + + @Override + public String getLogValue() { + if (value) { + return "true"; + } + return "false"; + } + } + + public static Node createBooleanConstant(boolean value) { + return new MongoDBBooleanConstant(value); + } + + public static class MongoDBDoubleConstant extends MongoDBConstant { + + private final double value; + + public MongoDBDoubleConstant(double value) { + this.value = value; + } + + @Override + public void setValueInDocument(Document document, String key) { + document.append(key, value); + } + + @Override + public String getLogValue() { + return String.valueOf(value); + } + } + + public static Node createDoubleConstant(double value) { + return new MongoDBDoubleConstant(value); + } + + public static class MongoDBDateTimeConstant extends MongoDBConstant { + + private final BsonDateTime value; + + public MongoDBDateTimeConstant(long val) { + this.value = new BsonDateTime(val); + } + + @Override + public void setValueInDocument(Document document, String key) { + document.append(key, value); + } + + @Override + public String getLogValue() { + return String.valueOf(value); + } + } + + public static Node createDateTimeConstant(long value) { + return new MongoDBDateTimeConstant(value); + } + + public static class MongoDBTimestampConstant extends MongoDBConstant { + + private final BsonTimestamp value; + + public MongoDBTimestampConstant(long value) { + this.value = new BsonTimestamp(value); + } + + @Override + public void setValueInDocument(Document document, String key) { + document.append(key, value); + } + + @Override + public String getLogValue() { + return String.valueOf(value); + } + } + + public static Node createTimestampConstant(long value) { + return new MongoDBTimestampConstant(value); + } + +} diff --git a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java new file mode 100644 index 000000000..090016809 --- /dev/null +++ b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java @@ -0,0 +1,66 @@ +package sqlancer.mongodb.gen; + +import org.bson.Document; + +import sqlancer.Randomly; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; +import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; +import sqlancer.mongodb.ast.MongoDBConstant; +import sqlancer.mongodb.ast.MongoDBConstant.MongoDBBooleanConstant; +import sqlancer.mongodb.ast.MongoDBConstant.MongoDBDateTimeConstant; +import sqlancer.mongodb.ast.MongoDBConstant.MongoDBDoubleConstant; +import sqlancer.mongodb.ast.MongoDBConstant.MongoDBIntegerConstant; +import sqlancer.mongodb.ast.MongoDBConstant.MongoDBNullConstant; +import sqlancer.mongodb.ast.MongoDBConstant.MongoDBStringConstant; +import sqlancer.mongodb.ast.MongoDBConstant.MongoDBTimestampConstant; + +public class MongoDBConstantGenerator { + private final MongoDBGlobalState globalState; + + public MongoDBConstantGenerator(MongoDBGlobalState globalState) { + this.globalState = globalState; + } + + public void addRandomConstant(Document document, String key) { + MongoDBDataType type = MongoDBDataType.getRandom(); + addRandomConstantWithType(document, key, type); + } + + public void addRandomConstantWithType(Document document, String key, MongoDBDataType option) { + MongoDBConstant constant; + if (globalState.getDmbsSpecificOptions().testNullInserts && Randomly.getBooleanWithSmallProbability()) { + constant = new MongoDBNullConstant(); + constant.setValueInDocument(document, key); + return; + } + switch (option) { + case DATE_TIME: + constant = new MongoDBDateTimeConstant(globalState.getRandomly().getInteger()); + constant.setValueInDocument(document, key); + return; + + case BOOLEAN: + constant = new MongoDBBooleanConstant(Randomly.getBoolean()); + constant.setValueInDocument(document, key); + return; + case DOUBLE: + constant = new MongoDBDoubleConstant(globalState.getRandomly().getDouble()); + constant.setValueInDocument(document, key); + return; + case STRING: + constant = new MongoDBStringConstant(globalState.getRandomly().getString()); + constant.setValueInDocument(document, key); + return; + case INTEGER: + constant = new MongoDBIntegerConstant((int) globalState.getRandomly().getInteger()); + constant.setValueInDocument(document, key); + return; + case TIMESTAMP: + constant = new MongoDBTimestampConstant(globalState.getRandomly().getInteger()); + constant.setValueInDocument(document, key); + return; + default: + throw new AssertionError(option); + } + } +} diff --git a/src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java b/src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java new file mode 100644 index 000000000..8687fd45c --- /dev/null +++ b/src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.mongodb.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; +import sqlancer.mongodb.MongoDBQueryAdapter; +import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; +import sqlancer.mongodb.MongoDBSchema.MongoDBTable; +import sqlancer.mongodb.query.MongoDBCreateIndexQuery; + +public final class MongoDBIndexGenerator { + private MongoDBIndexGenerator() { + } + + public static MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { + MongoDBTable randomTable = globalState.getSchema().getRandomTable(); + List columns = Randomly.nonEmptySubset(randomTable.getColumns()); + MongoDBCreateIndexQuery createIndexQuery = new MongoDBCreateIndexQuery(randomTable); + for (MongoDBColumn column : columns) { + createIndexQuery.addIndex(column.getName(), Randomly.getBoolean()); + } + return createIndexQuery; + } +} diff --git a/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java b/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java new file mode 100644 index 000000000..4501971b4 --- /dev/null +++ b/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java @@ -0,0 +1,38 @@ +package sqlancer.mongodb.gen; + +import org.bson.Document; + +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; +import sqlancer.mongodb.MongoDBQueryAdapter; +import sqlancer.mongodb.MongoDBSchema.MongoDBTable; +import sqlancer.mongodb.query.MongoDBInsertQuery; + +public final class MongoDBInsertGenerator { + + private final MongoDBGlobalState globalState; + + private MongoDBInsertGenerator(MongoDBGlobalState globalState) { + this.globalState = globalState; + } + + public static MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { + return new MongoDBInsertGenerator(globalState).generate(); + } + + public MongoDBQueryAdapter generate() { + Document result = new Document(); + MongoDBTable table = globalState.getSchema().getRandomTable(); + MongoDBConstantGenerator constantGenerator = new MongoDBConstantGenerator(globalState); + + for (int i = 0; i < table.getColumns().size(); i++) { + if (!globalState.getDmbsSpecificOptions().testRandomTypes) { + constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), + table.getColumns().get(i).getType()); + } else { + constantGenerator.addRandomConstant(result, table.getColumns().get(i).getName()); + } + } + + return new MongoDBInsertQuery(table, result); + } +} diff --git a/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java b/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java new file mode 100644 index 000000000..8d54d9cf7 --- /dev/null +++ b/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java @@ -0,0 +1,71 @@ +package sqlancer.mongodb.query; + +import java.util.ArrayList; +import java.util.List; + +import org.bson.conversions.Bson; + +import com.mongodb.client.model.Indexes; + +import sqlancer.GlobalState; +import sqlancer.Main; +import sqlancer.mongodb.MongoDBConnection; +import sqlancer.mongodb.MongoDBQueryAdapter; +import sqlancer.mongodb.MongoDBSchema.MongoDBTable; + +public class MongoDBCreateIndexQuery extends MongoDBQueryAdapter { + + private final MongoDBTable table; + private final List indeces; + private final List logIndeces; + + public MongoDBCreateIndexQuery(MongoDBTable table) { + this.table = table; + this.indeces = new ArrayList<>(); + this.logIndeces = new ArrayList<>(); + } + + public void addIndex(String column, boolean ascending) { + if (ascending) { + indeces.add(Indexes.ascending(column)); + logIndeces.add(column + ": 1"); + } else { + indeces.add(Indexes.descending(column)); + logIndeces.add(column + ": -1"); + } + } + + @Override + public String getLogString() { + StringBuilder sb = new StringBuilder(); + sb.append("db.").append(table.getName()).append(".createIndex({"); + String helper = ""; + for (String index : logIndeces) { + sb.append(helper); + helper = ","; + sb.append(index); + } + sb.append("})\n"); + return sb.toString(); + } + + @Override + public boolean couldAffectSchema() { + return false; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + Main.nrSuccessfulActions.addAndGet(1); + Bson index; + if (indeces.size() > 1) { + index = Indexes.compoundIndex(indeces); + } else { + index = indeces.get(0); + } + globalState.getConnection().getDatabase().getCollection(table.getName()).createIndex(index); + return true; + } + +} diff --git a/src/sqlancer/mongodb/query/MongoDBInsertQuery.java b/src/sqlancer/mongodb/query/MongoDBInsertQuery.java new file mode 100644 index 000000000..8276a0895 --- /dev/null +++ b/src/sqlancer/mongodb/query/MongoDBInsertQuery.java @@ -0,0 +1,79 @@ +package sqlancer.mongodb.query; + +import org.bson.BsonDateTime; +import org.bson.BsonTimestamp; +import org.bson.Document; +import org.bson.types.ObjectId; + +import com.mongodb.client.result.InsertOneResult; + +import sqlancer.GlobalState; +import sqlancer.mongodb.MongoDBConnection; +import sqlancer.mongodb.MongoDBQueryAdapter; +import sqlancer.mongodb.MongoDBSchema.MongoDBTable; + +public class MongoDBInsertQuery extends MongoDBQueryAdapter { + boolean excluded; + private final MongoDBTable table; + private final Document documentToBeInserted; + + public MongoDBInsertQuery(MongoDBTable table, Document documentToBeInserted) { + this.table = table; + this.documentToBeInserted = documentToBeInserted; + this.excluded = false; + } + + @Override + public String getLogString() { + StringBuilder sb = new StringBuilder(); + sb.append("db." + table.getName() + ".insert({"); + String helper = ""; + for (String key : documentToBeInserted.keySet()) { + sb.append(helper); + helper = ", "; + if (documentToBeInserted.get(key) instanceof ObjectId) { + continue; + } + Object value = documentToBeInserted.get(key); + sb.append(key); + sb.append(": "); + sb.append(getStringRepresentation(value)); + } + sb.append("})\n"); + + return sb.toString(); + } + + private String getStringRepresentation(Object value) { + if (value instanceof Double) { + return String.valueOf(value); + } else if (value instanceof Integer) { + return "NumberInt(" + value + ")"; + } else if (value instanceof String) { + return "\"" + value + "\""; + } else if (value instanceof BsonDateTime) { + return "new Date(" + ((BsonDateTime) value).getValue() + ")"; + } else if (value instanceof BsonTimestamp) { + return "Timestamp(" + ((BsonTimestamp) value).getValue() + ",1)"; + } else if (value instanceof Boolean) { + return String.valueOf(value); + } else if (value == null) { + return "null"; + } else { + throw new IllegalStateException(); + } + } + + @Override + public boolean couldAffectSchema() { + return true; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + InsertOneResult result = globalState.getConnection().getDatabase().getCollection(table.getName()) + .insertOne(documentToBeInserted); + return result.wasAcknowledged(); + } +} From 46d168085531a9330323f64d7eff7ead261dec08 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 6 Jan 2021 22:30:07 +0100 Subject: [PATCH 0371/1284] Add query execution for MongoDB with lookup and project stages The project stage is similar to the SELECT in SQL where we can specify which columns are returned. The Lookup stage is similar to the LEFT OUTER JOIN of SQL, for this we have to specifically make new random columns that specify the join column. --- .../mongodb/ast/MongoDBDummyPredicate.java | 7 ++ src/sqlancer/mongodb/ast/MongoDBSelect.java | 76 +++++++++++++++++++ .../gen/MongoDBExpressionGenerator.java | 40 ++++++++++ .../mongodb/query/MongoDBSelectQuery.java | 75 ++++++++++++++++++ .../test/MongoDBColumnTestReference.java | 34 +++++++++ .../test/MongoDBQueryPartitioningBase.java | 51 ++++++++++++- .../MongoDBQueryPartitioningWhereTester.java | 22 ++++++ .../mongodb/visitor/MongoDBToLogVisitor.java | 72 ++++++++++++++++++ .../visitor/MongoDBToQueryVisitor.java | 63 +++++++++++++++ .../mongodb/visitor/MongoDBVisitor.java | 38 ++++++++++ 10 files changed, 477 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/mongodb/ast/MongoDBDummyPredicate.java create mode 100644 src/sqlancer/mongodb/ast/MongoDBSelect.java create mode 100644 src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java create mode 100644 src/sqlancer/mongodb/query/MongoDBSelectQuery.java create mode 100644 src/sqlancer/mongodb/test/MongoDBColumnTestReference.java create mode 100644 src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java create mode 100644 src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java create mode 100644 src/sqlancer/mongodb/visitor/MongoDBVisitor.java diff --git a/src/sqlancer/mongodb/ast/MongoDBDummyPredicate.java b/src/sqlancer/mongodb/ast/MongoDBDummyPredicate.java new file mode 100644 index 000000000..a9ecc3325 --- /dev/null +++ b/src/sqlancer/mongodb/ast/MongoDBDummyPredicate.java @@ -0,0 +1,7 @@ +package sqlancer.mongodb.ast; + +import sqlancer.common.ast.newast.Node; + +public class MongoDBDummyPredicate implements Node { + +} diff --git a/src/sqlancer/mongodb/ast/MongoDBSelect.java b/src/sqlancer/mongodb/ast/MongoDBSelect.java new file mode 100644 index 000000000..c6252893f --- /dev/null +++ b/src/sqlancer/mongodb/ast/MongoDBSelect.java @@ -0,0 +1,76 @@ +package sqlancer.mongodb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.Node; +import sqlancer.mongodb.test.MongoDBColumnTestReference; + +public class MongoDBSelect implements Node { + + private final String mainTableName; + private final MongoDBColumnTestReference joinColumn; + List projectionColumns; + List lookupList; + boolean hasFilter; + Node filterClause; + + public MongoDBSelect(String mainTableName, MongoDBColumnTestReference joinColumn) { + this.mainTableName = mainTableName; + this.joinColumn = joinColumn; + } + + public String getMainTableName() { + return mainTableName; + } + + public MongoDBColumnTestReference getJoinColumn() { + return joinColumn; + } + + public void setProjectionList(List fetchColumns) { + if (fetchColumns == null || fetchColumns.isEmpty()) { + throw new IllegalArgumentException(); + } + this.projectionColumns = fetchColumns; + } + + public List getProjectionList() { + if (projectionColumns == null) { + throw new IllegalStateException(); + } + return projectionColumns; + } + + public void setLookupList(List lookupList) { + if (lookupList == null || lookupList.isEmpty()) { + throw new IllegalArgumentException(); + } + this.lookupList = lookupList; + } + + public List getLookupList() { + if (lookupList == null) { + throw new IllegalStateException(); + } + return lookupList; + } + + public void setFilterClause(Node filterClause) { + if (filterClause == null) { + hasFilter = false; + this.filterClause = null; + return; + } + hasFilter = true; + this.filterClause = filterClause; + } + + public Node getFilterClause() { + return filterClause; + } + + public boolean hasWhere() { + return hasFilter; + } + +} diff --git a/src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java new file mode 100644 index 000000000..f2c05dda2 --- /dev/null +++ b/src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java @@ -0,0 +1,40 @@ +package sqlancer.mongodb.gen; + +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.mongodb.ast.MongoDBDummyPredicate; +import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.test.MongoDBColumnTestReference; + +public class MongoDBExpressionGenerator + extends UntypedExpressionGenerator, MongoDBColumnTestReference> { + /* + * private final MongoDBGlobalState globalState; + * + * public MongoDBExpressionGenerator(MongoDBGlobalState globalState) { this.globalState = globalState; } + */ + @Override + public Node generateConstant() { + throw new UnsupportedOperationException(); + } + + @Override + protected Node generateExpression(int depth) { + return new MongoDBDummyPredicate<>(); + } + + @Override + protected Node generateColumn() { + throw new UnsupportedOperationException(); + } + + @Override + public Node negatePredicate(Node predicate) { + return new MongoDBDummyPredicate<>(); + } + + @Override + public Node isNull(Node expr) { + return new MongoDBDummyPredicate<>(); + } +} diff --git a/src/sqlancer/mongodb/query/MongoDBSelectQuery.java b/src/sqlancer/mongodb/query/MongoDBSelectQuery.java new file mode 100644 index 000000000..91480ed4b --- /dev/null +++ b/src/sqlancer/mongodb/query/MongoDBSelectQuery.java @@ -0,0 +1,75 @@ +package sqlancer.mongodb.query; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.bson.Document; +import org.bson.conversions.Bson; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; + +import sqlancer.GlobalState; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.mongodb.MongoDBConnection; +import sqlancer.mongodb.MongoDBQueryAdapter; +import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBSelect; +import sqlancer.mongodb.visitor.MongoDBVisitor; + +public class MongoDBSelectQuery extends MongoDBQueryAdapter { + + private final MongoDBSelect select; + + private List resultSet; + + public MongoDBSelectQuery(MongoDBSelect select) { + this.select = select; + } + + @Override + public boolean couldAffectSchema() { + return false; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public > SQLancerResultSet executeAndGet(G globalState, + String... fills) throws Exception { + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(this.getLogString()); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + List pipeline = MongoDBVisitor.asQuery(select); + + MongoCollection collection = globalState.getConnection().getDatabase() + .getCollection(select.getMainTableName()); + MongoCursor cursor = collection.aggregate(pipeline).cursor(); + resultSet = new ArrayList<>(); + while (cursor.hasNext()) { + Document document = cursor.next(); + resultSet.add(document); + } + return null; + } + + @Override + public String getLogString() { + return MongoDBVisitor.asStringLog(select); + } + + public List getResultSet() { + return resultSet; + } + +} diff --git a/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java b/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java new file mode 100644 index 000000000..f6c8e9d94 --- /dev/null +++ b/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java @@ -0,0 +1,34 @@ +package sqlancer.mongodb.test; + +import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; + +public class MongoDBColumnTestReference { + + private final MongoDBColumn columnReference; + private final boolean inMainTable; + + public MongoDBColumnTestReference(MongoDBColumn columnReference, boolean inMainTable) { + this.columnReference = columnReference; + this.inMainTable = inMainTable; + } + + public String getQueryString() { + if (inMainTable) { + return this.columnReference.getName(); + } else { + return "join_" + this.columnReference.getTable().getName() + "." + this.columnReference.getName(); + } + } + + public boolean inMainTable() { + return inMainTable; + } + + public String getTableName() { + return this.columnReference.getTable().getName(); + } + + public String getPlainName() { + return this.columnReference.getName(); + } +} diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java index 1f59c1909..4fda2a64b 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -1,26 +1,75 @@ package sqlancer.mongodb.test; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; +import sqlancer.mongodb.MongoDBSchema; +import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; +import sqlancer.mongodb.MongoDBSchema.MongoDBTable; +import sqlancer.mongodb.MongoDBSchema.MongoDBTables; import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBSelect; +import sqlancer.mongodb.gen.MongoDBExpressionGenerator; public class MongoDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, MongoDBGlobalState> implements TestOracle { + protected MongoDBSchema schema; + protected MongoDBTables targetTables; + protected MongoDBTable mainTable; + protected List targetColumns; + protected MongoDBExpressionGenerator expressionGenerator; + protected MongoDBSelect select; + public MongoDBQueryPartitioningBase(MongoDBGlobalState state) { super(state); } @Override public void check() throws Exception { + schema = state.getSchema(); + targetTables = schema.getRandomTableNonEmptyTables(); + mainTable = targetTables.getTables().get(0); + generateTargetColumns(); + expressionGenerator = new MongoDBExpressionGenerator().setColumns(targetColumns); + initializeTernaryPredicateVariants(); + select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); + select.setProjectionList(targetColumns); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setLookupList(targetColumns); + } else { + select.setLookupList(Randomly.nonEmptySubset(targetColumns)); + } + } + private void generateTargetColumns() { + targetColumns = new ArrayList<>(); + for (MongoDBColumn c : mainTable.getColumns()) { + targetColumns.add(new MongoDBColumnTestReference(c, true)); + } + List joinsOtherTables = new ArrayList<>(); + for (int i = 1; i < targetTables.getTables().size(); i++) { + MongoDBTable procTable = targetTables.getTables().get(i); + for (MongoDBColumn c : procTable.getColumns()) { + joinsOtherTables.add(new MongoDBColumnTestReference(c, false)); + } + } + if (!joinsOtherTables.isEmpty()) { + int randNumber = state.getRandomly().getInteger(1, Math.min(joinsOtherTables.size(), 4)); + List subsetJoinsOtherTables = Randomly.nonEmptySubset(joinsOtherTables, + randNumber); + targetColumns.addAll(subsetJoinsOtherTables); + } } @Override protected ExpressionGenerator> getGen() { - throw new UnsupportedOperationException(); + return expressionGenerator; } } diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java index 5ee795011..11ae2c8ae 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java @@ -1,9 +1,31 @@ package sqlancer.mongodb.test; import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; +import sqlancer.mongodb.query.MongoDBSelectQuery; public class MongoDBQueryPartitioningWhereTester extends MongoDBQueryPartitioningBase { public MongoDBQueryPartitioningWhereTester(MongoDBGlobalState state) { super(state); } + + @Override + public void check() throws Exception { + super.check(); + + select.setFilterClause(null); + MongoDBSelectQuery q = new MongoDBSelectQuery(select); + q.executeAndGet(state); + /* + * List firstResultSet = q.getResultSet(); select.setWhereClause(predicate); q = new + * MongoDBFindQueryAdapter(select, mainTable); q.executeAndGet(state); List secondResultSet = + * q.resultSet; + * + * select.setWhereClause(negatedPredicate); q = new MongoDBFindQueryAdapter(select, mainTable); + * q.executeAndGet(state); List thirdResultSet = q.resultSet; + * + * secondResultSet.addAll(thirdResultSet); MongoDBComparatorHelper.assumeResultSetsAreEqual(firstResultSet, + * secondResultSet, q); + * + */ + } } diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java new file mode 100644 index 000000000..50d43a3ee --- /dev/null +++ b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java @@ -0,0 +1,72 @@ +package sqlancer.mongodb.visitor; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.mongodb.ast.MongoDBConstant; +import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBSelect; +import sqlancer.mongodb.test.MongoDBColumnTestReference; + +public class MongoDBToLogVisitor extends MongoDBVisitor { + + private String mainTableName; + private List lookups; + private String projects; + + @Override + public void visit(MongoDBConstant c) { + throw new UnsupportedOperationException(); + } + + @Override + public void visit(MongoDBSelect select) { + mainTableName = select.getMainTableName(); + setLookups(select); + setProjects(select); + } + + private void setLookups(MongoDBSelect select) { + lookups = new ArrayList<>(); + for (MongoDBColumnTestReference testReference : select.getLookupList()) { + if (testReference.inMainTable()) { + continue; + } + String newLookup = "{ $lookup: { from: \"" + testReference.getTableName() + "\", localField: \"" + + select.getJoinColumn().getPlainName() + "\", foreignField: \"" + testReference.getPlainName() + + "\", as: \"" + testReference.getQueryString() + "\"}},\n"; + lookups.add(newLookup); + } + } + + private void setProjects(MongoDBSelect select) { + StringBuilder sb = new StringBuilder(); + sb.append("{"); + String helper = ""; + for (MongoDBColumnTestReference reference : select.getProjectionList()) { + sb.append(helper); + helper = ","; + sb.append("\"").append(reference.getQueryString()).append("\"").append(": 1"); + } + sb.append("}"); + projects = sb.toString(); + } + + public String getStringLog() { + StringBuilder sb = new StringBuilder(); + sb.append("db.").append(mainTableName).append(".aggregate([\n"); + for (String lookup : lookups) { + sb.append(lookup); + } + // sb.append(",\n"); + // if(select.hasWhere()) { + // sb.append("{ $match: "); + // sb.append(mongoDBToQueryVisitor.getFilterLog()); + // sb.append("},\n"); + // } + sb.append("{ $project : "); + sb.append(projects); + sb.append("}])\n"); + return sb.toString(); + } +} diff --git a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java new file mode 100644 index 000000000..3ad57a3cb --- /dev/null +++ b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java @@ -0,0 +1,63 @@ +package sqlancer.mongodb.visitor; + +import static com.mongodb.client.model.Aggregates.project; +import static com.mongodb.client.model.Projections.fields; +import static com.mongodb.client.model.Projections.include; + +import java.util.ArrayList; +import java.util.List; + +import org.bson.conversions.Bson; + +import com.mongodb.client.model.Aggregates; + +import sqlancer.mongodb.ast.MongoDBConstant; +import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBSelect; +import sqlancer.mongodb.test.MongoDBColumnTestReference; + +public class MongoDBToQueryVisitor extends MongoDBVisitor { + + private List lookup; + private Bson projection; + + @Override + public void visit(MongoDBConstant c) { + throw new UnsupportedOperationException(); + } + + @Override + public void visit(MongoDBSelect select) { + setLookup(select); + // if(select.hasWhere()) { + // // DO STUFF + // } + setProjection(select); + } + + private void setLookup(MongoDBSelect select) { + lookup = new ArrayList<>(); + for (MongoDBColumnTestReference reference : select.getLookupList()) { + if (reference.inMainTable()) { + continue; + } + lookup.add(Aggregates.lookup(reference.getTableName(), select.getJoinColumn().getPlainName(), + reference.getPlainName(), reference.getQueryString())); + } + } + + private void setProjection(MongoDBSelect select) { + List stringProjects = new ArrayList<>(); + for (MongoDBColumnTestReference ref : select.getProjectionList()) { + stringProjects.add(ref.getQueryString()); + } + projection = project(fields(include(stringProjects))); + } + + public List getPipeline() { + List result = new ArrayList<>(lookup); + // TODO Patrick: Add Match if where... + result.add(projection); + return result; + } +} diff --git a/src/sqlancer/mongodb/visitor/MongoDBVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBVisitor.java new file mode 100644 index 000000000..8c207a564 --- /dev/null +++ b/src/sqlancer/mongodb/visitor/MongoDBVisitor.java @@ -0,0 +1,38 @@ +package sqlancer.mongodb.visitor; + +import java.util.List; + +import org.bson.conversions.Bson; + +import sqlancer.common.ast.newast.Node; +import sqlancer.mongodb.ast.MongoDBConstant; +import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBSelect; + +public abstract class MongoDBVisitor { + public abstract void visit(MongoDBConstant c); + + public abstract void visit(MongoDBSelect s); + + public void visit(Node expr) { + if (expr instanceof MongoDBConstant) { + visit((MongoDBConstant) expr); + } else if (expr instanceof MongoDBSelect) { + visit((MongoDBSelect) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + public static List asQuery(Node expr) { + MongoDBToQueryVisitor visitor = new MongoDBToQueryVisitor(); + visitor.visit(expr); + return visitor.getPipeline(); + } + + public static String asStringLog(Node expr) { + MongoDBToLogVisitor visitor = new MongoDBToLogVisitor(); + visitor.visit(expr); + return visitor.getStringLog(); + } +} From 53a25cff0086be71ff7e439a7274628330fcabb2 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 7 Jan 2021 11:21:00 +0100 Subject: [PATCH 0372/1284] Implement expression generation and filter query execution for MongoDB The query ast has multiple key difference to the SQL version. The core is the MongoDBSelect class that holds projection and lookup lists. The filter is then a tree similar to the SQL version. In order to execute and log, which are different because one are API calls and the other MongoDB shell commands there exist two visitors, the ToQueryVisitor and the ToLogVisitor. --- .../mongodb/MongoDBComparatorHelper.java | 45 ++++ .../ast/MongoDBBinaryComparisonNode.java | 16 ++ .../mongodb/ast/MongoDBBinaryLogicalNode.java | 16 ++ src/sqlancer/mongodb/ast/MongoDBConstant.java | 54 ++++- src/sqlancer/mongodb/ast/MongoDBSelect.java | 2 +- .../ast/MongoDBUnaryLogicalOperatorNode.java | 16 ++ ....java => MongoDBUnsupportedPredicate.java} | 2 +- .../mongodb/gen/MongoDBConstantGenerator.java | 24 ++ .../gen/MongoDBExpressionGenerator.java | 211 +++++++++++++++++- .../test/MongoDBColumnTestReference.java | 8 +- .../test/MongoDBQueryPartitioningBase.java | 2 +- .../MongoDBQueryPartitioningWhereTester.java | 32 ++- .../mongodb/visitor/MongoDBToLogVisitor.java | 59 ++++- .../visitor/MongoDBToQueryVisitor.java | 55 ++++- .../mongodb/visitor/MongoDBVisitor.java | 1 + 15 files changed, 497 insertions(+), 46 deletions(-) create mode 100644 src/sqlancer/mongodb/MongoDBComparatorHelper.java create mode 100644 src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java create mode 100644 src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java create mode 100644 src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java rename src/sqlancer/mongodb/ast/{MongoDBDummyPredicate.java => MongoDBUnsupportedPredicate.java} (53%) diff --git a/src/sqlancer/mongodb/MongoDBComparatorHelper.java b/src/sqlancer/mongodb/MongoDBComparatorHelper.java new file mode 100644 index 000000000..235e5ece9 --- /dev/null +++ b/src/sqlancer/mongodb/MongoDBComparatorHelper.java @@ -0,0 +1,45 @@ +package sqlancer.mongodb; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.bson.Document; + +import sqlancer.mongodb.query.MongoDBSelectQuery; + +public final class MongoDBComparatorHelper { + + private MongoDBComparatorHelper() { + } + + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, + MongoDBSelectQuery originalQuery) { + if (resultSet.size() != secondResultSet.size()) { + String assertionMessage = String.format("The Size of the result sets mismatch (%d and %d)!\n%s", + resultSet.size(), resultSet.size(), originalQuery.getLogString()); + throw new AssertionError(assertionMessage); + } + + Set firstHashSet = new HashSet<>(resultSet); + Set secondHashSet = new HashSet<>(secondResultSet); + + if (!firstHashSet.equals(secondHashSet)) { + Set firstResultSetMisses = new HashSet<>(firstHashSet); + firstResultSetMisses.removeAll(secondHashSet); + Set secondResultSetMisses = new HashSet<>(secondHashSet); + secondResultSetMisses.removeAll(firstHashSet); + StringBuilder firstMisses = new StringBuilder(); + for (Document document : firstResultSetMisses) { + firstMisses.append(document.toJson()).append(" "); + } + StringBuilder secondMisses = new StringBuilder(); + for (Document document : secondResultSetMisses) { + secondMisses.append(document.toJson()).append(" "); + } + String assertMessage = String.format("The Content of the result sets mismatch!\n %s \n %s\n %s", + firstMisses.toString(), secondMisses.toString(), originalQuery.getLogString()); + throw new AssertionError(assertMessage); + } + } +} diff --git a/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java b/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java new file mode 100644 index 000000000..f5e3ea173 --- /dev/null +++ b/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java @@ -0,0 +1,16 @@ +package sqlancer.mongodb.ast; + +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.mongodb.gen.MongoDBExpressionGenerator.MongoDBBinaryComparisonOperator; + +public class MongoDBBinaryComparisonNode extends NewBinaryOperatorNode { + public MongoDBBinaryComparisonNode(Node left, Node right, + MongoDBBinaryComparisonOperator op) { + super(left, right, op); + } + + public MongoDBBinaryComparisonOperator operator() { + return (MongoDBBinaryComparisonOperator) op; + } +} diff --git a/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java b/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java new file mode 100644 index 000000000..c3f18a248 --- /dev/null +++ b/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java @@ -0,0 +1,16 @@ +package sqlancer.mongodb.ast; + +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.mongodb.gen.MongoDBExpressionGenerator.MongoDBBinaryLogicalOperator; + +public class MongoDBBinaryLogicalNode extends NewBinaryOperatorNode { + public MongoDBBinaryLogicalNode(Node left, Node right, + MongoDBBinaryLogicalOperator op) { + super(left, right, op); + } + + public MongoDBBinaryLogicalOperator operator() { + return (MongoDBBinaryLogicalOperator) op; + } +} diff --git a/src/sqlancer/mongodb/ast/MongoDBConstant.java b/src/sqlancer/mongodb/ast/MongoDBConstant.java index 75caa0efd..f52c18666 100644 --- a/src/sqlancer/mongodb/ast/MongoDBConstant.java +++ b/src/sqlancer/mongodb/ast/MongoDBConstant.java @@ -14,6 +14,8 @@ private MongoDBConstant() { public abstract String getLogValue(); + public abstract Object getValue(); + public static class MongoDBNullConstant extends MongoDBConstant { @Override @@ -25,6 +27,15 @@ public void setValueInDocument(Document document, String key) { public String getLogValue() { return "null"; } + + @Override + public Object getValue() { + return null; + } + } + + public static Node createNullConstant() { + return new MongoDBNullConstant(); } public static class MongoDBIntegerConstant extends MongoDBConstant { @@ -42,7 +53,12 @@ public void setValueInDocument(Document document, String key) { @Override public String getLogValue() { - return String.valueOf(value); + return "NumberInt(" + value + ")"; + } + + @Override + public Integer getValue() { + return value; } } @@ -69,7 +85,12 @@ public void setValueInDocument(Document document, String key) { @Override public String getLogValue() { - return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"") + "\""; + return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n") + "\""; + } + + @Override + public String getValue() { + return value; } } @@ -92,10 +113,12 @@ public void setValueInDocument(Document document, String key) { @Override public String getLogValue() { - if (value) { - return "true"; - } - return "false"; + return String.valueOf(value); + } + + @Override + public Boolean getValue() { + return value; } } @@ -120,6 +143,11 @@ public void setValueInDocument(Document document, String key) { public String getLogValue() { return String.valueOf(value); } + + @Override + public Double getValue() { + return value; + } } public static Node createDoubleConstant(double value) { @@ -141,7 +169,12 @@ public void setValueInDocument(Document document, String key) { @Override public String getLogValue() { - return String.valueOf(value); + return "new Date(" + value.getValue() + ")"; + } + + @Override + public BsonDateTime getValue() { + return value; } } @@ -164,7 +197,12 @@ public void setValueInDocument(Document document, String key) { @Override public String getLogValue() { - return String.valueOf(value); + return "Timestamp(" + value.getValue() + ",1)"; + } + + @Override + public BsonTimestamp getValue() { + return value; } } diff --git a/src/sqlancer/mongodb/ast/MongoDBSelect.java b/src/sqlancer/mongodb/ast/MongoDBSelect.java index c6252893f..39b021ac7 100644 --- a/src/sqlancer/mongodb/ast/MongoDBSelect.java +++ b/src/sqlancer/mongodb/ast/MongoDBSelect.java @@ -69,7 +69,7 @@ public Node getFilterClause() { return filterClause; } - public boolean hasWhere() { + public boolean hasFilter() { return hasFilter; } diff --git a/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java b/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java new file mode 100644 index 000000000..01b2b0499 --- /dev/null +++ b/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java @@ -0,0 +1,16 @@ +package sqlancer.mongodb.ast; + +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.mongodb.gen.MongoDBExpressionGenerator.MongoDBUnaryLogicalOperator; + +public class MongoDBUnaryLogicalOperatorNode extends NewUnaryPrefixOperatorNode { + + public MongoDBUnaryLogicalOperatorNode(Node expr, MongoDBUnaryLogicalOperator op) { + super(expr, op); + } + + public MongoDBUnaryLogicalOperator operator() { + return (MongoDBUnaryLogicalOperator) op; + } +} diff --git a/src/sqlancer/mongodb/ast/MongoDBDummyPredicate.java b/src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java similarity index 53% rename from src/sqlancer/mongodb/ast/MongoDBDummyPredicate.java rename to src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java index a9ecc3325..eae143e7d 100644 --- a/src/sqlancer/mongodb/ast/MongoDBDummyPredicate.java +++ b/src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java @@ -2,6 +2,6 @@ import sqlancer.common.ast.newast.Node; -public class MongoDBDummyPredicate implements Node { +public class MongoDBUnsupportedPredicate implements Node { } diff --git a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java index 090016809..abcf522d9 100644 --- a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java @@ -3,6 +3,7 @@ import org.bson.Document; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; import sqlancer.mongodb.ast.MongoDBConstant; @@ -13,6 +14,7 @@ import sqlancer.mongodb.ast.MongoDBConstant.MongoDBNullConstant; import sqlancer.mongodb.ast.MongoDBConstant.MongoDBStringConstant; import sqlancer.mongodb.ast.MongoDBConstant.MongoDBTimestampConstant; +import sqlancer.mongodb.ast.MongoDBExpression; public class MongoDBConstantGenerator { private final MongoDBGlobalState globalState; @@ -21,6 +23,28 @@ public MongoDBConstantGenerator(MongoDBGlobalState globalState) { this.globalState = globalState; } + public Node generateConstantWithType(MongoDBDataType option) { + if (Randomly.getBooleanWithSmallProbability()) { + return MongoDBConstant.createNullConstant(); + } + switch (option) { + case DATE_TIME: + return MongoDBConstant.createDateTimeConstant(globalState.getRandomly().getInteger()); + case BOOLEAN: + return MongoDBConstant.createBooleanConstant(Randomly.getBoolean()); + case DOUBLE: + return MongoDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case STRING: + return MongoDBConstant.createStringConstant(globalState.getRandomly().getString()); + case INTEGER: + return MongoDBConstant.createIntegerConstant((int) globalState.getRandomly().getInteger()); + case TIMESTAMP: + return MongoDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(option); + } + } + public void addRandomConstant(Document document, String key) { MongoDBDataType type = MongoDBDataType.getRandom(); addRandomConstantWithType(document, key, type); diff --git a/src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java index f2c05dda2..597e2c3e8 100644 --- a/src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java @@ -1,40 +1,227 @@ package sqlancer.mongodb.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.bson.conversions.Bson; + +import com.mongodb.client.model.Filters; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.mongodb.ast.MongoDBDummyPredicate; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; +import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; +import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; +import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; +import sqlancer.mongodb.ast.MongoDBConstant; import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; +import sqlancer.mongodb.ast.MongoDBUnsupportedPredicate; import sqlancer.mongodb.test.MongoDBColumnTestReference; public class MongoDBExpressionGenerator extends UntypedExpressionGenerator, MongoDBColumnTestReference> { - /* - * private final MongoDBGlobalState globalState; - * - * public MongoDBExpressionGenerator(MongoDBGlobalState globalState) { this.globalState = globalState; } - */ + + private final MongoDBGlobalState globalState; + + private enum NonLeafExpression { + BINARY_LOGICAL, UNARY_LOGICAL + } + + public MongoDBExpressionGenerator(MongoDBGlobalState globalState) { + this.globalState = globalState; + } + @Override - public Node generateConstant() { - throw new UnsupportedOperationException(); + public Node generateLeafNode() { + MongoDBBinaryComparisonOperator operator = MongoDBBinaryComparisonOperator.getRandom(); + return new MongoDBBinaryComparisonNode(generateColumn(), generateConstant(), operator); } @Override protected Node generateExpression(int depth) { - return new MongoDBDummyPredicate<>(); + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + + List possibleOptions = new ArrayList<>(Arrays.asList(NonLeafExpression.values())); + NonLeafExpression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case BINARY_LOGICAL: + MongoDBBinaryLogicalOperator binaryOperator = MongoDBBinaryLogicalOperator.getRandom(); + return new MongoDBBinaryLogicalNode(generateExpression(depth + 1), generateExpression(depth + 1), + binaryOperator); + case UNARY_LOGICAL: + MongoDBUnaryLogicalOperator unaryOperator = MongoDBUnaryLogicalOperator.getRandom(); + return new MongoDBUnaryLogicalOperatorNode(generateExpression(depth + 1), unaryOperator); + default: + throw new AssertionError(); + } + } + + @Override + public Node generateConstant() { + MongoDBDataType type = MongoDBDataType.getRandom(); + MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); + return generator.generateConstantWithType(type); } @Override protected Node generateColumn() { - throw new UnsupportedOperationException(); + return Randomly.fromList(columns); } @Override public Node negatePredicate(Node predicate) { - return new MongoDBDummyPredicate<>(); + return new MongoDBUnaryLogicalOperatorNode(predicate, MongoDBUnaryLogicalOperator.NOT); } @Override public Node isNull(Node expr) { - return new MongoDBDummyPredicate<>(); + return new MongoDBUnsupportedPredicate<>(); + } + + public enum MongoDBUnaryLogicalOperator implements Operator { + NOT { + @Override + public Bson applyOperator(Bson inner) { + return Filters.nor(inner, Filters.exists("_id", false)); + } + + @Override + public String getTextRepresentation() { + return "{$nor: [{ _id: {$exists: false}}, "; + } + }; + + public abstract Bson applyOperator(Bson inner); + + public static MongoDBUnaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum MongoDBBinaryLogicalOperator implements Operator { + AND { + @Override + public Bson applyOperator(Bson left, Bson right) { + return Filters.and(left, right); + } + + @Override + public String getTextRepresentation() { + return "$and"; + } + }, + OR { + @Override + public Bson applyOperator(Bson left, Bson right) { + return Filters.or(left, right); + } + + @Override + public String getTextRepresentation() { + return "$or"; + } + }, + NOR { + @Override + public Bson applyOperator(Bson left, Bson right) { + return Filters.nor(left, right); + } + + @Override + public String getTextRepresentation() { + return "$nor"; + } + }; + + public abstract Bson applyOperator(Bson left, Bson right); + + public static MongoDBBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum MongoDBBinaryComparisonOperator implements Operator { + EQUALS { + @Override + public Bson applyOperator(String columnName, MongoDBConstant constant) { + return Filters.eq(columnName, constant.getValue()); + } + + @Override + public String getTextRepresentation() { + return "$eq"; + } + }, + NOT_EQUALS { + @Override + public Bson applyOperator(String columnName, MongoDBConstant constant) { + return Filters.ne(columnName, constant.getValue()); + } + + @Override + public String getTextRepresentation() { + return "$ne"; + } + }, + GREATER { + @Override + public Bson applyOperator(String columnName, MongoDBConstant constant) { + return Filters.gt(columnName, constant.getValue()); + } + + @Override + public String getTextRepresentation() { + return "$gt"; + } + + }, + LESS { + @Override + public Bson applyOperator(String columnName, MongoDBConstant constant) { + return Filters.lt(columnName, constant.getValue()); + } + + @Override + public String getTextRepresentation() { + return "$lt"; + } + + }, + GREATER_EQUAL { + @Override + public Bson applyOperator(String columnName, MongoDBConstant constant) { + return Filters.gte(columnName, constant.getValue()); + + } + + @Override + public String getTextRepresentation() { + return "$gte"; + } + + }, + LESS_EQUAL { + @Override + public Bson applyOperator(String columnName, MongoDBConstant constant) { + return Filters.lte(columnName, constant.getValue()); + } + + @Override + public String getTextRepresentation() { + return "$lte"; + } + }; + + public abstract Bson applyOperator(String columnName, MongoDBConstant constant); + + public static MongoDBBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } } } diff --git a/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java b/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java index f6c8e9d94..59a2a6724 100644 --- a/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java +++ b/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java @@ -1,8 +1,10 @@ package sqlancer.mongodb.test; +import sqlancer.common.ast.newast.Node; import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; +import sqlancer.mongodb.ast.MongoDBExpression; -public class MongoDBColumnTestReference { +public class MongoDBColumnTestReference implements Node { private final MongoDBColumn columnReference; private final boolean inMainTable; @@ -31,4 +33,8 @@ public String getTableName() { public String getPlainName() { return this.columnReference.getName(); } + + public MongoDBColumn getColumnReference() { + return columnReference; + } } diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java index 4fda2a64b..c44cc3bed 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -37,7 +37,7 @@ public void check() throws Exception { targetTables = schema.getRandomTableNonEmptyTables(); mainTable = targetTables.getTables().get(0); generateTargetColumns(); - expressionGenerator = new MongoDBExpressionGenerator().setColumns(targetColumns); + expressionGenerator = new MongoDBExpressionGenerator(state).setColumns(targetColumns); initializeTernaryPredicateVariants(); select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); select.setProjectionList(targetColumns); diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java index 11ae2c8ae..8c9522aaa 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java @@ -1,5 +1,10 @@ package sqlancer.mongodb.test; +import java.util.List; + +import org.bson.Document; + +import sqlancer.mongodb.MongoDBComparatorHelper; import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; import sqlancer.mongodb.query.MongoDBSelectQuery; @@ -15,17 +20,20 @@ public void check() throws Exception { select.setFilterClause(null); MongoDBSelectQuery q = new MongoDBSelectQuery(select); q.executeAndGet(state); - /* - * List firstResultSet = q.getResultSet(); select.setWhereClause(predicate); q = new - * MongoDBFindQueryAdapter(select, mainTable); q.executeAndGet(state); List secondResultSet = - * q.resultSet; - * - * select.setWhereClause(negatedPredicate); q = new MongoDBFindQueryAdapter(select, mainTable); - * q.executeAndGet(state); List thirdResultSet = q.resultSet; - * - * secondResultSet.addAll(thirdResultSet); MongoDBComparatorHelper.assumeResultSetsAreEqual(firstResultSet, - * secondResultSet, q); - * - */ + + List firstResultSet = q.getResultSet(); + select.setFilterClause(predicate); + q = new MongoDBSelectQuery(select); + q.executeAndGet(state); + List secondResultSet = q.getResultSet(); + + select.setFilterClause(negatedPredicate); + q = new MongoDBSelectQuery(select); + q.executeAndGet(state); + List thirdResultSet = q.getResultSet(); + + secondResultSet.addAll(thirdResultSet); + MongoDBComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, q); + } } diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java index 50d43a3ee..4d68ac09e 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java @@ -3,16 +3,56 @@ import java.util.ArrayList; import java.util.List; +import sqlancer.common.ast.newast.Node; +import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; +import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; import sqlancer.mongodb.ast.MongoDBConstant; import sqlancer.mongodb.ast.MongoDBExpression; import sqlancer.mongodb.ast.MongoDBSelect; +import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; import sqlancer.mongodb.test.MongoDBColumnTestReference; public class MongoDBToLogVisitor extends MongoDBVisitor { private String mainTableName; private List lookups; + private String filter; private String projects; + private boolean hasFilter; + + public String visitLog(Node expr) { + if (expr instanceof MongoDBUnaryLogicalOperatorNode) { + return visit((MongoDBUnaryLogicalOperatorNode) expr); + } else if (expr instanceof MongoDBBinaryLogicalNode) { + return visit((MongoDBBinaryLogicalNode) expr); + } else if (expr instanceof MongoDBBinaryComparisonNode) { + return visit((MongoDBBinaryComparisonNode) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + public String visit(MongoDBUnaryLogicalOperatorNode expr) { + String inner = visitLog(expr.getExpr()); + return expr.operator().getTextRepresentation() + inner + "]}"; + } + + public String visit(MongoDBBinaryLogicalNode expr) { + String left = visitLog(expr.getLeft()); + String right = visitLog(expr.getRight()); + + return "{" + expr.operator().getTextRepresentation() + ":[" + left + "," + right + "]}"; + } + + public String visit(MongoDBBinaryComparisonNode expr) { + Node left = expr.getLeft(); + Node right = expr.getRight(); + assert left instanceof MongoDBColumnTestReference; + assert right instanceof MongoDBConstant; + + return "{\"" + ((MongoDBColumnTestReference) left).getQueryString() + "\": {" + + expr.operator().getTextRepresentation() + ": " + ((MongoDBConstant) right).getLogValue() + "}}"; + } @Override public void visit(MongoDBConstant c) { @@ -21,11 +61,19 @@ public void visit(MongoDBConstant c) { @Override public void visit(MongoDBSelect select) { + hasFilter = select.hasFilter(); mainTableName = select.getMainTableName(); setLookups(select); + if (hasFilter) { + setFilter(select); + } setProjects(select); } + private void setFilter(MongoDBSelect select) { + filter = visitLog(select.getFilterClause()); + } + private void setLookups(MongoDBSelect select) { lookups = new ArrayList<>(); for (MongoDBColumnTestReference testReference : select.getLookupList()) { @@ -58,12 +106,11 @@ public String getStringLog() { for (String lookup : lookups) { sb.append(lookup); } - // sb.append(",\n"); - // if(select.hasWhere()) { - // sb.append("{ $match: "); - // sb.append(mongoDBToQueryVisitor.getFilterLog()); - // sb.append("},\n"); - // } + if (hasFilter) { + sb.append("{ $match: "); + sb.append(filter); + sb.append("},\n"); + } sb.append("{ $project : "); sb.append(projects); sb.append("}])\n"); diff --git a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java index 3ad57a3cb..25fd94dbf 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java @@ -1,5 +1,6 @@ package sqlancer.mongodb.visitor; +import static com.mongodb.client.model.Aggregates.match; import static com.mongodb.client.model.Aggregates.project; import static com.mongodb.client.model.Projections.fields; import static com.mongodb.client.model.Projections.include; @@ -11,15 +12,54 @@ import com.mongodb.client.model.Aggregates; +import sqlancer.common.ast.newast.Node; +import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; +import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; import sqlancer.mongodb.ast.MongoDBConstant; import sqlancer.mongodb.ast.MongoDBExpression; import sqlancer.mongodb.ast.MongoDBSelect; +import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; import sqlancer.mongodb.test.MongoDBColumnTestReference; public class MongoDBToQueryVisitor extends MongoDBVisitor { private List lookup; + private Bson filter; private Bson projection; + private boolean hasFilter; + + public Bson visitBson(Node expr) { + if (expr instanceof MongoDBUnaryLogicalOperatorNode) { + return visit((MongoDBUnaryLogicalOperatorNode) expr); + } else if (expr instanceof MongoDBBinaryLogicalNode) { + return visit((MongoDBBinaryLogicalNode) expr); + } else if (expr instanceof MongoDBBinaryComparisonNode) { + return visit((MongoDBBinaryComparisonNode) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + public Bson visit(MongoDBUnaryLogicalOperatorNode expr) { + Bson inner = visitBson(expr.getExpr()); + return expr.operator().applyOperator(inner); + } + + public Bson visit(MongoDBBinaryLogicalNode expr) { + Bson left = visitBson(expr.getLeft()); + Bson right = visitBson(expr.getRight()); + return expr.operator().applyOperator(left, right); + } + + public Bson visit(MongoDBBinaryComparisonNode expr) { + Node left = expr.getLeft(); + Node right = expr.getRight(); + assert left instanceof MongoDBColumnTestReference; + assert right instanceof MongoDBConstant; + + String columnName = ((MongoDBColumnTestReference) left).getQueryString(); + return expr.operator().applyOperator(columnName, (MongoDBConstant) right); + } @Override public void visit(MongoDBConstant c) { @@ -28,13 +68,18 @@ public void visit(MongoDBConstant c) { @Override public void visit(MongoDBSelect select) { + hasFilter = select.hasFilter(); setLookup(select); - // if(select.hasWhere()) { - // // DO STUFF - // } + if (hasFilter) { + setFilter(select); + } setProjection(select); } + private void setFilter(MongoDBSelect select) { + filter = match(this.visitBson(select.getFilterClause())); + } + private void setLookup(MongoDBSelect select) { lookup = new ArrayList<>(); for (MongoDBColumnTestReference reference : select.getLookupList()) { @@ -56,7 +101,9 @@ private void setProjection(MongoDBSelect select) { public List getPipeline() { List result = new ArrayList<>(lookup); - // TODO Patrick: Add Match if where... + if (hasFilter) { + result.add(filter); + } result.add(projection); return result; } diff --git a/src/sqlancer/mongodb/visitor/MongoDBVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBVisitor.java index 8c207a564..67b6ca741 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBVisitor.java @@ -10,6 +10,7 @@ import sqlancer.mongodb.ast.MongoDBSelect; public abstract class MongoDBVisitor { + public abstract void visit(MongoDBConstant c); public abstract void visit(MongoDBSelect s); From 1cd7c47c038cd60f343988049f31223e0f4031c1 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 26 Jan 2021 13:29:54 +0100 Subject: [PATCH 0373/1284] Expand expression generation to computed functions for MongoDB Computed functions allows to perform arithmetic operations in queries. It is part of the projection pipeline stage, where a new field is added that holds the result. This update adds support for random computed fields with functions such as add, multiply, pow, sqrt and more. --- src/sqlancer/mongodb/MongoDBOptions.java | 3 + .../ast/MongoDBBinaryComparisonNode.java | 2 +- .../mongodb/ast/MongoDBBinaryLogicalNode.java | 2 +- src/sqlancer/mongodb/ast/MongoDBConstant.java | 39 ++++++++ src/sqlancer/mongodb/ast/MongoDBSelect.java | 19 ++++ .../ast/MongoDBUnaryLogicalOperatorNode.java | 2 +- .../MongoDBComputedExpressionGenerator.java | 89 +++++++++++++++++++ ...a => MongoDBMatchExpressionGenerator.java} | 4 +- .../test/MongoDBQueryPartitioningBase.java | 21 ++++- .../visitor/MongoDBToQueryVisitor.java | 50 ++++++++++- 10 files changed, 222 insertions(+), 9 deletions(-) create mode 100644 src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java rename src/sqlancer/mongodb/gen/{MongoDBExpressionGenerator.java => MongoDBMatchExpressionGenerator.java} (98%) diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index a6306b515..af598c34e 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -28,6 +28,9 @@ public class MongoDBOptions implements DBMSSpecificOptions oracles = Arrays.asList(QUERY_PARTITIONING); diff --git a/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java b/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java index f5e3ea173..21675250a 100644 --- a/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java +++ b/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java @@ -2,7 +2,7 @@ import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBExpressionGenerator.MongoDBBinaryComparisonOperator; +import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator; public class MongoDBBinaryComparisonNode extends NewBinaryOperatorNode { public MongoDBBinaryComparisonNode(Node left, Node right, diff --git a/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java b/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java index c3f18a248..efb8d8294 100644 --- a/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java +++ b/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java @@ -2,7 +2,7 @@ import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBExpressionGenerator.MongoDBBinaryLogicalOperator; +import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator; public class MongoDBBinaryLogicalNode extends NewBinaryOperatorNode { public MongoDBBinaryLogicalNode(Node left, Node right, diff --git a/src/sqlancer/mongodb/ast/MongoDBConstant.java b/src/sqlancer/mongodb/ast/MongoDBConstant.java index f52c18666..86f783b48 100644 --- a/src/sqlancer/mongodb/ast/MongoDBConstant.java +++ b/src/sqlancer/mongodb/ast/MongoDBConstant.java @@ -1,5 +1,7 @@ package sqlancer.mongodb.ast; +import java.io.Serializable; + import org.bson.BsonDateTime; import org.bson.BsonTimestamp; import org.bson.Document; @@ -16,6 +18,8 @@ private MongoDBConstant() { public abstract Object getValue(); + public abstract Serializable getSerializedValue(); + public static class MongoDBNullConstant extends MongoDBConstant { @Override @@ -32,6 +36,11 @@ public String getLogValue() { public Object getValue() { return null; } + + @Override + public Serializable getSerializedValue() { + return null; + } } public static Node createNullConstant() { @@ -60,6 +69,11 @@ public String getLogValue() { public Integer getValue() { return value; } + + @Override + public Serializable getSerializedValue() { + return value; + } } public static Node createIntegerConstant(int value) { @@ -92,6 +106,11 @@ public String getLogValue() { public String getValue() { return value; } + + @Override + public Serializable getSerializedValue() { + return value; + } } public static Node createStringConstant(String value) { @@ -120,6 +139,11 @@ public String getLogValue() { public Boolean getValue() { return value; } + + @Override + public Serializable getSerializedValue() { + return value; + } } public static Node createBooleanConstant(boolean value) { @@ -148,6 +172,11 @@ public String getLogValue() { public Double getValue() { return value; } + + @Override + public Serializable getSerializedValue() { + return value; + } } public static Node createDoubleConstant(double value) { @@ -176,6 +205,11 @@ public String getLogValue() { public BsonDateTime getValue() { return value; } + + @Override + public Serializable getSerializedValue() { + return value.getValue(); + } } public static Node createDateTimeConstant(long value) { @@ -204,6 +238,11 @@ public String getLogValue() { public BsonTimestamp getValue() { return value; } + + @Override + public Serializable getSerializedValue() { + return value.getValue(); + } } public static Node createTimestampConstant(long value) { diff --git a/src/sqlancer/mongodb/ast/MongoDBSelect.java b/src/sqlancer/mongodb/ast/MongoDBSelect.java index 39b021ac7..de65fb01d 100644 --- a/src/sqlancer/mongodb/ast/MongoDBSelect.java +++ b/src/sqlancer/mongodb/ast/MongoDBSelect.java @@ -13,6 +13,8 @@ public class MongoDBSelect implements Node { List lookupList; boolean hasFilter; Node filterClause; + boolean hasComputed; + List> computedClauses; public MongoDBSelect(String mainTableName, MongoDBColumnTestReference joinColumn) { this.mainTableName = mainTableName; @@ -73,4 +75,21 @@ public boolean hasFilter() { return hasFilter; } + public void setComputedClause(List> computedClause) { + if (computedClause == null) { + hasComputed = false; + this.computedClauses = null; + return; + } + hasComputed = true; + this.computedClauses = computedClause; + } + + public List> getComputedClause() { + return computedClauses; + } + + public boolean hasComputed() { + return hasComputed; + } } diff --git a/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java b/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java index 01b2b0499..a34fe27e5 100644 --- a/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java +++ b/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java @@ -2,7 +2,7 @@ import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBExpressionGenerator.MongoDBUnaryLogicalOperator; +import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBUnaryLogicalOperator; public class MongoDBUnaryLogicalOperatorNode extends NewUnaryPrefixOperatorNode { diff --git a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java new file mode 100644 index 000000000..1090b3f2f --- /dev/null +++ b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java @@ -0,0 +1,89 @@ +package sqlancer.mongodb.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; +import sqlancer.mongodb.MongoDBSchema; +import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.test.MongoDBColumnTestReference; + +public class MongoDBComputedExpressionGenerator + extends UntypedExpressionGenerator, MongoDBColumnTestReference> { + + private final MongoDBGlobalState globalState; + + @Override + public Node generateLeafNode() { + ComputedFunction function = ComputedFunction.getRandom(); + List> expressions = new ArrayList<>(); + for (int i = 0; i < function.getNrArgs(); i++) { + expressions.add(super.generateLeafNode()); + } + return new NewFunctionNode<>(expressions, function); + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + ComputedFunction func = ComputedFunction.getRandom(); + return new NewFunctionNode<>(generateExpressions(depth + 1, func.getNrArgs()), func); + } + + public MongoDBComputedExpressionGenerator(MongoDBGlobalState globalState) { + this.globalState = globalState; + } + + public enum ComputedFunction { + ADD(2, "$add"), MULTIPLY(2, "$multiply"), DIVIDE(2, "$divide"), POW(2, "$pow"), SQRT(1, "$sqrt"), + LOG(2, "$log"), AVG(2, "$avg"), EXP(1, "$exp"); + + private final int nrArgs; + private final String operatorName; + + ComputedFunction(int nrArgs, String operatorName) { + this.nrArgs = nrArgs; + this.operatorName = operatorName; + } + + public static ComputedFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + public String getOperator() { + return operatorName; + } + } + + @Override + public Node generateConstant() { + MongoDBSchema.MongoDBDataType type = MongoDBSchema.MongoDBDataType.getRandom(); + MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); + return generator.generateConstantWithType(type); + } + + @Override + protected Node generateColumn() { + return Randomly.fromList(columns); + } + + @Override + public Node negatePredicate(Node predicate) { + throw new UnsupportedOperationException(); + } + + @Override + public Node isNull(Node expr) { + throw new UnsupportedOperationException(); + } +} diff --git a/src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java similarity index 98% rename from src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java rename to src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java index 597e2c3e8..df6791794 100644 --- a/src/sqlancer/mongodb/gen/MongoDBExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java @@ -22,7 +22,7 @@ import sqlancer.mongodb.ast.MongoDBUnsupportedPredicate; import sqlancer.mongodb.test.MongoDBColumnTestReference; -public class MongoDBExpressionGenerator +public class MongoDBMatchExpressionGenerator extends UntypedExpressionGenerator, MongoDBColumnTestReference> { private final MongoDBGlobalState globalState; @@ -31,7 +31,7 @@ private enum NonLeafExpression { BINARY_LOGICAL, UNARY_LOGICAL } - public MongoDBExpressionGenerator(MongoDBGlobalState globalState) { + public MongoDBMatchExpressionGenerator(MongoDBGlobalState globalState) { this.globalState = globalState; } diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java index c44cc3bed..c33bcf9e6 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -15,7 +15,8 @@ import sqlancer.mongodb.MongoDBSchema.MongoDBTables; import sqlancer.mongodb.ast.MongoDBExpression; import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.gen.MongoDBExpressionGenerator; +import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator; +import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; public class MongoDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, MongoDBGlobalState> implements TestOracle { @@ -24,7 +25,7 @@ public class MongoDBQueryPartitioningBase protected MongoDBTables targetTables; protected MongoDBTable mainTable; protected List targetColumns; - protected MongoDBExpressionGenerator expressionGenerator; + protected MongoDBMatchExpressionGenerator expressionGenerator; protected MongoDBSelect select; public MongoDBQueryPartitioningBase(MongoDBGlobalState state) { @@ -37,7 +38,7 @@ public void check() throws Exception { targetTables = schema.getRandomTableNonEmptyTables(); mainTable = targetTables.getTables().get(0); generateTargetColumns(); - expressionGenerator = new MongoDBExpressionGenerator(state).setColumns(targetColumns); + expressionGenerator = new MongoDBMatchExpressionGenerator(state).setColumns(targetColumns); initializeTernaryPredicateVariants(); select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); select.setProjectionList(targetColumns); @@ -46,6 +47,20 @@ public void check() throws Exception { } else { select.setLookupList(Randomly.nonEmptySubset(targetColumns)); } + if (state.getDmbsSpecificOptions().testComputedValues) { + generateComputedColumns(); + } + } + + private void generateComputedColumns() { + List> computedColumns = new ArrayList<>(); + int numberComputedColumns = state.getRandomly().getInteger(1, 4); + MongoDBComputedExpressionGenerator generator = new MongoDBComputedExpressionGenerator(state) + .setColumns(targetColumns); + for (int i = 0; i < numberComputedColumns; i++) { + computedColumns.add(generator.generateExpression()); + } + select.setComputedClause(computedColumns); } private void generateTargetColumns() { diff --git a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java index 25fd94dbf..1ebeee84e 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java @@ -5,13 +5,17 @@ import static com.mongodb.client.model.Projections.fields; import static com.mongodb.client.model.Projections.include; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import org.bson.Document; import org.bson.conversions.Bson; import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Projections; +import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.Node; import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; @@ -19,6 +23,7 @@ import sqlancer.mongodb.ast.MongoDBExpression; import sqlancer.mongodb.ast.MongoDBSelect; import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; +import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator.ComputedFunction; import sqlancer.mongodb.test.MongoDBColumnTestReference; public class MongoDBToQueryVisitor extends MongoDBVisitor { @@ -40,6 +45,39 @@ public Bson visitBson(Node expr) { } } + public Document visitComputed(Node expr) { + if (expr instanceof NewFunctionNode) { + return visitComputed((NewFunctionNode) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + public Document visitComputed(NewFunctionNode expr) { + List visitedArgs = new ArrayList<>(); + for (int i = 0; i < expr.getArgs().size(); i++) { + if (expr.getArgs().get(i) instanceof MongoDBConstant) { + visitedArgs.add(((MongoDBConstant) expr.getArgs().get(i)).getSerializedValue()); + continue; + } + if (expr.getArgs().get(i) instanceof MongoDBColumnTestReference) { + visitedArgs.add("$" + ((MongoDBColumnTestReference) expr.getArgs().get(i)).getQueryString()); + continue; + } + if (expr.getArgs().get(i) instanceof NewFunctionNode) { + visitedArgs.add(visitComputed((NewFunctionNode) expr.getArgs().get(i))); + } else { + throw new AssertionError(); + } + } + if (expr.getFunc() instanceof ComputedFunction) { + return new Document(((ComputedFunction) expr.getFunc()).getOperator(), visitedArgs); + } else { + throw new AssertionError(expr.getClass()); + } + + } + public Bson visit(MongoDBUnaryLogicalOperatorNode expr) { Bson inner = visitBson(expr.getExpr()); return expr.operator().applyOperator(inner); @@ -96,7 +134,17 @@ private void setProjection(MongoDBSelect select) { for (MongoDBColumnTestReference ref : select.getProjectionList()) { stringProjects.add(ref.getQueryString()); } - projection = project(fields(include(stringProjects))); + List projections = new ArrayList<>(); + projections.add(include(stringProjects)); + if (select.hasComputed()) { + String name = "computed"; + int number = 0; + for (Node expressionNode : select.getComputedClause()) { + projections.add(Projections.computed(name + number, visitComputed(expressionNode))); + number++; + } + } + projection = project(fields(projections)); } public List getPipeline() { From 147ce91db809ec8e196f5edd4cf7d394fbfb88e6 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 26 Jan 2021 15:01:22 +0100 Subject: [PATCH 0374/1284] Add regular expressions as part of the randomized expression generation in MongoDB Similar to binary comparison regular expression is a new Leaf node that can be added in the tree. To generate the pattern the existing random string generator is used. --- src/sqlancer/mongodb/MongoDBOptions.java | 3 ++ .../mongodb/ast/MongoDBRegexNode.java | 17 ++++++++ .../gen/MongoDBMatchExpressionGenerator.java | 39 ++++++++++++++++++- .../mongodb/visitor/MongoDBToLogVisitor.java | 14 +++++++ .../visitor/MongoDBToQueryVisitor.java | 15 +++++++ 5 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/mongodb/ast/MongoDBRegexNode.java diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index af598c34e..5c16bfd4d 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -31,6 +31,9 @@ public class MongoDBOptions implements DBMSSpecificOptions oracles = Arrays.asList(QUERY_PARTITIONING); diff --git a/src/sqlancer/mongodb/ast/MongoDBRegexNode.java b/src/sqlancer/mongodb/ast/MongoDBRegexNode.java new file mode 100644 index 000000000..154ebeffa --- /dev/null +++ b/src/sqlancer/mongodb/ast/MongoDBRegexNode.java @@ -0,0 +1,17 @@ +package sqlancer.mongodb.ast; + +import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBRegexOperator.REGEX; + +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBRegexOperator; + +public class MongoDBRegexNode extends NewBinaryOperatorNode { + public MongoDBRegexNode(Node left, Node right) { + super(left, right, REGEX); + } + + public MongoDBRegexOperator operator() { + return (MongoDBRegexOperator) op; + } +} diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java index df6791794..272a52d30 100644 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java @@ -18,6 +18,7 @@ import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; import sqlancer.mongodb.ast.MongoDBConstant; import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBRegexNode; import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; import sqlancer.mongodb.ast.MongoDBUnsupportedPredicate; import sqlancer.mongodb.test.MongoDBColumnTestReference; @@ -27,6 +28,11 @@ public class MongoDBMatchExpressionGenerator private final MongoDBGlobalState globalState; + private enum LeafExpression { + BINARY_COMPARISON, REGEX + + } + private enum NonLeafExpression { BINARY_LOGICAL, UNARY_LOGICAL } @@ -37,8 +43,21 @@ public MongoDBMatchExpressionGenerator(MongoDBGlobalState globalState) { @Override public Node generateLeafNode() { - MongoDBBinaryComparisonOperator operator = MongoDBBinaryComparisonOperator.getRandom(); - return new MongoDBBinaryComparisonNode(generateColumn(), generateConstant(), operator); + List possibleOptions = new ArrayList<>(Arrays.asList(LeafExpression.values())); + if (!globalState.getDmbsSpecificOptions().testWithRegex) { + possibleOptions.remove(LeafExpression.REGEX); + } + LeafExpression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case BINARY_COMPARISON: + MongoDBBinaryComparisonOperator operator = MongoDBBinaryComparisonOperator.getRandom(); + return new MongoDBBinaryComparisonNode(generateColumn(), generateConstant(), operator); + case REGEX: + return new MongoDBRegexNode(generateColumn(), + new MongoDBConstantGenerator(globalState).generateConstantWithType(MongoDBDataType.STRING)); + default: + throw new AssertionError(); + } } @Override @@ -224,4 +243,20 @@ public static MongoDBBinaryComparisonOperator getRandom() { return Randomly.fromOptions(values()); } } + + public enum MongoDBRegexOperator implements Operator { + REGEX { + @Override + public Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex) { + return Filters.regex(columnName, regex.getStringValue(), ""); + } + + @Override + public String getTextRepresentation() { + return "$regex"; + } + }; + + public abstract Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex); + } } diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java index 4d68ac09e..bef5e5952 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java @@ -8,6 +8,7 @@ import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; import sqlancer.mongodb.ast.MongoDBConstant; import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBRegexNode; import sqlancer.mongodb.ast.MongoDBSelect; import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; import sqlancer.mongodb.test.MongoDBColumnTestReference; @@ -27,6 +28,8 @@ public String visitLog(Node expr) { return visit((MongoDBBinaryLogicalNode) expr); } else if (expr instanceof MongoDBBinaryComparisonNode) { return visit((MongoDBBinaryComparisonNode) expr); + } else if (expr instanceof MongoDBRegexNode) { + return visit((MongoDBRegexNode) expr); } else { throw new AssertionError(expr.getClass()); } @@ -54,6 +57,17 @@ public String visit(MongoDBBinaryComparisonNode expr) { + expr.operator().getTextRepresentation() + ": " + ((MongoDBConstant) right).getLogValue() + "}}"; } + public String visit(MongoDBRegexNode expr) { + Node left = expr.getLeft(); + Node right = expr.getRight(); + assert left instanceof MongoDBColumnTestReference; + assert right instanceof MongoDBConstant.MongoDBStringConstant; + + return "{\"" + ((MongoDBColumnTestReference) left).getQueryString() + "\": {" + + expr.operator().getTextRepresentation() + ": \'" + + ((MongoDBConstant.MongoDBStringConstant) right).getStringValue() + "\', $options: \'\'}}"; + } + @Override public void visit(MongoDBConstant c) { throw new UnsupportedOperationException(); diff --git a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java index 1ebeee84e..fb027add6 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java @@ -20,7 +20,9 @@ import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; import sqlancer.mongodb.ast.MongoDBConstant; +import sqlancer.mongodb.ast.MongoDBConstant.MongoDBStringConstant; import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBRegexNode; import sqlancer.mongodb.ast.MongoDBSelect; import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator.ComputedFunction; @@ -40,6 +42,8 @@ public Bson visitBson(Node expr) { return visit((MongoDBBinaryLogicalNode) expr); } else if (expr instanceof MongoDBBinaryComparisonNode) { return visit((MongoDBBinaryComparisonNode) expr); + } else if (expr instanceof MongoDBRegexNode) { + return visit((MongoDBRegexNode) expr); } else { throw new AssertionError(expr.getClass()); } @@ -89,6 +93,17 @@ public Bson visit(MongoDBBinaryLogicalNode expr) { return expr.operator().applyOperator(left, right); } + public Bson visit(MongoDBRegexNode expr) { + Node left = expr.getLeft(); + Node right = expr.getRight(); + + assert left instanceof MongoDBColumnTestReference; + assert right instanceof MongoDBStringConstant; + + String columnName = ((MongoDBColumnTestReference) left).getQueryString(); + return expr.operator().applyOperator(columnName, (MongoDBStringConstant) right); + } + public Bson visit(MongoDBBinaryComparisonNode expr) { Node left = expr.getLeft(); Node right = expr.getRight(); From 5d280448a932db6eba83d1bb7572846bdce36418 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sun, 31 Jan 2021 01:37:01 +0100 Subject: [PATCH 0375/1284] Ignore expected errors in MongoDB queries and add expected errors for regular expression Changes to the MongoDBComparatorHelper allow for expected errors to be ignored if they are thrown in a query execution. This feature is mainly necessary for all the illformed regular expressions that the randomized string produces. --- .../mongodb/MongoDBComparatorHelper.java | 28 ++++++++++++ src/sqlancer/mongodb/MongoDBQueryAdapter.java | 6 --- .../mongodb/gen/MongoDBConstantGenerator.java | 3 -- .../gen/MongoDBMatchExpressionGenerator.java | 4 +- .../query/MongoDBCreateIndexQuery.java | 6 +++ .../query/MongoDBCreateTableQuery.java | 6 +++ .../mongodb/query/MongoDBInsertQuery.java | 6 +++ .../mongodb/query/MongoDBSelectQuery.java | 43 +++++++++++++++++++ .../MongoDBQueryPartitioningWhereTester.java | 10 ++--- .../mongodb/visitor/MongoDBToLogVisitor.java | 2 - .../visitor/MongoDBToQueryVisitor.java | 4 +- 11 files changed, 98 insertions(+), 20 deletions(-) diff --git a/src/sqlancer/mongodb/MongoDBComparatorHelper.java b/src/sqlancer/mongodb/MongoDBComparatorHelper.java index 235e5ece9..02996cfda 100644 --- a/src/sqlancer/mongodb/MongoDBComparatorHelper.java +++ b/src/sqlancer/mongodb/MongoDBComparatorHelper.java @@ -6,6 +6,9 @@ import org.bson.Document; +import sqlancer.IgnoreMeException; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; import sqlancer.mongodb.query.MongoDBSelectQuery; public final class MongoDBComparatorHelper { @@ -13,6 +16,31 @@ public final class MongoDBComparatorHelper { private MongoDBComparatorHelper() { } + public static List getResultSetAsDocumentList(MongoDBSelectQuery adapter, MongoDBGlobalState state) + throws Exception { + ExpectedErrors errors = adapter.getExpectedErrors(); + List result; + try { + adapter.executeAndGet(state); + result = adapter.getResultSet(); + return result; + + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw e; + } + + if (e.getMessage() == null) { + throw new AssertionError(adapter.getLogString(), e); + } + + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + throw new AssertionError(adapter.getLogString(), e); + } + } + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, MongoDBSelectQuery originalQuery) { if (resultSet.size() != secondResultSet.size()) { diff --git a/src/sqlancer/mongodb/MongoDBQueryAdapter.java b/src/sqlancer/mongodb/MongoDBQueryAdapter.java index 438163486..e2add3242 100644 --- a/src/sqlancer/mongodb/MongoDBQueryAdapter.java +++ b/src/sqlancer/mongodb/MongoDBQueryAdapter.java @@ -1,6 +1,5 @@ package sqlancer.mongodb; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.Query; public abstract class MongoDBQueryAdapter extends Query { @@ -13,9 +12,4 @@ public String getQueryString() { public String getUnterminatedQueryString() { throw new UnsupportedOperationException(); } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } } diff --git a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java index abcf522d9..c0a8bdb77 100644 --- a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java @@ -24,9 +24,6 @@ public MongoDBConstantGenerator(MongoDBGlobalState globalState) { } public Node generateConstantWithType(MongoDBDataType option) { - if (Randomly.getBooleanWithSmallProbability()) { - return MongoDBConstant.createNullConstant(); - } switch (option) { case DATE_TIME: return MongoDBConstant.createDateTimeConstant(globalState.getRandomly().getInteger()); diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java index 272a52d30..e11de1165 100644 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java @@ -30,7 +30,6 @@ public class MongoDBMatchExpressionGenerator private enum LeafExpression { BINARY_COMPARISON, REGEX - } private enum NonLeafExpression { @@ -85,6 +84,9 @@ protected Node generateExpression(int depth) { public Node generateConstant() { MongoDBDataType type = MongoDBDataType.getRandom(); MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); + if (Randomly.getBooleanWithSmallProbability()) { + return MongoDBConstant.createNullConstant(); + } return generator.generateConstantWithType(type); } diff --git a/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java b/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java index 8d54d9cf7..c873b5924 100644 --- a/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java +++ b/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java @@ -9,6 +9,7 @@ import sqlancer.GlobalState; import sqlancer.Main; +import sqlancer.common.query.ExpectedErrors; import sqlancer.mongodb.MongoDBConnection; import sqlancer.mongodb.MongoDBQueryAdapter; import sqlancer.mongodb.MongoDBSchema.MongoDBTable; @@ -68,4 +69,9 @@ public boolean couldAffectSchema() { return true; } + @Override + public ExpectedErrors getExpectedErrors() { + return new ExpectedErrors(); + } + } diff --git a/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java b/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java index da4381079..7bc174c77 100644 --- a/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java +++ b/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java @@ -12,6 +12,7 @@ import sqlancer.GlobalState; import sqlancer.Main; +import sqlancer.common.query.ExpectedErrors; import sqlancer.mongodb.MongoDBConnection; import sqlancer.mongodb.MongoDBQueryAdapter; @@ -44,6 +45,11 @@ public boolean couldAffectSchema() { return true; } + @Override + public ExpectedErrors getExpectedErrors() { + return new ExpectedErrors(); + } + @Override public String getLogString() { String helper = ""; diff --git a/src/sqlancer/mongodb/query/MongoDBInsertQuery.java b/src/sqlancer/mongodb/query/MongoDBInsertQuery.java index 8276a0895..8a41a1341 100644 --- a/src/sqlancer/mongodb/query/MongoDBInsertQuery.java +++ b/src/sqlancer/mongodb/query/MongoDBInsertQuery.java @@ -8,6 +8,7 @@ import com.mongodb.client.result.InsertOneResult; import sqlancer.GlobalState; +import sqlancer.common.query.ExpectedErrors; import sqlancer.mongodb.MongoDBConnection; import sqlancer.mongodb.MongoDBQueryAdapter; import sqlancer.mongodb.MongoDBSchema.MongoDBTable; @@ -76,4 +77,9 @@ public boolean couldAffectSchema() { .insertOne(documentToBeInserted); return result.wasAcknowledged(); } + + @Override + public ExpectedErrors getExpectedErrors() { + return new ExpectedErrors(); + } } diff --git a/src/sqlancer/mongodb/query/MongoDBSelectQuery.java b/src/sqlancer/mongodb/query/MongoDBSelectQuery.java index 91480ed4b..0bcfd4d60 100644 --- a/src/sqlancer/mongodb/query/MongoDBSelectQuery.java +++ b/src/sqlancer/mongodb/query/MongoDBSelectQuery.java @@ -11,6 +11,7 @@ import com.mongodb.client.MongoCursor; import sqlancer.GlobalState; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLancerResultSet; import sqlancer.mongodb.MongoDBConnection; import sqlancer.mongodb.MongoDBQueryAdapter; @@ -39,6 +40,48 @@ public boolean couldAffectSchema() { throw new UnsupportedOperationException(); } + @Override + public ExpectedErrors getExpectedErrors() { + ExpectedErrors errors = new ExpectedErrors(); + // REGEX + errors.add("Regular expression is invalid: nothing to repeat"); + errors.add("Regular expression is invalid: missing terminating ] for character class"); + errors.add("Regular expression is invalid: unmatched parentheses"); + errors.add("Regular expression is invalid: missing )"); + errors.add("Regular expression is invalid: invalid UTF-8 string"); + errors.add("Regular expression is invalid: \\k is not followed by a braced, angle-bracketed, or quoted name"); + errors.add("Regular expression is invalid: missing opening brace after \\\\o"); + errors.add("Regular expression is invalid: reference to non-existent subpattern"); + errors.add("Regular expression is invalid: \\ at end of pattern"); + errors.add("Regular expression is invalid: PCRE does not support \\L, \\l, \\N{name}, \\U, or \\u"); + errors.add("Regular expression is invalid: (?R or (?[+-]digits must be followed by )"); + errors.add("Regular expression is invalid: unknown property name after \\P or \\p"); + errors.add("Regular expression is invalid: (*VERB) not recognized or malformed"); + errors.add("Regular expression is invalid: a numbered reference must not be zero"); + errors.add("Regular expression is invalid: unrecognized character after (? or (?-"); + errors.add("Regular expression is invalid: \\c at end of pattern"); + errors.add("Regular expression is invalid: malformed \\P or \\p sequence"); + errors.add("Regular expression is invalid: range out of order in character class"); + errors.add("Regular expression is invalid: group name must start with a non-digit"); + errors.add("Regular expression is invalid: \\c must be followed by an ASCII character"); + errors.add("Regular expression is invalid: subpattern name expected"); + errors.add("Regular expression is invalid: POSIX collating elements are not supported"); + errors.add("Regular expression is invalid: closing ) for (?C expected"); + errors.add("Regular expression is invalid: syntax error in subpattern name (missing terminator)"); + errors.add("Regular expression is invalid: \\\\N is not supported in a class"); + errors.add("Regular expression is invalid: non-octal character in \\o{} (closing brace missing?)"); + errors.add("Regular expression is invalid: non-hex character in \\x{} (closing brace missing?)"); + errors.add( + "Regular expression is invalid: \\g is not followed by a braced, angle-bracketed, or quoted name/number or by a plain number"); + errors.add("Regular expression is invalid: digits missing in \\x{} or \\o{}"); + errors.add("Regular expression is invalid: malformed number or name after (?("); + errors.add("Regular expression is invalid: digit expected after (?+"); + errors.add("Regular expression is invalid: assertion expected after (?( or (?(?C)"); + errors.add("Regular expression is invalid: unrecognized character after (?P"); + + return errors; + } + @Override public > SQLancerResultSet executeAndGet(G globalState, String... fills) throws Exception { diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java index 8c9522aaa..e0f4716f6 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java @@ -1,5 +1,7 @@ package sqlancer.mongodb.test; +import static sqlancer.mongodb.MongoDBComparatorHelper.getResultSetAsDocumentList; + import java.util.List; import org.bson.Document; @@ -20,17 +22,15 @@ public void check() throws Exception { select.setFilterClause(null); MongoDBSelectQuery q = new MongoDBSelectQuery(select); q.executeAndGet(state); + List firstResultSet = getResultSetAsDocumentList(q, state); - List firstResultSet = q.getResultSet(); select.setFilterClause(predicate); q = new MongoDBSelectQuery(select); - q.executeAndGet(state); - List secondResultSet = q.getResultSet(); + List secondResultSet = getResultSetAsDocumentList(q, state); select.setFilterClause(negatedPredicate); q = new MongoDBSelectQuery(select); - q.executeAndGet(state); - List thirdResultSet = q.getResultSet(); + List thirdResultSet = getResultSetAsDocumentList(q, state); secondResultSet.addAll(thirdResultSet); MongoDBComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, q); diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java index bef5e5952..5c68fd8bb 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java @@ -60,8 +60,6 @@ public String visit(MongoDBBinaryComparisonNode expr) { public String visit(MongoDBRegexNode expr) { Node left = expr.getLeft(); Node right = expr.getRight(); - assert left instanceof MongoDBColumnTestReference; - assert right instanceof MongoDBConstant.MongoDBStringConstant; return "{\"" + ((MongoDBColumnTestReference) left).getQueryString() + "\": {" + expr.operator().getTextRepresentation() + ": \'" diff --git a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java index fb027add6..be1150d63 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java @@ -97,10 +97,8 @@ public Bson visit(MongoDBRegexNode expr) { Node left = expr.getLeft(); Node right = expr.getRight(); - assert left instanceof MongoDBColumnTestReference; - assert right instanceof MongoDBStringConstant; - String columnName = ((MongoDBColumnTestReference) left).getQueryString(); + return expr.operator().applyOperator(columnName, (MongoDBStringConstant) right); } From bd525f635d7263d79591b63dfb830bcfc613d78d Mon Sep 17 00:00:00 2001 From: Patrick Date: Sun, 31 Jan 2021 15:17:33 +0100 Subject: [PATCH 0376/1284] Add expected errors for computed fields in project stage MongoDB Due to the randomly generated computed field tree that has random types as leaf nodes, there are exceptions that we ignore such as wrong type, value not positive for sqrt for example and similar issues. --- .../mongodb/query/MongoDBSelectQuery.java | 29 +++++++++++++++++++ .../MongoDBQueryPartitioningWhereTester.java | 1 - 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mongodb/query/MongoDBSelectQuery.java b/src/sqlancer/mongodb/query/MongoDBSelectQuery.java index 0bcfd4d60..1288f114c 100644 --- a/src/sqlancer/mongodb/query/MongoDBSelectQuery.java +++ b/src/sqlancer/mongodb/query/MongoDBSelectQuery.java @@ -43,6 +43,35 @@ public boolean couldAffectSchema() { @Override public ExpectedErrors getExpectedErrors() { ExpectedErrors errors = new ExpectedErrors(); + // ARITHMETIC + errors.add("Failed to optimize pipeline :: caused by :: Can't coerce out of range value"); + errors.add("Can't coerce out of range value"); + errors.add("date overflow in $add"); + errors.add("Failed to optimize pipeline :: caused by :: $sqrt only supports numeric types, not"); + errors.add("Failed to optimize pipeline :: caused by :: $sqrt's argument must be greater than or equal to 0"); + errors.add("Failed to optimize pipeline :: caused by :: $pow's base must be numeric, not"); + errors.add("Failed to optimize pipeline :: caused by :: $pow cannot take a base of 0 and a negative exponent"); + errors.add("Failed to optimize pipeline :: caused by :: $add only supports numeric or date types, not"); + errors.add("Failed to optimize pipeline :: caused by :: $exp only supports numeric types, not"); + errors.add("Failed to optimize pipeline :: caused by :: $log's base must be numeric, not"); + errors.add("Failed to optimize pipeline :: caused by :: $log's base must be a positive number not equal to 1"); + errors.add("Failed to optimize pipeline :: caused by :: $multiply only supports numeric types, not"); + errors.add("$log's argument must be numeric, not"); + errors.add("$log's argument must be a positive number, but"); + errors.add("$log's base must be numeric, not"); + errors.add("$log's base must be a positive number not equal to 1"); + errors.add("$divide only supports numeric types, not"); + errors.add("can't $divide by zero"); + errors.add("$pow's exponent must be numeric, not"); + errors.add("$pow's base must be numeric, not"); + errors.add("$pow cannot take a base of 0 and a negative exponent"); + errors.add("$add only supports numeric or date types, not"); + errors.add("only one date allowed in an $add expression"); + errors.add("$multiply only supports numeric types, not"); + errors.add("$exp only supports numeric types, not"); + errors.add("$sqrt's argument must be greater than or equal to 0"); + errors.add("$sqrt only supports numeric types, not"); + // REGEX errors.add("Regular expression is invalid: nothing to repeat"); errors.add("Regular expression is invalid: missing terminating ] for character class"); diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java index e0f4716f6..a9695d1d2 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java @@ -21,7 +21,6 @@ public void check() throws Exception { select.setFilterClause(null); MongoDBSelectQuery q = new MongoDBSelectQuery(select); - q.executeAndGet(state); List firstResultSet = getResultSetAsDocumentList(q, state); select.setFilterClause(predicate); From 6aa53858cb8d89b0a7de701335a28d9d2d51a76c Mon Sep 17 00:00:00 2001 From: Patrick Date: Sun, 31 Jan 2021 16:15:14 +0100 Subject: [PATCH 0377/1284] Add logging of computed fields in MongoDB --- .../mongodb/visitor/MongoDBToLogVisitor.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java index 5c68fd8bb..76b1241a4 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; +import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.Node; import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; @@ -11,6 +12,7 @@ import sqlancer.mongodb.ast.MongoDBRegexNode; import sqlancer.mongodb.ast.MongoDBSelect; import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; +import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator.ComputedFunction; import sqlancer.mongodb.test.MongoDBColumnTestReference; public class MongoDBToLogVisitor extends MongoDBVisitor { @@ -35,6 +37,49 @@ public String visitLog(Node expr) { } } + public String visitComputed(Node expr) { + if (expr instanceof NewFunctionNode) { + return visitComputed((NewFunctionNode) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + public String visitComputed(NewFunctionNode expr) { + List arguments = new ArrayList<>(); + for (int i = 0; i < expr.getArgs().size(); i++) { + if (expr.getArgs().get(i) instanceof MongoDBConstant) { + arguments.add(((MongoDBConstant) expr.getArgs().get(i)).getLogValue()); + continue; + } + if (expr.getArgs().get(i) instanceof MongoDBColumnTestReference) { + arguments.add("\"$" + ((MongoDBColumnTestReference) expr.getArgs().get(i)).getQueryString() + "\""); + continue; + } + if (expr.getArgs().get(i) instanceof NewFunctionNode) { + arguments.add(visitComputed((NewFunctionNode) expr.getArgs().get(i))); + } else { + throw new AssertionError(); + } + } + if (!(expr.getFunc() instanceof ComputedFunction)) { + throw new AssertionError(expr.getClass()); + } + + StringBuilder sb = new StringBuilder(); + sb.append("{"); + sb.append(((ComputedFunction) expr.getFunc()).getOperator()); + sb.append(": ["); + String helper = ""; + for (String arg : arguments) { + sb.append(helper); + helper = ", "; + sb.append(arg); + } + sb.append("]}"); + return sb.toString(); + } + public String visit(MongoDBUnaryLogicalOperatorNode expr) { String inner = visitLog(expr.getExpr()); return expr.operator().getTextRepresentation() + inner + "]}"; @@ -108,6 +153,17 @@ private void setProjects(MongoDBSelect select) { helper = ","; sb.append("\"").append(reference.getQueryString()).append("\"").append(": 1"); } + sb.append("\n"); + if (select.hasComputed()) { + String name = "computed"; + int number = 0; + for (Node expressionNode : select.getComputedClause()) { + sb.append(helper); + helper = ",\n"; + sb.append("\"" + name + number + "\": " + visitComputed(expressionNode)); + number++; + } + } sb.append("}"); projects = sb.toString(); } From 6020a07d7491fdbc0cef16a4de08ca1c35bbbee0 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sun, 31 Jan 2021 17:44:34 +0100 Subject: [PATCH 0378/1284] Add randomized regex options for the regular expression node in MongoDB The options are flags that can be set in a string such as "im". The expression generator now generates a random valid options for the regular expression operator in MongoDB. --- src/sqlancer/mongodb/ast/MongoDBRegexNode.java | 9 ++++++++- .../gen/MongoDBMatchExpressionGenerator.java | 15 +++++++++++---- .../mongodb/visitor/MongoDBToLogVisitor.java | 3 ++- .../mongodb/visitor/MongoDBToQueryVisitor.java | 2 +- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/mongodb/ast/MongoDBRegexNode.java b/src/sqlancer/mongodb/ast/MongoDBRegexNode.java index 154ebeffa..76c608586 100644 --- a/src/sqlancer/mongodb/ast/MongoDBRegexNode.java +++ b/src/sqlancer/mongodb/ast/MongoDBRegexNode.java @@ -7,8 +7,15 @@ import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBRegexOperator; public class MongoDBRegexNode extends NewBinaryOperatorNode { - public MongoDBRegexNode(Node left, Node right) { + private final String options; + + public MongoDBRegexNode(Node left, Node right, String options) { super(left, right, REGEX); + this.options = options; + } + + public String getOptions() { + return options; } public MongoDBRegexOperator operator() { diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java index e11de1165..e02064f5c 100644 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java @@ -53,7 +53,8 @@ public Node generateLeafNode() { return new MongoDBBinaryComparisonNode(generateColumn(), generateConstant(), operator); case REGEX: return new MongoDBRegexNode(generateColumn(), - new MongoDBConstantGenerator(globalState).generateConstantWithType(MongoDBDataType.STRING)); + new MongoDBConstantGenerator(globalState).generateConstantWithType(MongoDBDataType.STRING), + getRandomizedRegexOptions()); default: throw new AssertionError(); } @@ -90,6 +91,11 @@ public Node generateConstant() { return generator.generateConstantWithType(type); } + private String getRandomizedRegexOptions() { + List s = Randomly.subset("i", "m", "x", "s"); + return s.stream().reduce("", (current, newVal) -> current + newVal); + } + @Override protected Node generateColumn() { return Randomly.fromList(columns); @@ -249,8 +255,8 @@ public static MongoDBBinaryComparisonOperator getRandom() { public enum MongoDBRegexOperator implements Operator { REGEX { @Override - public Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex) { - return Filters.regex(columnName, regex.getStringValue(), ""); + public Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex, String options) { + return Filters.regex(columnName, regex.getStringValue(), options); } @Override @@ -259,6 +265,7 @@ public String getTextRepresentation() { } }; - public abstract Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex); + public abstract Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex, + String options); } } diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java index 76b1241a4..37465163b 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java @@ -108,7 +108,8 @@ public String visit(MongoDBRegexNode expr) { return "{\"" + ((MongoDBColumnTestReference) left).getQueryString() + "\": {" + expr.operator().getTextRepresentation() + ": \'" - + ((MongoDBConstant.MongoDBStringConstant) right).getStringValue() + "\', $options: \'\'}}"; + + ((MongoDBConstant.MongoDBStringConstant) right).getStringValue() + "\', $options: \'" + + expr.getOptions() + "\'}}"; } @Override diff --git a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java index be1150d63..dcb8718ce 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java @@ -99,7 +99,7 @@ public Bson visit(MongoDBRegexNode expr) { String columnName = ((MongoDBColumnTestReference) left).getQueryString(); - return expr.operator().applyOperator(columnName, (MongoDBStringConstant) right); + return expr.operator().applyOperator(columnName, (MongoDBStringConstant) right, expr.getOptions()); } public Bson visit(MongoDBBinaryComparisonNode expr) { From 5897c8673e77454538833aaa8b3bf86e533c118d Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 1 Feb 2021 10:25:04 +0100 Subject: [PATCH 0379/1284] Add a variation where the count is compared to the number of returned documents in MongoDB This new variation introduces a variant where we execute the pipeline query once with a count as a last pipeline stage and compare the output to the result set without the count stage. --- .../mongodb/MongoDBComparatorHelper.java | 24 +++++++++++++++++++ src/sqlancer/mongodb/MongoDBOptions.java | 3 +++ src/sqlancer/mongodb/ast/MongoDBSelect.java | 9 +++++++ .../MongoDBQueryPartitioningWhereTester.java | 10 ++++++++ .../mongodb/visitor/MongoDBToLogVisitor.java | 9 ++++++- .../visitor/MongoDBToQueryVisitor.java | 13 ++++++++++ 6 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mongodb/MongoDBComparatorHelper.java b/src/sqlancer/mongodb/MongoDBComparatorHelper.java index 02996cfda..c14f8cdbf 100644 --- a/src/sqlancer/mongodb/MongoDBComparatorHelper.java +++ b/src/sqlancer/mongodb/MongoDBComparatorHelper.java @@ -41,6 +41,30 @@ public static List getResultSetAsDocumentList(MongoDBSelectQuery adapt } } + public static void assumeCountIsEqual(List resultSet, List secondResultSet, + MongoDBSelectQuery originalQuery) { + int originalSize = resultSet.size(); + if (secondResultSet.isEmpty()) { + if (originalSize == 0) { + return; + } else { + String assertMessage = String.format("The Count of the result set mismatches!\n %s", + originalQuery.getLogString()); + throw new AssertionError(assertMessage); + } + } + if (secondResultSet.size() != 1) { + throw new AssertionError( + String.format("Count query result bigger than one \n %s", originalQuery.getLogString())); + } + int withCount = (int) secondResultSet.get(0).get("count"); + if (originalSize != withCount) { + String assertMessage = String.format("The Count of the result set mismatches!\n %s", + originalQuery.getLogString()); + throw new AssertionError(assertMessage); + } + } + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, MongoDBSelectQuery originalQuery) { if (resultSet.size() != secondResultSet.size()) { diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 5c16bfd4d..5023a6fb8 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -34,6 +34,9 @@ public class MongoDBOptions implements DBMSSpecificOptions oracles = Arrays.asList(QUERY_PARTITIONING); diff --git a/src/sqlancer/mongodb/ast/MongoDBSelect.java b/src/sqlancer/mongodb/ast/MongoDBSelect.java index de65fb01d..0fe91ba4a 100644 --- a/src/sqlancer/mongodb/ast/MongoDBSelect.java +++ b/src/sqlancer/mongodb/ast/MongoDBSelect.java @@ -15,6 +15,7 @@ public class MongoDBSelect implements Node { Node filterClause; boolean hasComputed; List> computedClauses; + private boolean withCountClause; public MongoDBSelect(String mainTableName, MongoDBColumnTestReference joinColumn) { this.mainTableName = mainTableName; @@ -92,4 +93,12 @@ public List> getComputedClause() { public boolean hasComputed() { return hasComputed; } + + public boolean getWithCountClause() { + return withCountClause; + } + + public void setWithCountClause(boolean withCountClause) { + this.withCountClause = withCountClause; + } } diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java index a9695d1d2..5a7507672 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java @@ -19,6 +19,8 @@ public MongoDBQueryPartitioningWhereTester(MongoDBGlobalState state) { public void check() throws Exception { super.check(); + select.setWithCountClause(false); + select.setFilterClause(null); MongoDBSelectQuery q = new MongoDBSelectQuery(select); List firstResultSet = getResultSetAsDocumentList(q, state); @@ -31,6 +33,14 @@ public void check() throws Exception { q = new MongoDBSelectQuery(select); List thirdResultSet = getResultSetAsDocumentList(q, state); + if (state.getDmbsSpecificOptions().testWithCount) { + select.setWithCountClause(true); + select.setFilterClause(predicate); + q = new MongoDBSelectQuery(select); + List forthResultSet = getResultSetAsDocumentList(q, state); + MongoDBComparatorHelper.assumeCountIsEqual(secondResultSet, forthResultSet, q); + } + secondResultSet.addAll(thirdResultSet); MongoDBComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, q); diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java index 37465163b..1fbae32a6 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java @@ -22,6 +22,7 @@ public class MongoDBToLogVisitor extends MongoDBVisitor { private String filter; private String projects; private boolean hasFilter; + private boolean withCount; public String visitLog(Node expr) { if (expr instanceof MongoDBUnaryLogicalOperatorNode) { @@ -126,6 +127,7 @@ public void visit(MongoDBSelect select) { setFilter(select); } setProjects(select); + withCount = select.getWithCountClause(); } private void setFilter(MongoDBSelect select) { @@ -182,7 +184,12 @@ public String getStringLog() { } sb.append("{ $project : "); sb.append(projects); - sb.append("}])\n"); + sb.append("}"); + if (withCount) { + sb.append(",\n"); + sb.append(" {$count: \"count\"}\n"); + } + sb.append("])\n"); return sb.toString(); } } diff --git a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java index dcb8718ce..8efbf97e4 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java @@ -33,7 +33,9 @@ public class MongoDBToQueryVisitor extends MongoDBVisitor { private List lookup; private Bson filter; private Bson projection; + private Bson count; private boolean hasFilter; + private boolean hasCountClause; public Bson visitBson(Node expr) { if (expr instanceof MongoDBUnaryLogicalOperatorNode) { @@ -125,6 +127,14 @@ public void visit(MongoDBSelect select) { setFilter(select); } setProjection(select); + hasCountClause = select.getWithCountClause(); + if (hasCountClause) { + setCount(); + } + } + + private void setCount() { + count = Aggregates.count("count"); } private void setFilter(MongoDBSelect select) { @@ -166,6 +176,9 @@ public List getPipeline() { result.add(filter); } result.add(projection); + if (hasCountClause) { + result.add(count); + } return result; } } From 85d1415a62adbf86ad2f7af8ff5ee913c42f3602 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 2 Feb 2021 23:30:15 +0100 Subject: [PATCH 0380/1284] Rework negation of MongoDB expression to better cover the api Until now not has been simulated by nor(id exists, bool_expression) and it has led to believe that the underlying structure works fine. Now after the rework the not gets evaluated and every logical operation inverted by the new NegateVisitor. At the lowest stage type problems remain. --- .../mongodb/MongoDBComparatorHelper.java | 2 +- .../gen/MongoDBMatchExpressionGenerator.java | 14 +- .../mongodb/visitor/MongoDBNegateVisitor.java | 150 ++++++++++++++++++ .../mongodb/visitor/MongoDBToLogVisitor.java | 2 +- .../mongodb/visitor/MongoDBVisitor.java | 6 + 5 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java diff --git a/src/sqlancer/mongodb/MongoDBComparatorHelper.java b/src/sqlancer/mongodb/MongoDBComparatorHelper.java index c14f8cdbf..8554a6662 100644 --- a/src/sqlancer/mongodb/MongoDBComparatorHelper.java +++ b/src/sqlancer/mongodb/MongoDBComparatorHelper.java @@ -69,7 +69,7 @@ public static void assumeResultSetsAreEqual(List resultSet, List, MongoDBColumnTestReference> { @@ -101,9 +102,17 @@ protected Node generateColumn() { return Randomly.fromList(columns); } + @Override + public Node generatePredicate() { + Node result = super.generatePredicate(); + return MongoDBNegateVisitor.cleanNegations(result); + } + @Override public Node negatePredicate(Node predicate) { - return new MongoDBUnaryLogicalOperatorNode(predicate, MongoDBUnaryLogicalOperator.NOT); + Node result = new MongoDBUnaryLogicalOperatorNode(predicate, + MongoDBUnaryLogicalOperator.NOT); + return MongoDBNegateVisitor.cleanNegations(result); } @Override @@ -115,12 +124,13 @@ public enum MongoDBUnaryLogicalOperator implements Operator { NOT { @Override public Bson applyOperator(Bson inner) { + // return Filters.not(inner); TODO: Patrick return Filters.nor(inner, Filters.exists("_id", false)); } @Override public String getTextRepresentation() { - return "{$nor: [{ _id: {$exists: false}}, "; + return "$not"; } }; diff --git a/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java new file mode 100644 index 000000000..23192c2b0 --- /dev/null +++ b/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java @@ -0,0 +1,150 @@ +package sqlancer.mongodb.visitor; + +import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator.AND; +import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator.NOR; +import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator.OR; +import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBUnaryLogicalOperator.NOT; + +import sqlancer.common.ast.newast.Node; +import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; +import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; +import sqlancer.mongodb.ast.MongoDBConstant; +import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBRegexNode; +import sqlancer.mongodb.ast.MongoDBSelect; +import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; + +public class MongoDBNegateVisitor extends MongoDBVisitor { + + private boolean negate; + Node negatedExpression; + + public MongoDBNegateVisitor(boolean negate) { + this.negate = negate; + } + + @Override + public void visit(Node expr) { + if (expr instanceof MongoDBConstant) { + visit((MongoDBConstant) expr); + } else if (expr instanceof MongoDBSelect) { + visit((MongoDBSelect) expr); + } else if (expr instanceof MongoDBBinaryComparisonNode) { + visit((MongoDBBinaryComparisonNode) expr); + } else if (expr instanceof MongoDBUnaryLogicalOperatorNode) { + visit((MongoDBUnaryLogicalOperatorNode) expr); + } else if (expr instanceof MongoDBRegexNode) { + visit((MongoDBRegexNode) expr); + } else if (expr instanceof MongoDBBinaryLogicalNode) { + visit((MongoDBBinaryLogicalNode) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + public void visit(MongoDBBinaryComparisonNode expr) { + + if (negate) { + negatedExpression = new MongoDBUnaryLogicalOperatorNode(expr, NOT); + // TODO: Patrick + /* + * switch (expr.operator()) { case EQUALS: negatedExpression = new + * MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.NOT_EQUALS); break; case NOT_EQUALS: + * negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.EQUALS); break; + * + * case LESS: negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER_EQUAL); break; case LESS_EQUAL: + * negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER); break; case GREATER: + * negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS_EQUAL); break; case GREATER_EQUAL: + * negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS); break; default: throw new + * UnsupportedOperationException(); } + */ + } else { + negatedExpression = expr; + } + } + + public void visit(MongoDBRegexNode expr) { + if (negate) { + negatedExpression = new MongoDBUnaryLogicalOperatorNode(expr, NOT); + } else { + negatedExpression = expr; + } + } + + public void visit(MongoDBUnaryLogicalOperatorNode expr) { + if (!(expr.operator().equals(NOT))) { + throw new UnsupportedOperationException(); + } + negate = !negate; + visit(expr.getExpr()); + } + + public void visit(MongoDBBinaryLogicalNode expr) { + boolean saveNegate = negate; + Node left; + Node right; + switch (expr.operator()) { + case OR: + negate = false; + visit(expr.getLeft()); + left = negatedExpression; + negate = false; + visit(expr.getRight()); + right = negatedExpression; + if (saveNegate) { + negatedExpression = new MongoDBBinaryLogicalNode(left, right, NOR); + } else { + negatedExpression = new MongoDBBinaryLogicalNode(left, right, OR); + } + break; + case AND: + negate = saveNegate; + visit(expr.getLeft()); + left = negatedExpression; + negate = saveNegate; + visit(expr.getRight()); + right = negatedExpression; + if (saveNegate) { + negatedExpression = new MongoDBBinaryLogicalNode(left, right, OR); + } else { + negatedExpression = new MongoDBBinaryLogicalNode(left, right, AND); + } + break; + case NOR: + negate = false; + visit(expr.getLeft()); + left = negatedExpression; + negate = false; + visit(expr.getRight()); + right = negatedExpression; + if (saveNegate) { + negatedExpression = new MongoDBBinaryLogicalNode(left, right, OR); + } else { + negatedExpression = new MongoDBBinaryLogicalNode(left, right, NOR); + } + break; + default: + throw new UnsupportedOperationException(expr.getOperatorRepresentation()); + } + } + + @Override + public void visit(MongoDBConstant c) { + negatedExpression = c; + } + + @Override + public void visit(MongoDBSelect s) { + throw new UnsupportedOperationException(); + } + + public Node getNegatedExpression() { + return negatedExpression; + } +} diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java index 1fbae32a6..4c55e17a6 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java @@ -83,7 +83,7 @@ public String visitComputed(NewFunctionNode expr) { public String visit(MongoDBUnaryLogicalOperatorNode expr) { String inner = visitLog(expr.getExpr()); - return expr.operator().getTextRepresentation() + inner + "]}"; + return "{ " + expr.operator().getTextRepresentation() + ": [" + inner + "]}"; } public String visit(MongoDBBinaryLogicalNode expr) { diff --git a/src/sqlancer/mongodb/visitor/MongoDBVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBVisitor.java index 67b6ca741..e02a50f02 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBVisitor.java @@ -36,4 +36,10 @@ public static String asStringLog(Node expr) { visitor.visit(expr); return visitor.getStringLog(); } + + public static Node cleanNegations(Node expr) { + MongoDBNegateVisitor visitor = new MongoDBNegateVisitor(false); + visitor.visit(expr); + return visitor.getNegatedExpression(); + } } From 498b9de9a8b1ecb1dbcbb4c72916b964b0f7fd8f Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 3 Feb 2021 00:03:46 +0100 Subject: [PATCH 0381/1284] Fix MongoDB unsuccessful and successful query counts --- src/sqlancer/mongodb/MongoDBComparatorHelper.java | 6 +++--- src/sqlancer/mongodb/query/MongoDBInsertQuery.java | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/mongodb/MongoDBComparatorHelper.java b/src/sqlancer/mongodb/MongoDBComparatorHelper.java index 8554a6662..b03e24422 100644 --- a/src/sqlancer/mongodb/MongoDBComparatorHelper.java +++ b/src/sqlancer/mongodb/MongoDBComparatorHelper.java @@ -7,6 +7,7 @@ import org.bson.Document; import sqlancer.IgnoreMeException; +import sqlancer.Main; import sqlancer.common.query.ExpectedErrors; import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; import sqlancer.mongodb.query.MongoDBSelectQuery; @@ -22,18 +23,17 @@ public static List getResultSetAsDocumentList(MongoDBSelectQuery adapt List result; try { adapter.executeAndGet(state); + Main.nrSuccessfulActions.addAndGet(1); result = adapter.getResultSet(); return result; - } catch (Exception e) { + Main.nrUnsuccessfulActions.addAndGet(1); if (e instanceof IgnoreMeException) { throw e; } - if (e.getMessage() == null) { throw new AssertionError(adapter.getLogString(), e); } - if (errors.errorIsExpected(e.getMessage())) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/mongodb/query/MongoDBInsertQuery.java b/src/sqlancer/mongodb/query/MongoDBInsertQuery.java index 8a41a1341..127dc82d0 100644 --- a/src/sqlancer/mongodb/query/MongoDBInsertQuery.java +++ b/src/sqlancer/mongodb/query/MongoDBInsertQuery.java @@ -8,6 +8,7 @@ import com.mongodb.client.result.InsertOneResult; import sqlancer.GlobalState; +import sqlancer.Main; import sqlancer.common.query.ExpectedErrors; import sqlancer.mongodb.MongoDBConnection; import sqlancer.mongodb.MongoDBQueryAdapter; @@ -73,6 +74,7 @@ public boolean couldAffectSchema() { @Override public > boolean execute(G globalState, String... fills) throws Exception { + Main.nrSuccessfulActions.addAndGet(1); InsertOneResult result = globalState.getConnection().getDatabase().getCollection(table.getName()) .insertOne(documentToBeInserted); return result.wasAcknowledged(); From 78a67d152f101be16bc74de455cbf0e39d9680cf Mon Sep 17 00:00:00 2001 From: Patrick Date: Sat, 6 Feb 2021 17:14:40 +0100 Subject: [PATCH 0382/1284] Add a new way to negate a query that replaces the old placeholder This change reveals some issues with how we form queries for mongodb. The core of it seems to be that neither greater equal nor smaller include null values and it is rather hard to define a query that finds the ones that are null. The query that projects includes all. --- src/sqlancer/mongodb/MongoDBSchema.java | 14 +++++- .../mongodb/gen/MongoDBConstantGenerator.java | 11 +++-- .../gen/MongoDBMatchExpressionGenerator.java | 16 +++++-- .../test/MongoDBQueryPartitioningBase.java | 23 ++++----- .../mongodb/visitor/MongoDBNegateVisitor.java | 47 ++++++++++++------- 5 files changed, 73 insertions(+), 38 deletions(-) diff --git a/src/sqlancer/mongodb/MongoDBSchema.java b/src/sqlancer/mongodb/MongoDBSchema.java index 5bf3cf2b5..535e0d5f8 100644 --- a/src/sqlancer/mongodb/MongoDBSchema.java +++ b/src/sqlancer/mongodb/MongoDBSchema.java @@ -56,7 +56,19 @@ public BsonType getBsonType() { }; public static MongoDBDataType getRandom() { - return Randomly.fromOptions(values()); + // TODO: If String is enabled, there are type issues. Find a way to have a cast or operation on top of the + // query + // TODO: to solve this issue. + MongoDBDataType[] valuesWithoutString = new MongoDBDataType[values().length - 1]; + int i = 0; + for (MongoDBDataType type : values()) { + if (type.equals(STRING)) { + continue; + } + valuesWithoutString[i] = type; + i++; + } + return Randomly.fromOptions(valuesWithoutString); } } diff --git a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java index c0a8bdb77..bda5c87e6 100644 --- a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java @@ -12,7 +12,6 @@ import sqlancer.mongodb.ast.MongoDBConstant.MongoDBDoubleConstant; import sqlancer.mongodb.ast.MongoDBConstant.MongoDBIntegerConstant; import sqlancer.mongodb.ast.MongoDBConstant.MongoDBNullConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBStringConstant; import sqlancer.mongodb.ast.MongoDBConstant.MongoDBTimestampConstant; import sqlancer.mongodb.ast.MongoDBExpression; @@ -68,10 +67,12 @@ public void addRandomConstantWithType(Document document, String key, MongoDBData constant = new MongoDBDoubleConstant(globalState.getRandomly().getDouble()); constant.setValueInDocument(document, key); return; - case STRING: - constant = new MongoDBStringConstant(globalState.getRandomly().getString()); - constant.setValueInDocument(document, key); - return; + // TODO: If String is enabled, there are type issues. Find a way to have a cast or operation on top of the query + // TODO: to solve this issue. + // case STRING: + // constant = new MongoDBStringConstant(globalState.getRandomly().getString()); + // constant.setValueInDocument(document, key); + // return; case INTEGER: constant = new MongoDBIntegerConstant((int) globalState.getRandomly().getInteger()); constant.setValueInDocument(document, key); diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java index c298295c7..6414ea185 100644 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java @@ -51,7 +51,10 @@ public Node generateLeafNode() { switch (expr) { case BINARY_COMPARISON: MongoDBBinaryComparisonOperator operator = MongoDBBinaryComparisonOperator.getRandom(); - return new MongoDBBinaryComparisonNode(generateColumn(), generateConstant(), operator); + MongoDBColumnTestReference reference = (MongoDBColumnTestReference) generateColumn(); + + return new MongoDBBinaryComparisonNode(reference, + generateConstant(reference.getColumnReference().getType()), operator); case REGEX: return new MongoDBRegexNode(generateColumn(), new MongoDBConstantGenerator(globalState).generateConstantWithType(MongoDBDataType.STRING), @@ -92,6 +95,14 @@ public Node generateConstant() { return generator.generateConstantWithType(type); } + public Node generateConstant(MongoDBDataType type) { + MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); + // if (Randomly.getBooleanWithSmallProbability()) { + // return MongoDBConstant.createNullConstant(); + // } + return generator.generateConstantWithType(type); + } + private String getRandomizedRegexOptions() { List s = Randomly.subset("i", "m", "x", "s"); return s.stream().reduce("", (current, newVal) -> current + newVal); @@ -124,8 +135,7 @@ public enum MongoDBUnaryLogicalOperator implements Operator { NOT { @Override public Bson applyOperator(Bson inner) { - // return Filters.not(inner); TODO: Patrick - return Filters.nor(inner, Filters.exists("_id", false)); + return Filters.not(inner); } @Override diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java index c33bcf9e6..6421482a0 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -42,11 +42,12 @@ public void check() throws Exception { initializeTernaryPredicateVariants(); select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); select.setProjectionList(targetColumns); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setLookupList(targetColumns); - } else { - select.setLookupList(Randomly.nonEmptySubset(targetColumns)); - } + // TODO: Find a new way to split the query such that it can handle null fields to activate lookups + // if (Randomly.getBooleanWithRatherLowProbability()) { + select.setLookupList(targetColumns); + /* + * } else { select.setLookupList(Randomly.nonEmptySubset(targetColumns)); } + */ if (state.getDmbsSpecificOptions().testComputedValues) { generateComputedColumns(); } @@ -69,12 +70,12 @@ private void generateTargetColumns() { targetColumns.add(new MongoDBColumnTestReference(c, true)); } List joinsOtherTables = new ArrayList<>(); - for (int i = 1; i < targetTables.getTables().size(); i++) { - MongoDBTable procTable = targetTables.getTables().get(i); - for (MongoDBColumn c : procTable.getColumns()) { - joinsOtherTables.add(new MongoDBColumnTestReference(c, false)); - } - } + // TODO: Find a new way to split the query such that it can handle null fields to activate lookups + /* + * for (int i = 1; i < targetTables.getTables().size(); i++) { MongoDBTable procTable = + * targetTables.getTables().get(i); for (MongoDBColumn c : procTable.getColumns()) { joinsOtherTables.add(new + * MongoDBColumnTestReference(c, false)); } } + */ if (!joinsOtherTables.isEmpty()) { int randNumber = state.getRandomly().getInteger(1, Math.min(joinsOtherTables.size(), 4)); List subsetJoinsOtherTables = Randomly.nonEmptySubset(joinsOtherTables, diff --git a/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java index 23192c2b0..39b607f67 100644 --- a/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java +++ b/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java @@ -13,6 +13,7 @@ import sqlancer.mongodb.ast.MongoDBRegexNode; import sqlancer.mongodb.ast.MongoDBSelect; import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; +import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; public class MongoDBNegateVisitor extends MongoDBVisitor { @@ -46,24 +47,34 @@ public void visit(MongoDBBinaryComparisonNode expr) { if (negate) { negatedExpression = new MongoDBUnaryLogicalOperatorNode(expr, NOT); - // TODO: Patrick - /* - * switch (expr.operator()) { case EQUALS: negatedExpression = new - * MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.NOT_EQUALS); break; case NOT_EQUALS: - * negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.EQUALS); break; - * - * case LESS: negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER_EQUAL); break; case LESS_EQUAL: - * negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER); break; case GREATER: - * negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS_EQUAL); break; case GREATER_EQUAL: - * negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - * MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS); break; default: throw new - * UnsupportedOperationException(); } - */ + switch (expr.operator()) { + case EQUALS: + negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.NOT_EQUALS); + break; + case NOT_EQUALS: + negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.EQUALS); + break; + case LESS: + negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER_EQUAL); + break; + case LESS_EQUAL: + negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER); + break; + case GREATER: + negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS_EQUAL); + break; + case GREATER_EQUAL: + negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), + MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS); + break; + default: + throw new UnsupportedOperationException(); + } } else { negatedExpression = expr; } From dfae8afeb11f83d3c75b6b184074693115e530f4 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sun, 28 Feb 2021 15:12:08 +0100 Subject: [PATCH 0383/1284] Introduce ArangoDB to SQLancer and support creating collection and inserting data For four data types of ArangoDB integer, double, string and boolean, this commit supports creating collection, keeping track of the schema, creating tables, collections and inserting randomized data. --- pom.xml | 5 + src/sqlancer/Main.java | 2 + src/sqlancer/arangodb/ArangoDBConnection.java | 31 +++++ .../arangodb/ArangoDBLoggableFactory.java | 40 ++++++ src/sqlancer/arangodb/ArangoDBOptions.java | 41 ++++++ src/sqlancer/arangodb/ArangoDBProvider.java | 128 ++++++++++++++++++ .../arangodb/ArangoDBQueryAdapter.java | 15 ++ .../arangodb/ArangoDBQueryProvider.java | 6 + src/sqlancer/arangodb/ArangoDBSchema.java | 66 +++++++++ .../arangodb/ast/ArangoDBConstant.java | 126 +++++++++++++++++ .../arangodb/ast/ArangoDBExpression.java | 4 + .../arangodb/gen/ArangoDBInsertGenerator.java | 39 ++++++ .../arangodb/gen/ArangoDBTableGenerator.java | 44 ++++++ .../query/ArangoDBConstantGenerator.java | 46 +++++++ .../query/ArangoDBCreateTableQuery.java | 45 ++++++ .../arangodb/query/ArangoDBInsertQuery.java | 50 +++++++ .../test/ArangoDBQueryPartitioningBase.java | 27 ++++ .../ArangoDBQueryPartitioningWhereTester.java | 9 ++ 18 files changed, 724 insertions(+) create mode 100644 src/sqlancer/arangodb/ArangoDBConnection.java create mode 100644 src/sqlancer/arangodb/ArangoDBLoggableFactory.java create mode 100644 src/sqlancer/arangodb/ArangoDBOptions.java create mode 100644 src/sqlancer/arangodb/ArangoDBProvider.java create mode 100644 src/sqlancer/arangodb/ArangoDBQueryAdapter.java create mode 100644 src/sqlancer/arangodb/ArangoDBQueryProvider.java create mode 100644 src/sqlancer/arangodb/ArangoDBSchema.java create mode 100644 src/sqlancer/arangodb/ast/ArangoDBConstant.java create mode 100644 src/sqlancer/arangodb/ast/ArangoDBExpression.java create mode 100644 src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java create mode 100644 src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java create mode 100644 src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java create mode 100644 src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java create mode 100644 src/sqlancer/arangodb/query/ArangoDBInsertQuery.java create mode 100644 src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java create mode 100644 src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java diff --git a/pom.xml b/pom.xml index c0f0bbd9b..2b8783fb8 100644 --- a/pom.xml +++ b/pom.xml @@ -271,6 +271,11 @@ mongodb-driver-sync 4.1.1 + + com.arangodb + arangodb-java-driver + 6.9.0 + diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index d606ee0f5..bfc0cabd8 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -21,6 +21,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.JCommander.Builder; +import sqlancer.arangodb.ArangoDBProvider; import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.cockroachdb.CockroachDBProvider; @@ -556,6 +557,7 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new DuckDBProvider()); providers.add(new H2Provider()); providers.add(new MongoDBProvider()); + providers.add(new ArangoDBProvider()); return providers; } diff --git a/src/sqlancer/arangodb/ArangoDBConnection.java b/src/sqlancer/arangodb/ArangoDBConnection.java new file mode 100644 index 000000000..b3e5b85d3 --- /dev/null +++ b/src/sqlancer/arangodb/ArangoDBConnection.java @@ -0,0 +1,31 @@ +package sqlancer.arangodb; + +import com.arangodb.ArangoDB; +import com.arangodb.ArangoDatabase; + +import sqlancer.SQLancerDBConnection; + +public class ArangoDBConnection implements SQLancerDBConnection { + + private final ArangoDB client; + private final ArangoDatabase database; + + public ArangoDBConnection(ArangoDB client, ArangoDatabase database) { + this.client = client; + this.database = database; + } + + @Override + public String getDatabaseVersion() throws Exception { + return client.getVersion().getVersion(); + } + + @Override + public void close() throws Exception { + client.shutdown(); + } + + public ArangoDatabase getDatabase() { + return database; + } +} diff --git a/src/sqlancer/arangodb/ArangoDBLoggableFactory.java b/src/sqlancer/arangodb/ArangoDBLoggableFactory.java new file mode 100644 index 000000000..927d9f320 --- /dev/null +++ b/src/sqlancer/arangodb/ArangoDBLoggableFactory.java @@ -0,0 +1,40 @@ +package sqlancer.arangodb; + +import java.util.Arrays; + +import sqlancer.common.log.Loggable; +import sqlancer.common.log.LoggableFactory; +import sqlancer.common.log.LoggedString; +import sqlancer.common.query.Query; + +public class ArangoDBLoggableFactory extends LoggableFactory { + @Override + protected Loggable createLoggable(String input, String suffix) { + return new LoggedString(input + suffix); + } + + @Override + public Query getQueryForStateToReproduce(String queryString) { + throw new UnsupportedOperationException(); + } + + @Override + public Query commentOutQuery(Query query) { + throw new UnsupportedOperationException(); + } + + @Override + protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { + StringBuilder sb = new StringBuilder(); + sb.append("// Time: ").append(time).append("\n"); + sb.append("// Database: ").append(databaseName).append("\n"); + sb.append("// Database version: ").append(databaseVersion).append("\n"); + sb.append("// seed value: ").append(seedValue).append("\n"); + return new LoggedString(sb.toString()); + } + + @Override + public Loggable convertStacktraceToLoggable(Throwable throwable) { + return new LoggedString(Arrays.toString(throwable.getStackTrace()) + "\n" + throwable.getMessage()); + } +} diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java new file mode 100644 index 000000000..04e1038f9 --- /dev/null +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -0,0 +1,41 @@ +package sqlancer.arangodb; + +import static sqlancer.arangodb.ArangoDBOptions.ArangoDBOracleFactory.QUERY_PARTITIONING; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.arangodb.test.ArangoDBQueryPartitioningWhereTester; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; + +public class ArangoDBOptions implements DBMSSpecificOptions { + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(QUERY_PARTITIONING); + + @Parameter(names = "--test-random-type-inserts", description = "Insert random types instead of schema types.") + public boolean testRandomTypeInserts; + + @Override + public List getTestOracleFactory() { + return oracles; + } + + public enum ArangoDBOracleFactory implements OracleFactory { + QUERY_PARTITIONING { + @Override + public TestOracle create(ArangoDBProvider.ArangoDBGlobalState globalState) throws Exception { + List oracles = new ArrayList<>(); + oracles.add(new ArangoDBQueryPartitioningWhereTester(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + + } +} diff --git a/src/sqlancer/arangodb/ArangoDBProvider.java b/src/sqlancer/arangodb/ArangoDBProvider.java new file mode 100644 index 000000000..28160107d --- /dev/null +++ b/src/sqlancer/arangodb/ArangoDBProvider.java @@ -0,0 +1,128 @@ +package sqlancer.arangodb; + +import java.util.ArrayList; +import java.util.List; + +import com.arangodb.ArangoDB; +import com.arangodb.ArangoDatabase; + +import sqlancer.AbstractAction; +import sqlancer.ExecutionTimer; +import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; +import sqlancer.ProviderAdapter; +import sqlancer.Randomly; +import sqlancer.StatementExecutor; +import sqlancer.arangodb.gen.ArangoDBInsertGenerator; +import sqlancer.arangodb.gen.ArangoDBTableGenerator; +import sqlancer.common.log.LoggableFactory; +import sqlancer.common.query.Query; + +public class ArangoDBProvider + extends ProviderAdapter { + + public ArangoDBProvider() { + super(ArangoDBGlobalState.class, ArangoDBOptions.class); + } + + enum Action implements AbstractAction { + INSERT(ArangoDBInsertGenerator::getQuery); + + private final ArangoDBQueryProvider queryProvider; + + Action(ArangoDBQueryProvider queryProvider) { + this.queryProvider = queryProvider; + } + + @Override + public Query getQuery(ArangoDBGlobalState globalState) throws Exception { + return queryProvider.getQuery(globalState); + } + } + + private static int mapActions(ArangoDBGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + default: + throw new AssertionError(a); + } + } + + public static class ArangoDBGlobalState extends GlobalState { + + private final List schemaTables = new ArrayList<>(); + + public synchronized void addTable(ArangoDBSchema.ArangoDBTable table) { + schemaTables.add(table); + } + + @Override + protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { + boolean logExecutionTime = getOptions().logExecutionTime(); + if (success && getOptions().printSucceedingStatements()) { + System.out.println(q.getLogString()); + } + if (logExecutionTime) { + getLogger().writeCurrent("//" + timer.end().asString()); + } + if (q.couldAffectSchema()) { + updateSchema(); + } + } + + @Override + protected ArangoDBSchema readSchema() throws Exception { + return new ArangoDBSchema(schemaTables); + } + } + + @Override + protected void checkViewsAreValid(ArangoDBGlobalState globalState) { + + } + + @Override + public void generateDatabase(ArangoDBGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { + boolean success; + do { + ArangoDBQueryAdapter queryAdapter = new ArangoDBTableGenerator().getQuery(globalState); + success = globalState.executeStatement(queryAdapter); + } while (!success); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + ArangoDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public ArangoDBConnection createDatabase(ArangoDBGlobalState globalState) throws Exception { + ArangoDB arangoDB = new ArangoDB.Builder().user(globalState.getOptions().getUserName()) + .password(globalState.getOptions().getPassword()).build(); + ArangoDatabase database = arangoDB.db(globalState.getDatabaseName()); + try { + database.drop(); + } catch (Exception ignored) { + + } + arangoDB.createDatabase(globalState.getDatabaseName()); + database = arangoDB.db(globalState.getDatabaseName()); + return new ArangoDBConnection(arangoDB, database); + } + + @Override + public String getDBMSName() { + return "arangodb"; + } + + @Override + public LoggableFactory getLoggableFactory() { + return new ArangoDBLoggableFactory(); + } +} diff --git a/src/sqlancer/arangodb/ArangoDBQueryAdapter.java b/src/sqlancer/arangodb/ArangoDBQueryAdapter.java new file mode 100644 index 000000000..2d1b77af0 --- /dev/null +++ b/src/sqlancer/arangodb/ArangoDBQueryAdapter.java @@ -0,0 +1,15 @@ +package sqlancer.arangodb; + +import sqlancer.common.query.Query; + +public abstract class ArangoDBQueryAdapter extends Query { + @Override + public String getQueryString() { + throw new UnsupportedOperationException(); + } + + @Override + public String getUnterminatedQueryString() { + throw new UnsupportedOperationException(); + } +} diff --git a/src/sqlancer/arangodb/ArangoDBQueryProvider.java b/src/sqlancer/arangodb/ArangoDBQueryProvider.java new file mode 100644 index 000000000..94a4ffda3 --- /dev/null +++ b/src/sqlancer/arangodb/ArangoDBQueryProvider.java @@ -0,0 +1,6 @@ +package sqlancer.arangodb; + +@FunctionalInterface +public interface ArangoDBQueryProvider { + ArangoDBQueryAdapter getQuery(S globalState) throws Exception; +} diff --git a/src/sqlancer/arangodb/ArangoDBSchema.java b/src/sqlancer/arangodb/ArangoDBSchema.java new file mode 100644 index 000000000..e3ceb3586 --- /dev/null +++ b/src/sqlancer/arangodb/ArangoDBSchema.java @@ -0,0 +1,66 @@ +package sqlancer.arangodb; + +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; + +public class ArangoDBSchema extends AbstractSchema { + + public enum ArangoDBDataType { + INTEGER, DOUBLE, STRING, BOOLEAN; + + public static ArangoDBDataType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public static class ArangoDBColumn extends AbstractTableColumn { + + private final boolean isId; + private final boolean isNullable; + + public ArangoDBColumn(String name, ArangoDBDataType type, boolean isId, boolean isNullable) { + super(name, null, type); + this.isId = isId; + this.isNullable = isNullable; + } + + public boolean isId() { + return isId; + } + + public boolean isNullable() { + return isNullable; + } + } + + public ArangoDBSchema(List databaseTables) { + super(databaseTables); + } + + public static class ArangoDBTables extends AbstractTables { + + public ArangoDBTables(List tables) { + super(tables); + } + } + + public static class ArangoDBTable + extends AbstractTable { + + public ArangoDBTable(String name, List columns, boolean isView) { + super(name, columns, Collections.emptyList(), isView); + } + + @Override + public long getNrRows(ArangoDBProvider.ArangoDBGlobalState globalState) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/src/sqlancer/arangodb/ast/ArangoDBConstant.java b/src/sqlancer/arangodb/ast/ArangoDBConstant.java new file mode 100644 index 000000000..cb60761c8 --- /dev/null +++ b/src/sqlancer/arangodb/ast/ArangoDBConstant.java @@ -0,0 +1,126 @@ +package sqlancer.arangodb.ast; + +import com.arangodb.entity.BaseDocument; + +import sqlancer.common.ast.newast.Node; + +public abstract class ArangoDBConstant implements Node { + private ArangoDBConstant() { + + } + + public abstract void setValueInDocument(BaseDocument document, String key); + + public abstract void getLogValue(); + + public abstract Object getValue(); + + public static class ArangoDBIntegerConstant extends ArangoDBConstant { + + private final int value; + + public ArangoDBIntegerConstant(int value) { + this.value = value; + } + + @Override + public void setValueInDocument(BaseDocument document, String key) { + document.addAttribute(key, value); + } + + @Override + public void getLogValue() { + throw new UnsupportedOperationException(); + } + + @Override + public Object getValue() { + return value; + } + } + + public static Node createIntegerConstant(int value) { + return new ArangoDBIntegerConstant(value); + } + + public static class ArangoDBStringConstant extends ArangoDBConstant { + private final String value; + + public ArangoDBStringConstant(String value) { + this.value = value; + } + + @Override + public void setValueInDocument(BaseDocument document, String key) { + document.addAttribute(key, value); + } + + @Override + public void getLogValue() { + throw new UnsupportedOperationException(); + } + + @Override + public Object getValue() { + return value; + } + } + + public static Node createStringConstant(String value) { + return new ArangoDBStringConstant(value); + } + + public static class ArangoDBBooleanConstant extends ArangoDBConstant { + private final boolean value; + + public ArangoDBBooleanConstant(boolean value) { + this.value = value; + } + + @Override + public void setValueInDocument(BaseDocument document, String key) { + document.addAttribute(key, value); + } + + @Override + public void getLogValue() { + throw new UnsupportedOperationException(); + } + + @Override + public Object getValue() { + return value; + } + } + + public static Node createBooleanConstant(boolean value) { + return new ArangoDBBooleanConstant(value); + } + + public static class ArangoDBDoubleConstant extends ArangoDBConstant { + private final double value; + + public ArangoDBDoubleConstant(double value) { + this.value = value; + } + + @Override + public void setValueInDocument(BaseDocument document, String key) { + document.addAttribute(key, value); + } + + @Override + public void getLogValue() { + throw new UnsupportedOperationException(); + } + + @Override + public Object getValue() { + return value; + } + } + + public static Node createDoubleConstant(double value) { + return new ArangoDBDoubleConstant(value); + } +} diff --git a/src/sqlancer/arangodb/ast/ArangoDBExpression.java b/src/sqlancer/arangodb/ast/ArangoDBExpression.java new file mode 100644 index 000000000..facbbfe9e --- /dev/null +++ b/src/sqlancer/arangodb/ast/ArangoDBExpression.java @@ -0,0 +1,4 @@ +package sqlancer.arangodb.ast; + +public interface ArangoDBExpression { +} diff --git a/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java new file mode 100644 index 000000000..9a27ccd57 --- /dev/null +++ b/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java @@ -0,0 +1,39 @@ +package sqlancer.arangodb.gen; + +import com.arangodb.entity.BaseDocument; + +import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.arangodb.ArangoDBQueryAdapter; +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.arangodb.query.ArangoDBConstantGenerator; +import sqlancer.arangodb.query.ArangoDBInsertQuery; + +public final class ArangoDBInsertGenerator { + + private final ArangoDBProvider.ArangoDBGlobalState globalState; + + private ArangoDBInsertGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { + this.globalState = globalState; + } + + public static ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { + return new ArangoDBInsertGenerator(globalState).generate(); + } + + private ArangoDBQueryAdapter generate() { + BaseDocument result = new BaseDocument(); + ArangoDBSchema.ArangoDBTable table = globalState.getSchema().getRandomTable(); + ArangoDBConstantGenerator constantGenerator = new ArangoDBConstantGenerator(globalState); + + for (int i = 0; i < table.getColumns().size(); i++) { + if (!globalState.getDmbsSpecificOptions().testRandomTypeInserts) { + constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), + table.getColumns().get(i).getType()); + } else { + constantGenerator.addRandomConstant(result, table.getColumns().get(i).getName()); + } + } + + return new ArangoDBInsertQuery(table, result); + } +} diff --git a/src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java new file mode 100644 index 000000000..1236c3ce4 --- /dev/null +++ b/src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java @@ -0,0 +1,44 @@ +package sqlancer.arangodb.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.arangodb.ArangoDBQueryAdapter; +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.arangodb.query.ArangoDBCreateTableQuery; + +public class ArangoDBTableGenerator { + + private ArangoDBSchema.ArangoDBTable table; + private final List columnsToBeAdded = new ArrayList<>(); + + public ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { + String tableName = globalState.getSchema().getFreeTableName(); + ArangoDBCreateTableQuery createTableQuery = new ArangoDBCreateTableQuery(tableName); + table = new ArangoDBSchema.ArangoDBTable(tableName, columnsToBeAdded, false); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + createColumn(columnName); + } + globalState.addTable(table); + return createTableQuery; + } + + private ArangoDBSchema.ArangoDBDataType createColumn(String columnName) { + ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); + ArangoDBSchema.ArangoDBColumn newColumn = new ArangoDBSchema.ArangoDBColumn(columnName, dataType, false, false); + newColumn.setTable(table); + columnsToBeAdded.add(newColumn); + return dataType; + } + + public String getTableName() { + return table.getName(); + } + + public ArangoDBSchema.ArangoDBTable getGeneratedTable() { + return table; + } +} diff --git a/src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java b/src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java new file mode 100644 index 000000000..406e8adca --- /dev/null +++ b/src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java @@ -0,0 +1,46 @@ +package sqlancer.arangodb.query; + +import com.arangodb.entity.BaseDocument; + +import sqlancer.Randomly; +import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.arangodb.ast.ArangoDBConstant; + +public class ArangoDBConstantGenerator { + private final ArangoDBProvider.ArangoDBGlobalState globalState; + + public ArangoDBConstantGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { + this.globalState = globalState; + } + + public void addRandomConstant(BaseDocument document, String key) { + ArangoDBSchema.ArangoDBDataType type = ArangoDBSchema.ArangoDBDataType.getRandom(); + addRandomConstantWithType(document, key, type); + } + + public void addRandomConstantWithType(BaseDocument document, String key, ArangoDBSchema.ArangoDBDataType dataType) { + ArangoDBConstant constant; + switch (dataType) { + case STRING: + constant = new ArangoDBConstant.ArangoDBStringConstant(globalState.getRandomly().getString()); + constant.setValueInDocument(document, key); + return; + case DOUBLE: + constant = new ArangoDBConstant.ArangoDBDoubleConstant(globalState.getRandomly().getDouble()); + constant.setValueInDocument(document, key); + return; + case BOOLEAN: + constant = new ArangoDBConstant.ArangoDBBooleanConstant(Randomly.getBoolean()); + constant.setValueInDocument(document, key); + return; + case INTEGER: + constant = new ArangoDBConstant.ArangoDBIntegerConstant((int) globalState.getRandomly().getInteger()); + constant.setValueInDocument(document, key); + return; + default: + throw new AssertionError(dataType); + } + + } +} diff --git a/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java b/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java new file mode 100644 index 000000000..888c5d653 --- /dev/null +++ b/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java @@ -0,0 +1,45 @@ +package sqlancer.arangodb.query; + +import sqlancer.GlobalState; +import sqlancer.Main; +import sqlancer.arangodb.ArangoDBConnection; +import sqlancer.arangodb.ArangoDBQueryAdapter; +import sqlancer.common.query.ExpectedErrors; + +public class ArangoDBCreateTableQuery extends ArangoDBQueryAdapter { + + private final String tableName; + + public ArangoDBCreateTableQuery(String tableName) { + this.tableName = tableName; + } + + @Override + public boolean couldAffectSchema() { + return true; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + try { + globalState.getConnection().getDatabase().createCollection(tableName); + Main.nrSuccessfulActions.addAndGet(1); + return true; + } catch (Exception e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw e; + } + } + + @Override + public ExpectedErrors getExpectedErrors() { + return new ExpectedErrors(); + } + + @Override + public String getLogString() { + // TODO Patrick + return ""; + } +} diff --git a/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java b/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java new file mode 100644 index 000000000..327ccb99a --- /dev/null +++ b/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java @@ -0,0 +1,50 @@ +package sqlancer.arangodb.query; + +import com.arangodb.entity.BaseDocument; + +import sqlancer.GlobalState; +import sqlancer.Main; +import sqlancer.arangodb.ArangoDBConnection; +import sqlancer.arangodb.ArangoDBQueryAdapter; +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.common.query.ExpectedErrors; + +public class ArangoDBInsertQuery extends ArangoDBQueryAdapter { + + private final ArangoDBSchema.ArangoDBTable table; + private final BaseDocument documentToBeInserted; + + public ArangoDBInsertQuery(ArangoDBSchema.ArangoDBTable table, BaseDocument documentToBeInserted) { + this.table = table; + this.documentToBeInserted = documentToBeInserted; + } + + @Override + public boolean couldAffectSchema() { + return true; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + try { + globalState.getConnection().getDatabase().collection(table.getName()).insertDocument(documentToBeInserted); + Main.nrSuccessfulActions.addAndGet(1); + return true; + } catch (Exception e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw e; + } + } + + @Override + public ExpectedErrors getExpectedErrors() { + return new ExpectedErrors(); + } + + @Override + public String getLogString() { + // TODO Patrick + return ""; + } +} diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java new file mode 100644 index 000000000..36768d01e --- /dev/null +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java @@ -0,0 +1,27 @@ +package sqlancer.arangodb.test; + +import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.arangodb.ast.ArangoDBExpression; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; + +public class ArangoDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, ArangoDBProvider.ArangoDBGlobalState> + implements TestOracle { + + protected ArangoDBQueryPartitioningBase(ArangoDBProvider.ArangoDBGlobalState state) { + super(state); + } + + @Override + protected ExpressionGenerator> getGen() { + throw new UnsupportedOperationException(); + } + + @Override + public void check() throws Exception { + + } +} diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java new file mode 100644 index 000000000..cbd380366 --- /dev/null +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java @@ -0,0 +1,9 @@ +package sqlancer.arangodb.test; + +import sqlancer.arangodb.ArangoDBProvider; + +public class ArangoDBQueryPartitioningWhereTester extends ArangoDBQueryPartitioningBase { + public ArangoDBQueryPartitioningWhereTester(ArangoDBProvider.ArangoDBGlobalState state) { + super(state); + } +} From 66919d4bf271ebf022cb0ee3833e9f5170bc6152 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sun, 28 Feb 2021 16:59:55 +0100 Subject: [PATCH 0384/1284] Complete log implementation for inserts and collection creation --- .../query/ArangoDBCreateTableQuery.java | 3 +-- .../arangodb/query/ArangoDBInsertQuery.java | 20 +++++++++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java b/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java index 888c5d653..00b3276d0 100644 --- a/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java +++ b/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java @@ -39,7 +39,6 @@ public ExpectedErrors getExpectedErrors() { @Override public String getLogString() { - // TODO Patrick - return ""; + return "db._create(\"" + tableName + "\")"; } } diff --git a/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java b/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java index 327ccb99a..9a3612062 100644 --- a/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java +++ b/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java @@ -1,5 +1,7 @@ package sqlancer.arangodb.query; +import java.util.Map; + import com.arangodb.entity.BaseDocument; import sqlancer.GlobalState; @@ -44,7 +46,21 @@ public ExpectedErrors getExpectedErrors() { @Override public String getLogString() { - // TODO Patrick - return ""; + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("db._query(\"INSERT { "); + String filler = ""; + for (Map.Entry stringObjectEntry : documentToBeInserted.getProperties().entrySet()) { + stringBuilder.append(filler); + filler = ", "; + stringBuilder.append(stringObjectEntry.getKey()).append(": "); + Object value = stringObjectEntry.getValue(); + if (value instanceof String) { + stringBuilder.append("'").append(value).append("'"); + } else { + stringBuilder.append(value); + } + } + stringBuilder.append("} IN ").append(table.getName()).append("\")"); + return stringBuilder.toString(); } } From ec3f710a98e1f810b7b4d312e6483051214e1483 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 2 Mar 2021 15:47:31 +0100 Subject: [PATCH 0385/1284] Implement ternary partitioning logic where tester for ArangoDB ArangoDB now randomly generates queries that support binary comparison, binary logic operations such as or/and and the unary prefix not. Also everything is logged and the results are checked with the new ComparatorHelper. --- .../arangodb/ArangoDBComparatorHelper.java | 73 +++++++++ src/sqlancer/arangodb/ArangoDBSchema.java | 4 + .../arangodb/ast/ArangoDBConstant.java | 30 +--- src/sqlancer/arangodb/ast/ArangoDBSelect.java | 59 ++++++++ .../ast/ArangoDBUnsupportedPredicate.java | 6 + .../ArangoDBFilterExpressionGenerator.java | 139 ++++++++++++++++++ .../arangodb/query/ArangoDBSelectQuery.java | 65 ++++++++ .../test/ArangoDBQueryPartitioningBase.java | 26 +++- .../ArangoDBQueryPartitioningWhereTester.java | 29 ++++ .../visitor/ArangoDBToQueryVisitor.java | 92 ++++++++++++ .../arangodb/visitor/ArangoDBVisitor.java | 47 ++++++ 11 files changed, 545 insertions(+), 25 deletions(-) create mode 100644 src/sqlancer/arangodb/ArangoDBComparatorHelper.java create mode 100644 src/sqlancer/arangodb/ast/ArangoDBSelect.java create mode 100644 src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java create mode 100644 src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java create mode 100644 src/sqlancer/arangodb/query/ArangoDBSelectQuery.java create mode 100644 src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java create mode 100644 src/sqlancer/arangodb/visitor/ArangoDBVisitor.java diff --git a/src/sqlancer/arangodb/ArangoDBComparatorHelper.java b/src/sqlancer/arangodb/ArangoDBComparatorHelper.java new file mode 100644 index 000000000..70cc451fe --- /dev/null +++ b/src/sqlancer/arangodb/ArangoDBComparatorHelper.java @@ -0,0 +1,73 @@ +package sqlancer.arangodb; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.arangodb.entity.BaseDocument; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.arangodb.query.ArangoDBSelectQuery; +import sqlancer.common.query.ExpectedErrors; + +public final class ArangoDBComparatorHelper { + + private ArangoDBComparatorHelper() { + + } + + public static List getResultSetAsDocumentList(ArangoDBSelectQuery query, + ArangoDBProvider.ArangoDBGlobalState state) throws Exception { + ExpectedErrors errors = query.getExpectedErrors(); + List result; + try { + query.executeAndGet(state); + Main.nrSuccessfulActions.addAndGet(1); + result = query.getResultSet(); + return result; + } catch (Exception e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (e instanceof IgnoreMeException) { + throw e; + } + if (e.getMessage() == null) { + throw new AssertionError(query.getLogString(), e); + } + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + throw new AssertionError(query.getLogString(), e); + } + + } + + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, + ArangoDBSelectQuery originalQuery) { + if (resultSet.size() != secondResultSet.size()) { + String assertionMessage = String.format("The Size of the result sets mismatch (%d and %d)!\n%s", + resultSet.size(), secondResultSet.size(), originalQuery.getLogString()); + throw new AssertionError(assertionMessage); + } + Set firstHashSet = new HashSet<>(resultSet); + Set secondHashSet = new HashSet<>(secondResultSet); + + if (!firstHashSet.equals(secondHashSet)) { + Set firstResultSetMisses = new HashSet<>(firstHashSet); + firstResultSetMisses.removeAll(secondHashSet); + Set secondResultSetMisses = new HashSet<>(secondHashSet); + secondResultSetMisses.removeAll(firstHashSet); + StringBuilder firstMisses = new StringBuilder(); + for (BaseDocument document : firstResultSetMisses) { + firstMisses.append(document).append(" "); + } + StringBuilder secondMisses = new StringBuilder(); + for (BaseDocument document : secondResultSetMisses) { + secondMisses.append(document).append(" "); + } + String assertMessage = String.format("The Content of the result sets mismatch!\n %s \n %s\n %s", + firstMisses.toString(), secondMisses.toString(), originalQuery.getLogString()); + throw new AssertionError(assertMessage); + } + } +} diff --git a/src/sqlancer/arangodb/ArangoDBSchema.java b/src/sqlancer/arangodb/ArangoDBSchema.java index e3ceb3586..35e251b8b 100644 --- a/src/sqlancer/arangodb/ArangoDBSchema.java +++ b/src/sqlancer/arangodb/ArangoDBSchema.java @@ -63,4 +63,8 @@ public long getNrRows(ArangoDBProvider.ArangoDBGlobalState globalState) { throw new UnsupportedOperationException(); } } + + public ArangoDBTables getRandomTableNonEmptyTables() { + return new ArangoDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } } diff --git a/src/sqlancer/arangodb/ast/ArangoDBConstant.java b/src/sqlancer/arangodb/ast/ArangoDBConstant.java index cb60761c8..351dbd822 100644 --- a/src/sqlancer/arangodb/ast/ArangoDBConstant.java +++ b/src/sqlancer/arangodb/ast/ArangoDBConstant.java @@ -11,8 +11,6 @@ private ArangoDBConstant() { public abstract void setValueInDocument(BaseDocument document, String key); - public abstract void getLogValue(); - public abstract Object getValue(); public static class ArangoDBIntegerConstant extends ArangoDBConstant { @@ -28,11 +26,6 @@ public void setValueInDocument(BaseDocument document, String key) { document.addAttribute(key, value); } - @Override - public void getLogValue() { - throw new UnsupportedOperationException(); - } - @Override public Object getValue() { return value; @@ -55,14 +48,9 @@ public void setValueInDocument(BaseDocument document, String key) { document.addAttribute(key, value); } - @Override - public void getLogValue() { - throw new UnsupportedOperationException(); - } - @Override public Object getValue() { - return value; + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; } } @@ -82,11 +70,6 @@ public void setValueInDocument(BaseDocument document, String key) { document.addAttribute(key, value); } - @Override - public void getLogValue() { - throw new UnsupportedOperationException(); - } - @Override public Object getValue() { return value; @@ -101,7 +84,11 @@ public static class ArangoDBDoubleConstant extends ArangoDBConstant { private final double value; public ArangoDBDoubleConstant(double value) { - this.value = value; + if (Double.isInfinite(value) || Double.isNaN(value)) { + this.value = 0.0; + } else { + this.value = value; + } } @Override @@ -109,11 +96,6 @@ public void setValueInDocument(BaseDocument document, String key) { document.addAttribute(key, value); } - @Override - public void getLogValue() { - throw new UnsupportedOperationException(); - } - @Override public Object getValue() { return value; diff --git a/src/sqlancer/arangodb/ast/ArangoDBSelect.java b/src/sqlancer/arangodb/ast/ArangoDBSelect.java new file mode 100644 index 000000000..cbba174c1 --- /dev/null +++ b/src/sqlancer/arangodb/ast/ArangoDBSelect.java @@ -0,0 +1,59 @@ +package sqlancer.arangodb.ast; + +import java.util.List; + +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.common.ast.newast.Node; + +public class ArangoDBSelect implements Node { + private List fromColumns; + private List projectionColumns; + private boolean hasFilter; + private Node filterClause; + + public List getFromColumns() { + if (fromColumns == null || fromColumns.isEmpty()) { + throw new IllegalStateException(); + } + return fromColumns; + } + + public void setFromColumns(List fromColumns) { + if (fromColumns == null || fromColumns.isEmpty()) { + throw new IllegalStateException(); + } + this.fromColumns = fromColumns; + } + + public List getProjectionColumns() { + if (projectionColumns == null) { + throw new IllegalStateException(); + } + return projectionColumns; + } + + public void setProjectionColumns(List projectionColumns) { + if (projectionColumns == null) { + throw new IllegalStateException(); + } + this.projectionColumns = projectionColumns; + } + + public void setFilterClause(Node filterClause) { + if (filterClause == null) { + hasFilter = false; + this.filterClause = null; + return; + } + hasFilter = true; + this.filterClause = filterClause; + } + + public Node getFilterClause() { + return filterClause; + } + + public boolean hasFilter() { + return hasFilter; + } +} diff --git a/src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java b/src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java new file mode 100644 index 000000000..eabd25578 --- /dev/null +++ b/src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java @@ -0,0 +1,6 @@ +package sqlancer.arangodb.ast; + +import sqlancer.common.ast.newast.Node; + +public class ArangoDBUnsupportedPredicate implements Node { +} diff --git a/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java new file mode 100644 index 000000000..902a33c68 --- /dev/null +++ b/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java @@ -0,0 +1,139 @@ +package sqlancer.arangodb.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.arangodb.ast.ArangoDBConstant; +import sqlancer.arangodb.ast.ArangoDBExpression; +import sqlancer.arangodb.ast.ArangoDBUnsupportedPredicate; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; + +public class ArangoDBFilterExpressionGenerator + extends UntypedExpressionGenerator, ArangoDBSchema.ArangoDBColumn> { + + private final ArangoDBProvider.ArangoDBGlobalState globalState; + + private enum Expression { + BINARY_LOGICAL, UNARY_PREFIX, BINARY_COMPARISON + } + + public ArangoDBFilterExpressionGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public Node generateConstant() { + ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); + switch (dataType) { + case INTEGER: + return ArangoDBConstant.createIntegerConstant((int) globalState.getRandomly().getInteger()); + case BOOLEAN: + return ArangoDBConstant.createBooleanConstant(Randomly.getBoolean()); + case DOUBLE: + return ArangoDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case STRING: + return ArangoDBConstant.createStringConstant(globalState.getRandomly().getString()); + default: + throw new AssertionError(dataType); + } + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expression = Randomly.fromList(possibleOptions); + switch (expression) { + case BINARY_COMPARISON: + BinaryOperatorNode.Operator op = ArangoDBBinaryComparisonOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case UNARY_PREFIX: + return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), + ArangoDBUnaryPrefixOperator.getRandom()); + case BINARY_LOGICAL: + op = ArangoDBBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + default: + throw new AssertionError(expression); + } + } + + @Override + protected Node generateColumn() { + ArangoDBSchema.ArangoDBColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode<>(column); + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, ArangoDBUnaryPrefixOperator.NOT); + } + + @Override + public Node isNull(Node expr) { + return new ArangoDBUnsupportedPredicate<>(); + } + + public enum ArangoDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("=="), NOT_EQUALS("!="), LESS_THAN("<"), LESS_OR_EQUAL("<="), GREATER_THAN(">"), GREATER_OR_EQUAL(">="); + + private final String representation; + + ArangoDBBinaryComparisonOperator(String representation) { + this.representation = representation; + } + + @Override + public String getTextRepresentation() { + return representation; + } + + public static ArangoDBBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum ArangoDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("!"); + + private final String representation; + + ArangoDBUnaryPrefixOperator(String representation) { + this.representation = representation; + } + + @Override + public String getTextRepresentation() { + return representation; + } + + public static ArangoDBUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum ArangoDBBinaryLogicalOperator implements BinaryOperatorNode.Operator { + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + } + +} diff --git a/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java b/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java new file mode 100644 index 000000000..400585ca2 --- /dev/null +++ b/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java @@ -0,0 +1,65 @@ +package sqlancer.arangodb.query; + +import java.io.IOException; +import java.util.List; + +import com.arangodb.ArangoCursor; +import com.arangodb.entity.BaseDocument; + +import sqlancer.GlobalState; +import sqlancer.arangodb.ArangoDBConnection; +import sqlancer.arangodb.ArangoDBQueryAdapter; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; + +public class ArangoDBSelectQuery extends ArangoDBQueryAdapter { + + private final String query; + + private List resultSet; + + public ArangoDBSelectQuery(String query) { + this.query = query; + } + + @Override + public boolean couldAffectSchema() { + return false; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public ExpectedErrors getExpectedErrors() { + return new ExpectedErrors(); + } + + @Override + public String getLogString() { + return "db._query(\"" + query + "\")"; + } + + @Override + public > SQLancerResultSet executeAndGet(G globalState, + String... fills) throws Exception { + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(this.getLogString()); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + ArangoCursor cursor = globalState.getConnection().getDatabase().query(query, BaseDocument.class); + resultSet = cursor.asListRemaining(); + return null; + } + + public List getResultSet() { + return resultSet; + } +} diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java index 36768d01e..97c5cc154 100644 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java @@ -1,7 +1,13 @@ package sqlancer.arangodb.test; +import java.util.List; + +import sqlancer.Randomly; import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.arangodb.ArangoDBSchema; import sqlancer.arangodb.ast.ArangoDBExpression; +import sqlancer.arangodb.ast.ArangoDBSelect; +import sqlancer.arangodb.gen.ArangoDBFilterExpressionGenerator; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; @@ -11,17 +17,35 @@ public class ArangoDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, ArangoDBProvider.ArangoDBGlobalState> implements TestOracle { + protected ArangoDBSchema schema; + protected List targetColumns; + protected ArangoDBFilterExpressionGenerator expressionGenerator; + protected ArangoDBSelect select; + protected ArangoDBQueryPartitioningBase(ArangoDBProvider.ArangoDBGlobalState state) { super(state); } @Override protected ExpressionGenerator> getGen() { - throw new UnsupportedOperationException(); + return expressionGenerator; } @Override public void check() throws Exception { + schema = state.getSchema(); + generateTargetColumns(); + expressionGenerator = new ArangoDBFilterExpressionGenerator(state).setColumns(targetColumns); + initializeTernaryPredicateVariants(); + select = new ArangoDBSelect<>(); + select.setFromColumns(targetColumns); + select.setProjectionColumns(Randomly.nonEmptySubset(targetColumns)); + } + private void generateTargetColumns() { + ArangoDBSchema.ArangoDBTables targetTables; + targetTables = schema.getRandomTableNonEmptyTables(); + List allColumns = targetTables.getColumns(); + targetColumns = Randomly.nonEmptySubset(allColumns); } } diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java index cbd380366..80b7d46bf 100644 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java @@ -1,9 +1,38 @@ package sqlancer.arangodb.test; +import static sqlancer.arangodb.ArangoDBComparatorHelper.assumeResultSetsAreEqual; +import static sqlancer.arangodb.ArangoDBComparatorHelper.getResultSetAsDocumentList; + +import java.util.List; + +import com.arangodb.entity.BaseDocument; + import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.arangodb.query.ArangoDBSelectQuery; +import sqlancer.arangodb.visitor.ArangoDBVisitor; public class ArangoDBQueryPartitioningWhereTester extends ArangoDBQueryPartitioningBase { public ArangoDBQueryPartitioningWhereTester(ArangoDBProvider.ArangoDBGlobalState state) { super(state); } + + @Override + public void check() throws Exception { + super.check(); + select.setFilterClause(null); + + ArangoDBSelectQuery query = ArangoDBVisitor.asSelectQuery(select); + List firstResultSet = getResultSetAsDocumentList(query, state); + + select.setFilterClause(predicate); + query = ArangoDBVisitor.asSelectQuery(select); + List secondResultSet = getResultSetAsDocumentList(query, state); + + select.setFilterClause(negatedPredicate); + query = ArangoDBVisitor.asSelectQuery(select); + List thirdResultSet = getResultSetAsDocumentList(query, state); + + secondResultSet.addAll(thirdResultSet); + assumeResultSetsAreEqual(firstResultSet, secondResultSet, query); + } } diff --git a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java new file mode 100644 index 000000000..160249df8 --- /dev/null +++ b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java @@ -0,0 +1,92 @@ +package sqlancer.arangodb.visitor; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.arangodb.ast.ArangoDBConstant; +import sqlancer.arangodb.ast.ArangoDBExpression; +import sqlancer.arangodb.ast.ArangoDBSelect; +import sqlancer.arangodb.query.ArangoDBSelectQuery; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class ArangoDBToQueryVisitor extends ArangoDBVisitor { + + private final StringBuilder stringBuilder; + + public ArangoDBToQueryVisitor() { + stringBuilder = new StringBuilder(); + } + + @Override + protected void visit(ArangoDBSelect expression) { + generateFrom(expression); + if (expression.hasFilter()) { + stringBuilder.append("FILTER "); + visit(expression.getFilterClause()); + stringBuilder.append(" "); + } + generateProject(expression); + + } + + @Override + protected void visit(ColumnReferenceNode expression) { + stringBuilder.append("r").append(expression.getColumn().getTable().getName()).append(".") + .append(expression.getColumn().getName()); + } + + @Override + protected void visit(ArangoDBConstant expression) { + stringBuilder.append(expression.getValue()); + } + + @Override + protected void visit(NewBinaryOperatorNode expression) { + stringBuilder.append("("); + visit(expression.getLeft()); + stringBuilder.append(" ").append(expression.getOperatorRepresentation()).append(" "); + visit(expression.getRight()); + stringBuilder.append(")"); + } + + @Override + protected void visit(NewUnaryPrefixOperatorNode expression) { + stringBuilder.append(expression.getOperatorRepresentation()).append("("); + visit(expression.getExpr()); + stringBuilder.append(")"); + } + + private void generateFrom(ArangoDBSelect expression) { + List forColumns = expression.getFromColumns(); + Set tables = new HashSet<>(); + for (ArangoDBSchema.ArangoDBColumn column : forColumns) { + tables.add(column.getTable()); + } + + for (ArangoDBSchema.ArangoDBTable table : tables) { + stringBuilder.append("FOR r").append(table.getName()).append(" IN ").append(table.getName()).append(" "); + } + } + + private void generateProject(ArangoDBSelect expression) { + List projectColumns = expression.getProjectionColumns(); + stringBuilder.append("RETURN {"); + String filler = ""; + for (ArangoDBSchema.ArangoDBColumn column : projectColumns) { + stringBuilder.append(filler); + filler = ", "; + stringBuilder.append(column.getTable().getName()).append("_").append(column.getName()).append(": r") + .append(column.getTable().getName()).append(".").append(column.getName()); + } + stringBuilder.append("}"); + } + + public ArangoDBSelectQuery getQuery() { + return new ArangoDBSelectQuery(stringBuilder.toString()); + } + +} diff --git a/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java new file mode 100644 index 000000000..322559952 --- /dev/null +++ b/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java @@ -0,0 +1,47 @@ +package sqlancer.arangodb.visitor; + +import sqlancer.arangodb.ast.ArangoDBConstant; +import sqlancer.arangodb.ast.ArangoDBExpression; +import sqlancer.arangodb.ast.ArangoDBSelect; +import sqlancer.arangodb.query.ArangoDBSelectQuery; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; + +public abstract class ArangoDBVisitor { + + protected abstract void visit(ArangoDBSelect expression); + + protected abstract void visit(ColumnReferenceNode expression); + + protected abstract void visit(ArangoDBConstant expression); + + protected abstract void visit(NewBinaryOperatorNode expression); + + protected abstract void visit(NewUnaryPrefixOperatorNode expression); + + public void visit(Node expressionNode) { + if (expressionNode instanceof ArangoDBSelect) { + visit((ArangoDBSelect) expressionNode); + } else if (expressionNode instanceof ColumnReferenceNode) { + visit((ColumnReferenceNode) expressionNode); + } else if (expressionNode instanceof ArangoDBConstant) { + visit((ArangoDBConstant) expressionNode); + } else if (expressionNode instanceof NewBinaryOperatorNode) { + visit((NewBinaryOperatorNode) expressionNode); + } else if (expressionNode instanceof NewUnaryPrefixOperatorNode) { + visit((NewUnaryPrefixOperatorNode) expressionNode); + } else { + System.out.println(expressionNode.getClass()); + System.exit(0); + throw new AssertionError(expressionNode); + } + } + + public static ArangoDBSelectQuery asSelectQuery(Node expressionNode) { + ArangoDBToQueryVisitor visitor = new ArangoDBToQueryVisitor(); + visitor.visit(expressionNode); + return visitor.getQuery(); + } +} From acf3d34ad4891cc7d1452734bdde79a07de36cf7 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 2 Mar 2021 17:00:12 +0100 Subject: [PATCH 0386/1284] Support adding indexes when inserting data for ArangoDB A new generator and query has been added to support the new functionality. --- src/sqlancer/arangodb/ArangoDBOptions.java | 3 ++ src/sqlancer/arangodb/ArangoDBProvider.java | 5 +- .../gen/ArangoDBCreateIndexGenerator.java | 21 ++++++++ .../query/ArangoDBCreateIndexQuery.java | 54 +++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java create mode 100644 src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index 04e1038f9..cdb7ee759 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -22,6 +22,9 @@ public class ArangoDBOptions implements DBMSSpecificOptions getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/arangodb/ArangoDBProvider.java b/src/sqlancer/arangodb/ArangoDBProvider.java index 28160107d..fbf6f153e 100644 --- a/src/sqlancer/arangodb/ArangoDBProvider.java +++ b/src/sqlancer/arangodb/ArangoDBProvider.java @@ -13,6 +13,7 @@ import sqlancer.ProviderAdapter; import sqlancer.Randomly; import sqlancer.StatementExecutor; +import sqlancer.arangodb.gen.ArangoDBCreateIndexGenerator; import sqlancer.arangodb.gen.ArangoDBInsertGenerator; import sqlancer.arangodb.gen.ArangoDBTableGenerator; import sqlancer.common.log.LoggableFactory; @@ -26,7 +27,7 @@ public ArangoDBProvider() { } enum Action implements AbstractAction { - INSERT(ArangoDBInsertGenerator::getQuery); + INSERT(ArangoDBInsertGenerator::getQuery), CREATE_INDEX(ArangoDBCreateIndexGenerator::getQuery); private final ArangoDBQueryProvider queryProvider; @@ -45,6 +46,8 @@ private static int mapActions(ArangoDBGlobalState globalState, Action a) { switch (a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case CREATE_INDEX: + return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumberIndexes); default: throw new AssertionError(a); } diff --git a/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java new file mode 100644 index 000000000..cc66a3ef9 --- /dev/null +++ b/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java @@ -0,0 +1,21 @@ +package sqlancer.arangodb.gen; + +import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.arangodb.ArangoDBQueryAdapter; +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.arangodb.query.ArangoDBCreateIndexQuery; + +public final class ArangoDBCreateIndexGenerator { + private ArangoDBCreateIndexGenerator() { + + } + + public static ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { + ArangoDBSchema.ArangoDBColumn column = null; + while (column == null) { + ArangoDBSchema.ArangoDBTable randomTable = globalState.getSchema().getRandomTable(); + column = randomTable.getRandomColumn(); + } + return new ArangoDBCreateIndexQuery(column); + } +} diff --git a/src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java b/src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java new file mode 100644 index 000000000..6c2cc1b75 --- /dev/null +++ b/src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java @@ -0,0 +1,54 @@ +package sqlancer.arangodb.query; + +import java.util.Collections; + +import com.arangodb.ArangoCollection; + +import sqlancer.GlobalState; +import sqlancer.Main; +import sqlancer.arangodb.ArangoDBConnection; +import sqlancer.arangodb.ArangoDBQueryAdapter; +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.common.query.ExpectedErrors; + +public class ArangoDBCreateIndexQuery extends ArangoDBQueryAdapter { + + private final ArangoDBSchema.ArangoDBColumn column; + + public ArangoDBCreateIndexQuery(ArangoDBSchema.ArangoDBColumn column) { + this.column = column; + } + + @Override + public boolean couldAffectSchema() { + return false; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + try { + ArangoCollection collection = globalState.getConnection().getDatabase() + .collection(column.getTable().getName()); + collection.ensureHashIndex(Collections.singletonList(column.getName()), null); + Main.nrSuccessfulActions.addAndGet(1); + return true; + } catch (Exception e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw e; + } + } + + @Override + public ExpectedErrors getExpectedErrors() { + return new ExpectedErrors(); + } + + @Override + public String getLogString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("db.").append(column.getTable().getName()) + .append(".ensureIndex({type: \"hash\", fields: [ \"").append(column.getName()).append("\" ]});"); + return stringBuilder.toString(); + } +} From 242c2458f6732c907145887f0bb7aee1fd2a4a02 Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 2 Mar 2021 22:14:44 +0100 Subject: [PATCH 0387/1284] Add support for computed functions for ArangoDB Similar to MongoDB, in ArangoDB variables for computed values are created and with a special keyword LET calculated. --- src/sqlancer/arangodb/ast/ArangoDBSelect.java | 20 +++++ .../ArangoDBComputedExpressionGenerator.java | 85 +++++++++++++++++++ .../test/ArangoDBQueryPartitioningBase.java | 14 +++ .../visitor/ArangoDBToQueryVisitor.java | 43 +++++++++- .../arangodb/visitor/ArangoDBVisitor.java | 30 ++++--- 5 files changed, 176 insertions(+), 16 deletions(-) create mode 100644 src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java diff --git a/src/sqlancer/arangodb/ast/ArangoDBSelect.java b/src/sqlancer/arangodb/ast/ArangoDBSelect.java index cbba174c1..9fb91d553 100644 --- a/src/sqlancer/arangodb/ast/ArangoDBSelect.java +++ b/src/sqlancer/arangodb/ast/ArangoDBSelect.java @@ -10,6 +10,8 @@ public class ArangoDBSelect implements Node { private List projectionColumns; private boolean hasFilter; private Node filterClause; + private boolean hasComputed; + private List> computedClause; public List getFromColumns() { if (fromColumns == null || fromColumns.isEmpty()) { @@ -56,4 +58,22 @@ public Node getFilterClause() { public boolean hasFilter() { return hasFilter; } + + public void setComputedClause(List> computedColumns) { + if (computedColumns == null || computedColumns.isEmpty()) { + hasComputed = false; + this.computedClause = null; + return; + } + hasComputed = true; + this.computedClause = computedColumns; + } + + public List> getComputedClause() { + return computedClause; + } + + public boolean hasComputed() { + return hasComputed; + } } diff --git a/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java new file mode 100644 index 000000000..8a3b98871 --- /dev/null +++ b/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java @@ -0,0 +1,85 @@ +package sqlancer.arangodb.gen; + +import sqlancer.Randomly; +import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.arangodb.ArangoDBSchema; +import sqlancer.arangodb.ast.ArangoDBConstant; +import sqlancer.arangodb.ast.ArangoDBExpression; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; + +public class ArangoDBComputedExpressionGenerator + extends UntypedExpressionGenerator, ArangoDBSchema.ArangoDBColumn> { + private final ArangoDBProvider.ArangoDBGlobalState globalState; + + public ArangoDBComputedExpressionGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public Node generateConstant() { + ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); + switch (dataType) { + case INTEGER: + return ArangoDBConstant.createIntegerConstant((int) globalState.getRandomly().getInteger()); + case BOOLEAN: + return ArangoDBConstant.createBooleanConstant(Randomly.getBoolean()); + case DOUBLE: + return ArangoDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case STRING: + return ArangoDBConstant.createStringConstant(globalState.getRandomly().getString()); + default: + throw new AssertionError(dataType); + } + } + + public enum ComputedFunction { + ADD(2, "+"), MINUS(2, "-"), MULTIPLY(2, "*"), DIVISION(2, "/"), MODULUS(2, "%"); + + private final int nrArgs; + private final String operatorName; + + ComputedFunction(int nrArgs, String operatorName) { + this.nrArgs = nrArgs; + this.operatorName = operatorName; + } + + public static ComputedFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + public String getOperatorName() { + return operatorName; + } + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + ComputedFunction function = ComputedFunction.getRandom(); + return new NewFunctionNode<>(generateExpressions(depth + 1, function.getNrArgs()), function); + } + + @Override + protected Node generateColumn() { + return new ColumnReferenceNode<>(Randomly.fromList(columns)); + } + + @Override + public Node negatePredicate(Node predicate) { + throw new UnsupportedOperationException(); + } + + @Override + public Node isNull(Node expr) { + throw new UnsupportedOperationException(); + } +} diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java index 97c5cc154..e260e01f3 100644 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java @@ -1,5 +1,6 @@ package sqlancer.arangodb.test; +import java.util.ArrayList; import java.util.List; import sqlancer.Randomly; @@ -7,6 +8,7 @@ import sqlancer.arangodb.ArangoDBSchema; import sqlancer.arangodb.ast.ArangoDBExpression; import sqlancer.arangodb.ast.ArangoDBSelect; +import sqlancer.arangodb.gen.ArangoDBComputedExpressionGenerator; import sqlancer.arangodb.gen.ArangoDBFilterExpressionGenerator; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.ExpressionGenerator; @@ -40,6 +42,18 @@ public void check() throws Exception { select = new ArangoDBSelect<>(); select.setFromColumns(targetColumns); select.setProjectionColumns(Randomly.nonEmptySubset(targetColumns)); + generateComputedClause(); + } + + private void generateComputedClause() { + List> computedColumns = new ArrayList<>(); + int numberComputedColumns = state.getRandomly().getInteger(0, 4); + ArangoDBComputedExpressionGenerator generator = new ArangoDBComputedExpressionGenerator(state); + generator.setColumns(targetColumns); + for (int i = 0; i < numberComputedColumns; i++) { + computedColumns.add(generator.generateExpression()); + } + select.setComputedClause(computedColumns); } private void generateTargetColumns() { diff --git a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java index 160249df8..05683623c 100644 --- a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java +++ b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java @@ -8,12 +8,15 @@ import sqlancer.arangodb.ast.ArangoDBConstant; import sqlancer.arangodb.ast.ArangoDBExpression; import sqlancer.arangodb.ast.ArangoDBSelect; +import sqlancer.arangodb.gen.ArangoDBComputedExpressionGenerator; import sqlancer.arangodb.query.ArangoDBSelectQuery; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; -public class ArangoDBToQueryVisitor extends ArangoDBVisitor { +public class ArangoDBToQueryVisitor extends ArangoDBVisitor { private final StringBuilder stringBuilder; @@ -24,17 +27,34 @@ public ArangoDBToQueryVisitor() { @Override protected void visit(ArangoDBSelect expression) { generateFrom(expression); + generateComputed(expression); + generateFilter(expression); + generateProject(expression); + } + + private void generateFilter(ArangoDBSelect expression) { if (expression.hasFilter()) { stringBuilder.append("FILTER "); visit(expression.getFilterClause()); stringBuilder.append(" "); } - generateProject(expression); + } + private void generateComputed(ArangoDBSelect expression) { + if (expression.hasComputed()) { + List> computedClause = expression.getComputedClause(); + int computedNumber = 0; + for (Node computedExpression : computedClause) { + stringBuilder.append("LET c").append(computedNumber).append(" = "); + visit(computedExpression); + stringBuilder.append(" "); + computedNumber++; + } + } } @Override - protected void visit(ColumnReferenceNode expression) { + protected void visit(ColumnReferenceNode expression) { stringBuilder.append("r").append(expression.getColumn().getTable().getName()).append(".") .append(expression.getColumn().getName()); } @@ -60,6 +80,23 @@ protected void visit(NewUnaryPrefixOperatorNode expression) stringBuilder.append(")"); } + @Override + protected void visit(NewFunctionNode expression) { + if (!(expression.getFunc() instanceof ArangoDBComputedExpressionGenerator.ComputedFunction)) { + throw new UnsupportedOperationException(); + } + ArangoDBComputedExpressionGenerator.ComputedFunction function = (ArangoDBComputedExpressionGenerator.ComputedFunction) expression + .getFunc(); + if (function.getNrArgs() != 2) { + throw new UnsupportedOperationException(); + } + stringBuilder.append("("); + visit(expression.getArgs().get(0)); + stringBuilder.append(" ").append(function.getOperatorName()).append(" "); + visit(expression.getArgs().get(1)); + stringBuilder.append(")"); + } + private void generateFrom(ArangoDBSelect expression) { List forColumns = expression.getFromColumns(); Set tables = new HashSet<>(); diff --git a/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java index 322559952..f1db84cf5 100644 --- a/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java +++ b/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java @@ -6,35 +6,39 @@ import sqlancer.arangodb.query.ArangoDBSelectQuery; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; -public abstract class ArangoDBVisitor { +public abstract class ArangoDBVisitor { - protected abstract void visit(ArangoDBSelect expression); + protected abstract void visit(ArangoDBSelect expression); - protected abstract void visit(ColumnReferenceNode expression); + protected abstract void visit(ColumnReferenceNode expression); protected abstract void visit(ArangoDBConstant expression); - protected abstract void visit(NewBinaryOperatorNode expression); + protected abstract void visit(NewBinaryOperatorNode expression); - protected abstract void visit(NewUnaryPrefixOperatorNode expression); + protected abstract void visit(NewUnaryPrefixOperatorNode expression); - public void visit(Node expressionNode) { + protected abstract void visit(NewFunctionNode expression); + + @SuppressWarnings("unchecked") + public void visit(Node expressionNode) { if (expressionNode instanceof ArangoDBSelect) { - visit((ArangoDBSelect) expressionNode); - } else if (expressionNode instanceof ColumnReferenceNode) { - visit((ColumnReferenceNode) expressionNode); + visit((ArangoDBSelect) expressionNode); + } else if (expressionNode instanceof ColumnReferenceNode) { + visit((ColumnReferenceNode) expressionNode); } else if (expressionNode instanceof ArangoDBConstant) { visit((ArangoDBConstant) expressionNode); } else if (expressionNode instanceof NewBinaryOperatorNode) { - visit((NewBinaryOperatorNode) expressionNode); + visit((NewBinaryOperatorNode) expressionNode); } else if (expressionNode instanceof NewUnaryPrefixOperatorNode) { - visit((NewUnaryPrefixOperatorNode) expressionNode); + visit((NewUnaryPrefixOperatorNode) expressionNode); + } else if (expressionNode instanceof NewFunctionNode) { + visit((NewFunctionNode) expressionNode); } else { - System.out.println(expressionNode.getClass()); - System.exit(0); throw new AssertionError(expressionNode); } } From 6de066ef8ad516219ebc5c53ffaa1000acf20d2e Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 2 Mar 2021 22:42:52 +0100 Subject: [PATCH 0388/1284] Enable computation values in filter expressions for ArangoDB --- .../gen/ArangoDBFilterExpressionGenerator.java | 18 ++++++++++++++++-- .../test/ArangoDBQueryPartitioningBase.java | 4 +++- .../visitor/ArangoDBToQueryVisitor.java | 8 ++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java index 902a33c68..1a2fc4b5e 100644 --- a/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java +++ b/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java @@ -21,6 +21,7 @@ public class ArangoDBFilterExpressionGenerator extends UntypedExpressionGenerator, ArangoDBSchema.ArangoDBColumn> { private final ArangoDBProvider.ArangoDBGlobalState globalState; + private int numberOfComputedVariables; private enum Expression { BINARY_LOGICAL, UNARY_PREFIX, BINARY_COMPARISON @@ -30,6 +31,10 @@ public ArangoDBFilterExpressionGenerator(ArangoDBProvider.ArangoDBGlobalState gl this.globalState = globalState; } + public void setNumberOfComputedVariables(int numberOfComputedVariables) { + this.numberOfComputedVariables = numberOfComputedVariables; + } + @Override public Node generateConstant() { ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); @@ -71,8 +76,17 @@ protected Node generateExpression(int depth) { @Override protected Node generateColumn() { - ArangoDBSchema.ArangoDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode<>(column); + ArangoDBSchema.ArangoDBTable dummy = new ArangoDBSchema.ArangoDBTable("", new ArrayList<>(), false); + if (Randomly.getBoolean() || numberOfComputedVariables == 0) { + ArangoDBSchema.ArangoDBColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode<>(column); + } else { + int maxNumber = globalState.getRandomly().getInteger(0, numberOfComputedVariables); + ArangoDBSchema.ArangoDBColumn column = new ArangoDBSchema.ArangoDBColumn("c" + maxNumber, + ArangoDBSchema.ArangoDBDataType.INTEGER, false, false); + column.setTable(dummy); + return new ColumnReferenceNode<>(column); + } } @Override diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java index e260e01f3..f583ed04f 100644 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java @@ -23,6 +23,7 @@ public class ArangoDBQueryPartitioningBase protected List targetColumns; protected ArangoDBFilterExpressionGenerator expressionGenerator; protected ArangoDBSelect select; + protected int numberComputedColumns; protected ArangoDBQueryPartitioningBase(ArangoDBProvider.ArangoDBGlobalState state) { super(state); @@ -35,9 +36,11 @@ protected ExpressionGenerator> getGen() { @Override public void check() throws Exception { + numberComputedColumns = state.getRandomly().getInteger(0, 4); schema = state.getSchema(); generateTargetColumns(); expressionGenerator = new ArangoDBFilterExpressionGenerator(state).setColumns(targetColumns); + expressionGenerator.setNumberOfComputedVariables(numberComputedColumns); initializeTernaryPredicateVariants(); select = new ArangoDBSelect<>(); select.setFromColumns(targetColumns); @@ -47,7 +50,6 @@ public void check() throws Exception { private void generateComputedClause() { List> computedColumns = new ArrayList<>(); - int numberComputedColumns = state.getRandomly().getInteger(0, 4); ArangoDBComputedExpressionGenerator generator = new ArangoDBComputedExpressionGenerator(state); generator.setColumns(targetColumns); for (int i = 0; i < numberComputedColumns; i++) { diff --git a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java index 05683623c..59f4ababa 100644 --- a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java +++ b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java @@ -55,8 +55,12 @@ private void generateComputed(ArangoDBSelect expression) { @Override protected void visit(ColumnReferenceNode expression) { - stringBuilder.append("r").append(expression.getColumn().getTable().getName()).append(".") - .append(expression.getColumn().getName()); + if (expression.getColumn().getTable().getName().equals("")) { + stringBuilder.append(expression.getColumn().getName()); + } else { + stringBuilder.append("r").append(expression.getColumn().getTable().getName()).append(".") + .append(expression.getColumn().getName()); + } } @Override From 97f5e0b83c435815b9364037ddf88af122aa566a Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 2 Mar 2021 23:52:29 +0100 Subject: [PATCH 0389/1284] Make query generation null safe with option in MongoDB --- src/sqlancer/mongodb/MongoDBOptions.java | 3 +++ src/sqlancer/mongodb/MongoDBProvider.java | 2 +- src/sqlancer/mongodb/MongoDBSchema.java | 14 ++++++----- .../MongoDBComputedExpressionGenerator.java | 2 +- .../mongodb/gen/MongoDBConstantGenerator.java | 12 ++++----- .../gen/MongoDBMatchExpressionGenerator.java | 8 +++--- .../mongodb/gen/MongoDBTableGenerator.java | 7 +++++- .../test/MongoDBQueryPartitioningBase.java | 25 ++++++++++--------- 8 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 5023a6fb8..4d894cdca 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -37,6 +37,9 @@ public class MongoDBOptions implements DBMSSpecificOptions oracles = Arrays.asList(QUERY_PARTITIONING); diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java index cf4375d01..cac7cb4ce 100644 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ b/src/sqlancer/mongodb/MongoDBProvider.java @@ -88,7 +88,7 @@ public void generateDatabase(MongoDBGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { boolean success; do { - MongoDBQueryAdapter query = new MongoDBTableGenerator().getQuery(globalState); + MongoDBQueryAdapter query = new MongoDBTableGenerator(globalState).getQuery(globalState); success = globalState.executeStatement(query); } while (!success); } diff --git a/src/sqlancer/mongodb/MongoDBSchema.java b/src/sqlancer/mongodb/MongoDBSchema.java index 535e0d5f8..4968ab9d1 100644 --- a/src/sqlancer/mongodb/MongoDBSchema.java +++ b/src/sqlancer/mongodb/MongoDBSchema.java @@ -55,14 +55,16 @@ public BsonType getBsonType() { } }; - public static MongoDBDataType getRandom() { - // TODO: If String is enabled, there are type issues. Find a way to have a cast or operation on top of the - // query - // TODO: to solve this issue. - MongoDBDataType[] valuesWithoutString = new MongoDBDataType[values().length - 1]; + public static MongoDBDataType getRandom(MongoDBGlobalState state) { + MongoDBDataType[] valuesWithoutString; + if (state.getDmbsSpecificOptions().nullSafety) { + valuesWithoutString = new MongoDBDataType[values().length - 1]; + } else { + valuesWithoutString = new MongoDBDataType[values().length]; + } int i = 0; for (MongoDBDataType type : values()) { - if (type.equals(STRING)) { + if (type.equals(STRING) && state.getDmbsSpecificOptions().nullSafety) { continue; } valuesWithoutString[i] = type; diff --git a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java index 1090b3f2f..fd5959ce6 100644 --- a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java @@ -67,7 +67,7 @@ public String getOperator() { @Override public Node generateConstant() { - MongoDBSchema.MongoDBDataType type = MongoDBSchema.MongoDBDataType.getRandom(); + MongoDBSchema.MongoDBDataType type = MongoDBSchema.MongoDBDataType.getRandom(globalState); MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); return generator.generateConstantWithType(type); } diff --git a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java index bda5c87e6..e81291543 100644 --- a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java @@ -42,7 +42,7 @@ public Node generateConstantWithType(MongoDBDataType option) } public void addRandomConstant(Document document, String key) { - MongoDBDataType type = MongoDBDataType.getRandom(); + MongoDBDataType type = MongoDBDataType.getRandom(globalState); addRandomConstantWithType(document, key, type); } @@ -67,12 +67,10 @@ public void addRandomConstantWithType(Document document, String key, MongoDBData constant = new MongoDBDoubleConstant(globalState.getRandomly().getDouble()); constant.setValueInDocument(document, key); return; - // TODO: If String is enabled, there are type issues. Find a way to have a cast or operation on top of the query - // TODO: to solve this issue. - // case STRING: - // constant = new MongoDBStringConstant(globalState.getRandomly().getString()); - // constant.setValueInDocument(document, key); - // return; + case STRING: + constant = new MongoDBConstant.MongoDBStringConstant(globalState.getRandomly().getString()); + constant.setValueInDocument(document, key); + return; case INTEGER: constant = new MongoDBIntegerConstant((int) globalState.getRandomly().getInteger()); constant.setValueInDocument(document, key); diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java index 6414ea185..191d0195e 100644 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java @@ -87,7 +87,7 @@ protected Node generateExpression(int depth) { @Override public Node generateConstant() { - MongoDBDataType type = MongoDBDataType.getRandom(); + MongoDBDataType type = MongoDBDataType.getRandom(globalState); MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); if (Randomly.getBooleanWithSmallProbability()) { return MongoDBConstant.createNullConstant(); @@ -97,9 +97,9 @@ public Node generateConstant() { public Node generateConstant(MongoDBDataType type) { MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); - // if (Randomly.getBooleanWithSmallProbability()) { - // return MongoDBConstant.createNullConstant(); - // } + if (Randomly.getBooleanWithSmallProbability() && !globalState.getDmbsSpecificOptions().nullSafety) { + return MongoDBConstant.createNullConstant(); + } return generator.generateConstantWithType(type); } diff --git a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java index f71ea6411..6a4f33d38 100644 --- a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java @@ -15,6 +15,11 @@ public class MongoDBTableGenerator { private MongoDBTable table; private final List columnsToBeAdded = new ArrayList<>(); + private final MongoDBGlobalState state; + + public MongoDBTableGenerator(MongoDBGlobalState state) { + this.state = state; + } public MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { String tableName = globalState.getSchema().getFreeTableName(); @@ -32,7 +37,7 @@ public MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { } private MongoDBDataType createColumn(String columnName) { - MongoDBDataType columnType = MongoDBDataType.getRandom(); + MongoDBDataType columnType = MongoDBDataType.getRandom(state); MongoDBColumn newColumn = new MongoDBColumn(columnName, columnType, false, false); newColumn.setTable(table); columnsToBeAdded.add(newColumn); diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java index 6421482a0..b26aa6028 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -42,12 +42,11 @@ public void check() throws Exception { initializeTernaryPredicateVariants(); select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); select.setProjectionList(targetColumns); - // TODO: Find a new way to split the query such that it can handle null fields to activate lookups - // if (Randomly.getBooleanWithRatherLowProbability()) { - select.setLookupList(targetColumns); - /* - * } else { select.setLookupList(Randomly.nonEmptySubset(targetColumns)); } - */ + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setLookupList(targetColumns); + } else { + select.setLookupList(Randomly.nonEmptySubset(targetColumns)); + } if (state.getDmbsSpecificOptions().testComputedValues) { generateComputedColumns(); } @@ -70,12 +69,14 @@ private void generateTargetColumns() { targetColumns.add(new MongoDBColumnTestReference(c, true)); } List joinsOtherTables = new ArrayList<>(); - // TODO: Find a new way to split the query such that it can handle null fields to activate lookups - /* - * for (int i = 1; i < targetTables.getTables().size(); i++) { MongoDBTable procTable = - * targetTables.getTables().get(i); for (MongoDBColumn c : procTable.getColumns()) { joinsOtherTables.add(new - * MongoDBColumnTestReference(c, false)); } } - */ + if (!state.getDmbsSpecificOptions().nullSafety) { + for (int i = 1; i < targetTables.getTables().size(); i++) { + MongoDBTable procTable = targetTables.getTables().get(i); + for (MongoDBColumn c : procTable.getColumns()) { + joinsOtherTables.add(new MongoDBColumnTestReference(c, false)); + } + } + } if (!joinsOtherTables.isEmpty()) { int randNumber = state.getRandomly().getInteger(1, Math.min(joinsOtherTables.size(), 4)); List subsetJoinsOtherTables = Randomly.nonEmptySubset(joinsOtherTables, From 8e7ef813f423c6a2a5b66bdc9a4cdfd272f21d40 Mon Sep 17 00:00:00 2001 From: Patrick Date: Wed, 3 Mar 2021 13:02:39 +0100 Subject: [PATCH 0390/1284] Introduce remove/reduce oracle to SQLancer for MongoDB This oracle generates a random query, executes it and if the result set is not empty, chooses a document at random and removes it from the collection. The query is executed again to check if the document is really removed and at the end a new document is generated and inserted to make sure that the dataset is not decreasing in size. --- .../common/oracle/RemoveReduceOracleBase.java | 29 ++++++ src/sqlancer/mongodb/MongoDBOptions.java | 12 ++- .../mongodb/query/MongoDBRemoveQuery.java | 59 ++++++++++++ .../mongodb/test/MongoDBRemoveReduceBase.java | 89 +++++++++++++++++++ .../test/MongoDBRemoveReduceTester.java | 49 ++++++++++ 5 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/common/oracle/RemoveReduceOracleBase.java create mode 100644 src/sqlancer/mongodb/query/MongoDBRemoveQuery.java create mode 100644 src/sqlancer/mongodb/test/MongoDBRemoveReduceBase.java create mode 100644 src/sqlancer/mongodb/test/MongoDBRemoveReduceTester.java diff --git a/src/sqlancer/common/oracle/RemoveReduceOracleBase.java b/src/sqlancer/common/oracle/RemoveReduceOracleBase.java new file mode 100644 index 000000000..c177f072e --- /dev/null +++ b/src/sqlancer/common/oracle/RemoveReduceOracleBase.java @@ -0,0 +1,29 @@ +package sqlancer.common.oracle; + +import sqlancer.GlobalState; +import sqlancer.common.gen.ExpressionGenerator; + +public abstract class RemoveReduceOracleBase> implements TestOracle { + + protected E predicate; + + protected final S state; + + protected RemoveReduceOracleBase(S state) { + this.state = state; + } + + protected void initializeRemoveReduceOracle() { + ExpressionGenerator gen = getGen(); + if (gen == null) { + throw new IllegalStateException(); + } + predicate = gen.generatePredicate(); + if (predicate == null) { + throw new IllegalStateException(); + } + } + + protected abstract ExpressionGenerator getGen(); + +} diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 4d894cdca..cf632085b 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -1,6 +1,7 @@ package sqlancer.mongodb; import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.QUERY_PARTITIONING; +import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.REMOVE_REDUCE; import java.util.ArrayList; import java.util.Arrays; @@ -13,6 +14,7 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.mongodb.test.MongoDBQueryPartitioningWhereTester; +import sqlancer.mongodb.test.MongoDBRemoveReduceTester; public class MongoDBOptions implements DBMSSpecificOptions { @@ -41,7 +43,7 @@ public class MongoDBOptions implements DBMSSpecificOptions oracles = Arrays.asList(QUERY_PARTITIONING); + public List oracles = Arrays.asList(QUERY_PARTITIONING, REMOVE_REDUCE); @Override public List getTestOracleFactory() { @@ -56,6 +58,14 @@ public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws oracles.add(new MongoDBQueryPartitioningWhereTester(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + REMOVE_REDUCE { + @Override + public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws Exception { + List oracles = new ArrayList<>(); + oracles.add(new MongoDBRemoveReduceTester(globalState)); + return new CompositeTestOracle(oracles, globalState); + } } } } diff --git a/src/sqlancer/mongodb/query/MongoDBRemoveQuery.java b/src/sqlancer/mongodb/query/MongoDBRemoveQuery.java new file mode 100644 index 000000000..6fe1c9e3f --- /dev/null +++ b/src/sqlancer/mongodb/query/MongoDBRemoveQuery.java @@ -0,0 +1,59 @@ +package sqlancer.mongodb.query; + +import org.bson.Document; +import org.bson.types.ObjectId; + +import com.mongodb.client.result.DeleteResult; + +import sqlancer.GlobalState; +import sqlancer.Main; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.mongodb.MongoDBConnection; +import sqlancer.mongodb.MongoDBQueryAdapter; +import sqlancer.mongodb.MongoDBSchema; + +public class MongoDBRemoveQuery extends MongoDBQueryAdapter { + + private final String objectId; + private final MongoDBSchema.MongoDBTable table; + + public MongoDBRemoveQuery(MongoDBSchema.MongoDBTable table, String objectId) { + this.objectId = objectId; + this.table = table; + } + + @Override + public boolean couldAffectSchema() { + return true; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + try { + DeleteResult result = globalState.getConnection().getDatabase().getCollection(table.getName()) + .deleteOne(new Document("_id", new ObjectId(objectId))); + if (result.wasAcknowledged()) { + Main.nrSuccessfulActions.addAndGet(1); + } else { + Main.nrUnsuccessfulActions.addAndGet(1); + } + return result.wasAcknowledged(); + } catch (Exception e) { + Main.nrUnsuccessfulActions.addAndGet(1); + return false; + } + } + + @Override + public ExpectedErrors getExpectedErrors() { + return new ExpectedErrors(); + } + + @Override + public String getLogString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("db.").append(table.getName()).append(".remove({'_id': '").append(objectId).append("'})"); + return stringBuilder.toString(); + } +} diff --git a/src/sqlancer/mongodb/test/MongoDBRemoveReduceBase.java b/src/sqlancer/mongodb/test/MongoDBRemoveReduceBase.java new file mode 100644 index 000000000..2a2a54744 --- /dev/null +++ b/src/sqlancer/mongodb/test/MongoDBRemoveReduceBase.java @@ -0,0 +1,89 @@ +package sqlancer.mongodb.test; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.RemoveReduceOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.mongodb.MongoDBProvider; +import sqlancer.mongodb.MongoDBSchema; +import sqlancer.mongodb.ast.MongoDBExpression; +import sqlancer.mongodb.ast.MongoDBSelect; +import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator; +import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; + +public class MongoDBRemoveReduceBase extends + RemoveReduceOracleBase, MongoDBProvider.MongoDBGlobalState> implements TestOracle { + + protected MongoDBSchema schema; + protected MongoDBSchema.MongoDBTables targetTables; + protected MongoDBSchema.MongoDBTable mainTable; + protected List targetColumns; + protected MongoDBMatchExpressionGenerator expressionGenerator; + protected MongoDBSelect select; + + protected MongoDBRemoveReduceBase(MongoDBProvider.MongoDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + schema = state.getSchema(); + targetTables = schema.getRandomTableNonEmptyTables(); + mainTable = targetTables.getTables().get(0); + generateTargetColumns(); + expressionGenerator = new MongoDBMatchExpressionGenerator(state).setColumns(targetColumns); + initializeRemoveReduceOracle(); + select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); + select.setProjectionList(targetColumns); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setLookupList(targetColumns); + } else { + select.setLookupList(Randomly.nonEmptySubset(targetColumns)); + } + if (state.getDmbsSpecificOptions().testComputedValues) { + generateComputedColumns(); + } + } + + private void generateTargetColumns() { + targetColumns = new ArrayList<>(); + for (MongoDBSchema.MongoDBColumn c : mainTable.getColumns()) { + targetColumns.add(new MongoDBColumnTestReference(c, true)); + } + List joinsOtherTables = new ArrayList<>(); + if (!state.getDmbsSpecificOptions().nullSafety) { + for (int i = 1; i < targetTables.getTables().size(); i++) { + MongoDBSchema.MongoDBTable procTable = targetTables.getTables().get(i); + for (MongoDBSchema.MongoDBColumn c : procTable.getColumns()) { + joinsOtherTables.add(new MongoDBColumnTestReference(c, false)); + } + } + } + if (!joinsOtherTables.isEmpty()) { + int randNumber = state.getRandomly().getInteger(1, Math.min(joinsOtherTables.size(), 4)); + List subsetJoinsOtherTables = Randomly.nonEmptySubset(joinsOtherTables, + randNumber); + targetColumns.addAll(subsetJoinsOtherTables); + } + } + + private void generateComputedColumns() { + List> computedColumns = new ArrayList<>(); + int numberComputedColumns = state.getRandomly().getInteger(1, 4); + MongoDBComputedExpressionGenerator generator = new MongoDBComputedExpressionGenerator(state) + .setColumns(targetColumns); + for (int i = 0; i < numberComputedColumns; i++) { + computedColumns.add(generator.generateExpression()); + } + select.setComputedClause(computedColumns); + } + + @Override + protected ExpressionGenerator> getGen() { + return expressionGenerator; + } +} diff --git a/src/sqlancer/mongodb/test/MongoDBRemoveReduceTester.java b/src/sqlancer/mongodb/test/MongoDBRemoveReduceTester.java new file mode 100644 index 000000000..829f63f23 --- /dev/null +++ b/src/sqlancer/mongodb/test/MongoDBRemoveReduceTester.java @@ -0,0 +1,49 @@ +package sqlancer.mongodb.test; + +import static sqlancer.mongodb.MongoDBComparatorHelper.getResultSetAsDocumentList; + +import java.util.List; + +import org.bson.Document; + +import sqlancer.Randomly; +import sqlancer.mongodb.MongoDBProvider; +import sqlancer.mongodb.MongoDBQueryAdapter; +import sqlancer.mongodb.gen.MongoDBInsertGenerator; +import sqlancer.mongodb.query.MongoDBRemoveQuery; +import sqlancer.mongodb.query.MongoDBSelectQuery; + +public class MongoDBRemoveReduceTester extends MongoDBRemoveReduceBase { + public MongoDBRemoveReduceTester(MongoDBProvider.MongoDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + + select.setWithCountClause(false); + + select.setFilterClause(predicate); + MongoDBSelectQuery selectQuery = new MongoDBSelectQuery(select); + List firstResultSet = getResultSetAsDocumentList(selectQuery, state); + if (firstResultSet == null || firstResultSet.isEmpty()) { + return; + } + + Document documentToRemove = Randomly.fromList(firstResultSet); + MongoDBRemoveQuery removeQuery = new MongoDBRemoveQuery(mainTable, documentToRemove.get("_id").toString()); + state.executeStatement(removeQuery); + + selectQuery = new MongoDBSelectQuery(select); + List secondResultSet = getResultSetAsDocumentList(selectQuery, state); + + MongoDBQueryAdapter insertQuery = MongoDBInsertGenerator.getQuery(state); + state.executeStatement(insertQuery); + + if (secondResultSet.size() + 1 != firstResultSet.size()) { + String assertMessage = "The Result Sizes mismatches!"; + throw new AssertionError(assertMessage); + } + } +} From b2afeefacff2274378682ad2f575a3524e43477f Mon Sep 17 00:00:00 2001 From: Patrick Date: Mon, 22 Mar 2021 23:00:04 +0100 Subject: [PATCH 0391/1284] Add Cosmos as a database configuration When using cosmos make sure to set the configuration string in CosmosProvider. --- src/sqlancer/Main.java | 2 + src/sqlancer/cosmos/CosmosProvider.java | 74 +++++++++++++++++++++++ src/sqlancer/mongodb/MongoDBProvider.java | 4 +- 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/cosmos/CosmosProvider.java diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index bfc0cabd8..c5da9ec98 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -28,6 +28,7 @@ import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.cosmos.CosmosProvider; import sqlancer.duckdb.DuckDBProvider; import sqlancer.h2.H2Provider; import sqlancer.mariadb.MariaDBProvider; @@ -557,6 +558,7 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new DuckDBProvider()); providers.add(new H2Provider()); providers.add(new MongoDBProvider()); + providers.add(new CosmosProvider()); providers.add(new ArangoDBProvider()); return providers; } diff --git a/src/sqlancer/cosmos/CosmosProvider.java b/src/sqlancer/cosmos/CosmosProvider.java new file mode 100644 index 000000000..a8681d3af --- /dev/null +++ b/src/sqlancer/cosmos/CosmosProvider.java @@ -0,0 +1,74 @@ +package sqlancer.cosmos; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; + +import sqlancer.IgnoreMeException; +import sqlancer.ProviderAdapter; +import sqlancer.Randomly; +import sqlancer.StatementExecutor; +import sqlancer.common.log.LoggableFactory; +import sqlancer.mongodb.MongoDBConnection; +import sqlancer.mongodb.MongoDBLoggableFactory; +import sqlancer.mongodb.MongoDBOptions; +import sqlancer.mongodb.MongoDBQueryAdapter; +import sqlancer.mongodb.gen.MongoDBTableGenerator; + +public class CosmosProvider extends + ProviderAdapter { + + public CosmosProvider() { + super(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState.class, MongoDBOptions.class); + } + + @Override + public void generateDatabase(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { + boolean success; + do { + MongoDBQueryAdapter query = new MongoDBTableGenerator(globalState).getQuery(globalState); + success = globalState.executeStatement(query); + } while (!success); + } + StatementExecutor se = new StatementExecutor<>( + globalState, sqlancer.mongodb.MongoDBProvider.Action.values(), + sqlancer.mongodb.MongoDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public MongoDBConnection createDatabase(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState globalState) + throws Exception { + String connectionString = ""; + if (connectionString.equals("")) { + throw new AssertionError("Please set connection string for cosmos database, located in CosmosProvider"); + } + MongoClientSettings settings = MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(connectionString)).build(); + MongoClient mongoClient = MongoClients.create(settings); + MongoDatabase database = mongoClient.getDatabase(globalState.getDatabaseName()); + database.drop(); + return new MongoDBConnection(mongoClient, database); + } + + @Override + public String getDBMSName() { + return "cosmos"; + } + + @Override + public LoggableFactory getLoggableFactory() { + return new MongoDBLoggableFactory(); + } + + @Override + protected void checkViewsAreValid(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState globalState) { + } +} diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java index cac7cb4ce..4712b5d9d 100644 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ b/src/sqlancer/mongodb/MongoDBProvider.java @@ -28,7 +28,7 @@ public MongoDBProvider() { super(MongoDBGlobalState.class, MongoDBOptions.class); } - enum Action implements AbstractAction { + public enum Action implements AbstractAction { INSERT(MongoDBInsertGenerator::getQuery), CREATE_INDEX(MongoDBIndexGenerator::getQuery); private final MongoDBQueryProvider queryProvider; @@ -43,7 +43,7 @@ public Query getQuery(MongoDBGlobalState globalState) throws } } - private static int mapActions(MongoDBGlobalState globalState, Action a) { + public static int mapActions(MongoDBGlobalState globalState, Action a) { Randomly r = globalState.getRandomly(); switch (a) { case INSERT: From d91ef8bd304b6afa61603722e0dd9aa974c2de2c Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 23 Mar 2021 17:51:49 +0300 Subject: [PATCH 0392/1284] Set options for TLPHaving correct behavior --- .../clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java | 1 + test/sqlancer/dbms/TestClickHouse.java | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index 3ab57ca1a..2cc2410d1 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -62,6 +62,7 @@ public void check() throws SQLException { ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.IS_NULL, false)); String thirdQueryString = ClickHouseVisitor.asString(select); String combinedString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL " + thirdQueryString; + combinedString += " SETTINGS aggregate_functions_null_for_empty=1, enable_optimize_predicate_expression=0"; // https://github.com/ClickHouse/ClickHouse/issues/12264 List secondResultSet = ComparatorHelper.getResultSetFirstColumnAsString(combinedString, errors, state); if (state.getOptions().logEachSelect()) { state.getLogger().writeCurrent(originalQueryString); diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 440deb1bc..2cf53eef2 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; import org.junit.jupiter.api.Test; + import sqlancer.Main; public class TestClickHouse { @@ -33,10 +34,9 @@ public void testClickHouseTLPHaving() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", "0", "--num-threads", "5", - "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPHaving")); // Disabled - // in CI - // https://github.com/ClickHouse/ClickHouse/issues/12264 + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPHaving")); } @Test From c601f4f65eaed2a49d5e848c1acf116ba2dafa00 Mon Sep 17 00:00:00 2001 From: EthanDBer Date: Thu, 25 Mar 2021 10:21:11 +0800 Subject: [PATCH 0393/1284] add constant support for the different DBMSs --- src/sqlancer/clickhouse/ClickHouseOptions.java | 3 ++- src/sqlancer/cockroachdb/CockroachDBOptions.java | 3 ++- src/sqlancer/mariadb/MariaDBOptions.java | 3 ++- src/sqlancer/mysql/MySQLOptions.java | 3 ++- src/sqlancer/postgres/PostgresOptions.java | 3 ++- src/sqlancer/tidb/TiDBOptions.java | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index e40488fc9..5c26cb03b 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -18,7 +18,8 @@ import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; import sqlancer.common.oracle.TestOracle; -@Parameters(separators = "=", commandDescription = "ClickHouse (default port: 8123, default host: localhost)") +@Parameters(separators = "=", commandDescription = "ClickHouse (default port: " + ClickHouseOptions.DEFAULT_PORT + + ", default host: " + ClickHouseOptions.DEFAULT_HOST) public class ClickHouseOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 8123; diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 149df7f60..2af62ae69 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -23,7 +23,8 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; -@Parameters(separators = "=", commandDescription = "CockroachDB (default port: 26257, default host: localhost)") +@Parameters(separators = "=", commandDescription = "CockroachDB (default port: " + CockroachDBOptions.DEFAULT_PORT + + " default host: " + CockroachDBOptions.DEFAULT_HOST) public class CockroachDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 26257; diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index ba9d7d8b4..63e142aec 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -14,7 +14,8 @@ import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.oracle.MariaDBNoRECOracle; -@Parameters(separators = "=", commandDescription = "MariaDB (default port: 3306, default host: localhost)") +@Parameters(separators = "=", commandDescription = "MariaDB (default port: " + MariaDBOptions.DEFAULT_PORT + + ", default host: " + MariaDBOptions.DEFAULT_HOST) public class MariaDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 377926d0f..5540f00a8 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -14,7 +14,8 @@ import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; -@Parameters(separators = "=", commandDescription = "MySQL (default port: 3306, default host: localhost)") +@Parameters(separators = "=", commandDescription = "MySQL (default port: " + MySQLOptions.DEFAULT_PORT + + ", default host: " + MySQLOptions.DEFAULT_HOST) public class MySQLOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 1614ef516..6c0a97847 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -19,7 +19,8 @@ import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; -@Parameters(separators = "=", commandDescription = "PostgreSQL (default port: 5432, default host: localhost)") +@Parameters(separators = "=", commandDescription = "PostgreSQL (default port: " + PostgresOptions.DEFAULT_PORT + + ", default host: " + PostgresOptions.DEFAULT_HOST) public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index b405e525d..e5da34fe0 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -17,7 +17,8 @@ import sqlancer.tidb.oracle.TiDBTLPHavingOracle; import sqlancer.tidb.oracle.TiDBTLPWhereOracle; -@Parameters(separators = "=", commandDescription = "PostgreSQL (default port: 4000, default host: localhost)") +@Parameters(separators = "=", commandDescription = "TiDB (default port: " + TiDBOptions.DEFAULT_PORT + + ", default host: " + TiDBOptions.DEFAULT_HOST) public class TiDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 4000; From ddd2d0219adaa97de4eba857050bc2f1858bbbc3 Mon Sep 17 00:00:00 2001 From: EthanDBer Date: Thu, 25 Mar 2021 10:21:11 +0800 Subject: [PATCH 0394/1284] add constant support for the different DBMSs --- src/sqlancer/clickhouse/ClickHouseOptions.java | 3 ++- src/sqlancer/cockroachdb/CockroachDBOptions.java | 3 ++- src/sqlancer/mariadb/MariaDBOptions.java | 3 ++- src/sqlancer/mysql/MySQLOptions.java | 3 ++- src/sqlancer/postgres/PostgresOptions.java | 3 ++- src/sqlancer/tidb/TiDBOptions.java | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index e40488fc9..5c26cb03b 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -18,7 +18,8 @@ import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; import sqlancer.common.oracle.TestOracle; -@Parameters(separators = "=", commandDescription = "ClickHouse (default port: 8123, default host: localhost)") +@Parameters(separators = "=", commandDescription = "ClickHouse (default port: " + ClickHouseOptions.DEFAULT_PORT + + ", default host: " + ClickHouseOptions.DEFAULT_HOST) public class ClickHouseOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 8123; diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 149df7f60..2af62ae69 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -23,7 +23,8 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; -@Parameters(separators = "=", commandDescription = "CockroachDB (default port: 26257, default host: localhost)") +@Parameters(separators = "=", commandDescription = "CockroachDB (default port: " + CockroachDBOptions.DEFAULT_PORT + + " default host: " + CockroachDBOptions.DEFAULT_HOST) public class CockroachDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 26257; diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index ba9d7d8b4..63e142aec 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -14,7 +14,8 @@ import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.oracle.MariaDBNoRECOracle; -@Parameters(separators = "=", commandDescription = "MariaDB (default port: 3306, default host: localhost)") +@Parameters(separators = "=", commandDescription = "MariaDB (default port: " + MariaDBOptions.DEFAULT_PORT + + ", default host: " + MariaDBOptions.DEFAULT_HOST) public class MariaDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 377926d0f..5540f00a8 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -14,7 +14,8 @@ import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; -@Parameters(separators = "=", commandDescription = "MySQL (default port: 3306, default host: localhost)") +@Parameters(separators = "=", commandDescription = "MySQL (default port: " + MySQLOptions.DEFAULT_PORT + + ", default host: " + MySQLOptions.DEFAULT_HOST) public class MySQLOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 1614ef516..6c0a97847 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -19,7 +19,8 @@ import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; -@Parameters(separators = "=", commandDescription = "PostgreSQL (default port: 5432, default host: localhost)") +@Parameters(separators = "=", commandDescription = "PostgreSQL (default port: " + PostgresOptions.DEFAULT_PORT + + ", default host: " + PostgresOptions.DEFAULT_HOST) public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index b405e525d..e5da34fe0 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -17,7 +17,8 @@ import sqlancer.tidb.oracle.TiDBTLPHavingOracle; import sqlancer.tidb.oracle.TiDBTLPWhereOracle; -@Parameters(separators = "=", commandDescription = "PostgreSQL (default port: 4000, default host: localhost)") +@Parameters(separators = "=", commandDescription = "TiDB (default port: " + TiDBOptions.DEFAULT_PORT + + ", default host: " + TiDBOptions.DEFAULT_HOST) public class TiDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 4000; From c520480e1082bca72d652a9f833d354b239c325f Mon Sep 17 00:00:00 2001 From: EthanDBer Date: Thu, 25 Mar 2021 10:50:19 +0800 Subject: [PATCH 0395/1284] add const support for the different DBMSs and use local variables --- .../clickhouse/ClickHouseProvider.java | 6 +- .../cockroachdb/CockroachDBProvider.java | 6 +- src/sqlancer/mariadb/MariaDBProvider.java | 12 +-- src/sqlancer/mysql/MySQLProvider.java | 12 +-- src/sqlancer/postgres/1 | 84 +++++++++++++++++++ src/sqlancer/tidb/TiDBProvider.java | 6 +- 6 files changed, 98 insertions(+), 28 deletions(-) create mode 100644 src/sqlancer/postgres/1 diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index eba287ac6..297369ded 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -22,8 +22,6 @@ import sqlancer.common.query.SQLQueryProvider; public class ClickHouseProvider extends SQLProviderAdapter { - protected String host; - protected int port; public ClickHouseProvider() { super(ClickHouseGlobalState.class, ClickHouseOptions.class); @@ -105,8 +103,8 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws Exception @Override public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQLException { - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); if (host == null) { host = ClickHouseOptions.DEFAULT_HOST; } diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 0093ccaac..8b675459e 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -35,8 +35,6 @@ import sqlancer.common.query.SQLQueryProvider; public class CockroachDBProvider extends SQLProviderAdapter { - protected String host; - protected int port; public CockroachDBProvider() { super(CockroachDBGlobalState.class, CockroachDBOptions.class); @@ -250,8 +248,8 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio @Override public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws SQLException { - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); if (host == null) { host = CockroachDBOptions.DEFAULT_HOST; } diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index b95e7d63e..dc0420fca 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -27,10 +27,6 @@ public class MariaDBProvider extends SQLProviderAdapter { public static final int MAX_EXPRESSION_DEPTH = 3; - protected String username; - protected String password; - protected String host; - protected int port; public MariaDBProvider() { super(MariaDBGlobalState.class, MariaDBOptions.class); @@ -171,10 +167,10 @@ public SQLConnection createDatabase(MariaDBGlobalState globalState) throws SQLEx globalState.getState().logStatement("DROP DATABASE IF EXISTS " + globalState.getDatabaseName()); globalState.getState().logStatement("CREATE DATABASE " + globalState.getDatabaseName()); globalState.getState().logStatement("USE " + globalState.getDatabaseName()); - username = globalState.getOptions().getUserName(); - password = globalState.getOptions().getPassword(); - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); if (host == null) { host = MariaDBOptions.DEFAULT_HOST; } diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 92b13839d..447325951 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -32,10 +32,6 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; public class MySQLProvider extends SQLProviderAdapter { - protected String username; - protected String password; - protected String host; - protected int port; public MySQLProvider() { super(MySQLGlobalState.class, MySQLOptions.class); @@ -154,10 +150,10 @@ public void generateDatabase(MySQLGlobalState globalState) throws Exception { @Override public SQLConnection createDatabase(MySQLGlobalState globalState) throws SQLException { - username = globalState.getOptions().getUserName(); - password = globalState.getOptions().getPassword(); - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); if (host == null) { host = MySQLOptions.DEFAULT_HOST; } diff --git a/src/sqlancer/postgres/1 b/src/sqlancer/postgres/1 new file mode 100644 index 000000000..e12eb799d --- /dev/null +++ b/src/sqlancer/postgres/1 @@ -0,0 +1,84 @@ +package sqlancer.postgres; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; +import sqlancer.postgres.oracle.PostgresNoRECOracle; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; + +@Parameters(separators = "=", commandDescription = "PostgreSQL (default port: " + PostgresOptions.DEFAULT_PORT +", default host: " + PostgresOptions.DEFAULT_HOST) +public class PostgresOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 5432; + + @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) + public boolean allowBulkInsert; + + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for PostgreSQL") + public List oracle = Arrays.asList(PostgresOracleFactory.QUERY_PARTITIONING); + + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) + public boolean testCollations = true; + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the PostgreSQL server", arity = 1) + public String connectionURL = String.format("postgresql://%s:%d/test", PostgresOptions.DEFAULT_HOST, + PostgresOptions.DEFAULT_PORT); + + public enum PostgresOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresNoRECOracle(globalState); + } + }, + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + HAVING { + + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + List oracles = new ArrayList<>(); + oracles.add(new PostgresTLPWhereOracle(globalState)); + oracles.add(new PostgresTLPHavingOracle(globalState)); + oracles.add(new PostgresTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + + } + + @Override + public List getTestOracleFactory() { + return oracle; + } + +} diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 32d5c684c..c744de3f2 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -29,8 +29,6 @@ import sqlancer.tidb.gen.TiDBViewGenerator; public class TiDBProvider extends SQLProviderAdapter { - protected String host; - protected int port; public TiDBProvider() { super(TiDBGlobalState.class, TiDBOptions.class); @@ -135,8 +133,8 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { @Override public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLException { - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); if (host == null) { host = TiDBOptions.DEFAULT_HOST; } From 318272f752ee06d582a026d1935ee1b528b85f6f Mon Sep 17 00:00:00 2001 From: EthanDBer <13810907+EthanDBer@users.noreply.github.com> Date: Thu, 25 Mar 2021 10:57:17 +0800 Subject: [PATCH 0396/1284] Delete 1 --- src/sqlancer/postgres/1 | 84 ----------------------------------------- 1 file changed, 84 deletions(-) delete mode 100644 src/sqlancer/postgres/1 diff --git a/src/sqlancer/postgres/1 b/src/sqlancer/postgres/1 deleted file mode 100644 index e12eb799d..000000000 --- a/src/sqlancer/postgres/1 +++ /dev/null @@ -1,84 +0,0 @@ -package sqlancer.postgres; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; -import sqlancer.postgres.oracle.PostgresNoRECOracle; -import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; - -@Parameters(separators = "=", commandDescription = "PostgreSQL (default port: " + PostgresOptions.DEFAULT_PORT +", default host: " + PostgresOptions.DEFAULT_HOST) -public class PostgresOptions implements DBMSSpecificOptions { - public static final String DEFAULT_HOST = "localhost"; - public static final int DEFAULT_PORT = 5432; - - @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) - public boolean allowBulkInsert; - - @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for PostgreSQL") - public List oracle = Arrays.asList(PostgresOracleFactory.QUERY_PARTITIONING); - - @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) - public boolean testCollations = true; - - @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the PostgreSQL server", arity = 1) - public String connectionURL = String.format("postgresql://%s:%d/test", PostgresOptions.DEFAULT_HOST, - PostgresOptions.DEFAULT_PORT); - - public enum PostgresOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresNoRECOracle(globalState); - } - }, - PQS { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresPivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - HAVING { - - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); - oracles.add(new PostgresTLPWhereOracle(globalState)); - oracles.add(new PostgresTLPHavingOracle(globalState)); - oracles.add(new PostgresTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }; - - } - - @Override - public List getTestOracleFactory() { - return oracle; - } - -} From 4e9eaeaaee3e2aae23ec17514fa236a13e681cdd Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 26 Mar 2021 20:11:33 +0300 Subject: [PATCH 0397/1284] disable check for found bug until it is fixed --- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index a340ac036..6b24d091d 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -30,6 +30,8 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add(" is not under aggregate function and not in GROUP BY"); errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () errors.add("OptimizedRegularExpression: cannot compile re2"); + errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 + errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation } public static void addExpressionHavingErrors(ExpectedErrors errors) { From bd8185ad26372e4534f3098d422265ec3ee6801e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 27 Mar 2021 23:00:02 +0100 Subject: [PATCH 0398/1284] Add a new H2 expected error for foreign constraints Consider the following test case: ```sql CREATE TABLE t0(c0 INT); CREATE TABLE t1(c0 INT, FOREIGN KEY(c0) REFERENCES t0(C0)); -- Constraint "PRIMARY KEY | UNIQUE (C0)" not found; ``` Due to a recent H2 change, the second `CREATE TABLE` statement now results in an error, while previously, it was executed successfully. --- src/sqlancer/h2/H2TableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java index 146919e8b..d0659983e 100644 --- a/src/sqlancer/h2/H2TableGenerator.java +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -102,6 +102,7 @@ public SQLQueryAdapter getQuery(H2GlobalState globalState) { } errors.add("are not comparable"); errors.add(" cannot be updatable by a referential constraint with"); // generated columns + errors.add("not found"); // Constraint "PRIMARY KEY | UNIQUE (C0)" not found; } sb.append(")"); return new SQLQueryAdapter(sb.toString(), errors, true); From e0b272dc818c48d353a3d8cba6fa70947114f919 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sun, 28 Mar 2021 17:06:06 +0200 Subject: [PATCH 0399/1284] Clean up and improve code for pull request See Pull request #319 on github.com/sqlancer for more information. --- .../arangodb/ArangoDBComparatorHelper.java | 2 +- src/sqlancer/arangodb/ArangoDBProvider.java | 5 +- .../arangodb/ArangoDBQueryAdapter.java | 1 + .../gen/ArangoDBCreateIndexGenerator.java | 7 +- .../visitor/ArangoDBToQueryVisitor.java | 1 + .../mongodb/MongoDBComparatorHelper.java | 2 +- src/sqlancer/mongodb/MongoDBConnection.java | 6 +- src/sqlancer/mongodb/MongoDBProvider.java | 2 +- src/sqlancer/mongodb/MongoDBSchema.java | 74 +++++-------------- .../gen/MongoDBMatchExpressionGenerator.java | 2 +- 10 files changed, 37 insertions(+), 65 deletions(-) diff --git a/src/sqlancer/arangodb/ArangoDBComparatorHelper.java b/src/sqlancer/arangodb/ArangoDBComparatorHelper.java index 70cc451fe..2a00a312d 100644 --- a/src/sqlancer/arangodb/ArangoDBComparatorHelper.java +++ b/src/sqlancer/arangodb/ArangoDBComparatorHelper.java @@ -27,10 +27,10 @@ public static List getResultSetAsDocumentList(ArangoDBSelectQuery result = query.getResultSet(); return result; } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); if (e instanceof IgnoreMeException) { throw e; } + Main.nrUnsuccessfulActions.addAndGet(1); if (e.getMessage() == null) { throw new AssertionError(query.getLogString(), e); } diff --git a/src/sqlancer/arangodb/ArangoDBProvider.java b/src/sqlancer/arangodb/ArangoDBProvider.java index fbf6f153e..d63d4a1cc 100644 --- a/src/sqlancer/arangodb/ArangoDBProvider.java +++ b/src/sqlancer/arangodb/ArangoDBProvider.java @@ -57,7 +57,7 @@ public static class ArangoDBGlobalState extends GlobalState schemaTables = new ArrayList<>(); - public synchronized void addTable(ArangoDBSchema.ArangoDBTable table) { + public void addTable(ArangoDBSchema.ArangoDBTable table) { schemaTables.add(table); } @@ -111,6 +111,9 @@ public ArangoDBConnection createDatabase(ArangoDBGlobalState globalState) throws ArangoDatabase database = arangoDB.db(globalState.getDatabaseName()); try { database.drop(); + // When the database does not exist, an ArangoDB exception is thrown. Since we are not sure + // if this is the first time the database is used, the simplest is dropping it and ignoring + // the exception. } catch (Exception ignored) { } diff --git a/src/sqlancer/arangodb/ArangoDBQueryAdapter.java b/src/sqlancer/arangodb/ArangoDBQueryAdapter.java index 2d1b77af0..34cdb3709 100644 --- a/src/sqlancer/arangodb/ArangoDBQueryAdapter.java +++ b/src/sqlancer/arangodb/ArangoDBQueryAdapter.java @@ -5,6 +5,7 @@ public abstract class ArangoDBQueryAdapter extends Query { @Override public String getQueryString() { + // Should not be called as it is used only in SQL dependent classes throw new UnsupportedOperationException(); } diff --git a/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java index cc66a3ef9..6a1b872da 100644 --- a/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java +++ b/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java @@ -11,11 +11,8 @@ private ArangoDBCreateIndexGenerator() { } public static ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { - ArangoDBSchema.ArangoDBColumn column = null; - while (column == null) { - ArangoDBSchema.ArangoDBTable randomTable = globalState.getSchema().getRandomTable(); - column = randomTable.getRandomColumn(); - } + ArangoDBSchema.ArangoDBTable randomTable = globalState.getSchema().getRandomTable(); + ArangoDBSchema.ArangoDBColumn column = randomTable.getRandomColumn(); return new ArangoDBCreateIndexQuery(column); } } diff --git a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java index 59f4ababa..f82995d5e 100644 --- a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java +++ b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java @@ -91,6 +91,7 @@ protected void visit(NewFunctionNode expression) { } ArangoDBComputedExpressionGenerator.ComputedFunction function = (ArangoDBComputedExpressionGenerator.ComputedFunction) expression .getFunc(); + // TODO: Support functions with a different number of arguments. if (function.getNrArgs() != 2) { throw new UnsupportedOperationException(); } diff --git a/src/sqlancer/mongodb/MongoDBComparatorHelper.java b/src/sqlancer/mongodb/MongoDBComparatorHelper.java index b03e24422..49b692645 100644 --- a/src/sqlancer/mongodb/MongoDBComparatorHelper.java +++ b/src/sqlancer/mongodb/MongoDBComparatorHelper.java @@ -27,10 +27,10 @@ public static List getResultSetAsDocumentList(MongoDBSelectQuery adapt result = adapter.getResultSet(); return result; } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); if (e instanceof IgnoreMeException) { throw e; } + Main.nrUnsuccessfulActions.addAndGet(1); if (e.getMessage() == null) { throw new AssertionError(adapter.getLogString(), e); } diff --git a/src/sqlancer/mongodb/MongoDBConnection.java b/src/sqlancer/mongodb/MongoDBConnection.java index c15a46be5..6971bd79c 100644 --- a/src/sqlancer/mongodb/MongoDBConnection.java +++ b/src/sqlancer/mongodb/MongoDBConnection.java @@ -1,5 +1,8 @@ package sqlancer.mongodb; +import org.bson.BsonDocument; +import org.bson.BsonString; + import com.mongodb.client.MongoClient; import com.mongodb.client.MongoDatabase; @@ -17,7 +20,8 @@ public MongoDBConnection(MongoClient client, MongoDatabase database) { @Override public String getDatabaseVersion() throws Exception { - return "4.4, Java API 4.1"; + return client.getDatabase("dbname").runCommand(new BsonDocument("buildinfo", new BsonString(""))).get("version") + .toString(); } @Override diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java index 4712b5d9d..5ff549d32 100644 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ b/src/sqlancer/mongodb/MongoDBProvider.java @@ -59,7 +59,7 @@ public static class MongoDBGlobalState extends GlobalState schemaTables = new ArrayList<>(); - public synchronized void addTable(MongoDBTable table) { + public void addTable(MongoDBTable table) { schemaTables.add(table); } diff --git a/src/sqlancer/mongodb/MongoDBSchema.java b/src/sqlancer/mongodb/MongoDBSchema.java index 4968ab9d1..e9a0afb99 100644 --- a/src/sqlancer/mongodb/MongoDBSchema.java +++ b/src/sqlancer/mongodb/MongoDBSchema.java @@ -1,7 +1,10 @@ package sqlancer.mongodb; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.bson.BsonType; @@ -17,67 +20,30 @@ public class MongoDBSchema extends AbstractSchema { - public enum MongoDBDataType implements HasBsonType { - INTEGER { - @Override - public BsonType getBsonType() { - return BsonType.INT32; - } - }, - STRING { - @Override - public BsonType getBsonType() { - return BsonType.STRING; - } - }, - BOOLEAN { - @Override - public BsonType getBsonType() { - return BsonType.BOOLEAN; - } - }, - DOUBLE { - @Override - public BsonType getBsonType() { - return BsonType.DOUBLE; - } - }, - DATE_TIME { - @Override - public BsonType getBsonType() { - return BsonType.DATE_TIME; - } - }, - TIMESTAMP { - @Override - public BsonType getBsonType() { - return BsonType.TIMESTAMP; - } - }; + public enum MongoDBDataType { + INTEGER(BsonType.INT32), STRING(BsonType.STRING), BOOLEAN(BsonType.BOOLEAN), DOUBLE(BsonType.DOUBLE), + DATE_TIME(BsonType.DATE_TIME), TIMESTAMP(BsonType.TIMESTAMP); + + private final BsonType bsonType; + + MongoDBDataType(BsonType type) { + this.bsonType = type; + } + + public BsonType getBsonType() { + return bsonType; + } public static MongoDBDataType getRandom(MongoDBGlobalState state) { - MongoDBDataType[] valuesWithoutString; + Set valueSet = new HashSet<>(Arrays.asList(values())); if (state.getDmbsSpecificOptions().nullSafety) { - valuesWithoutString = new MongoDBDataType[values().length - 1]; - } else { - valuesWithoutString = new MongoDBDataType[values().length]; + valueSet.remove(STRING); } - int i = 0; - for (MongoDBDataType type : values()) { - if (type.equals(STRING) && state.getDmbsSpecificOptions().nullSafety) { - continue; - } - valuesWithoutString[i] = type; - i++; - } - return Randomly.fromOptions(valuesWithoutString); + MongoDBDataType[] configuredValues = new MongoDBDataType[valueSet.size()]; + return Randomly.fromOptions(valueSet.toArray(configuredValues)); } } - public interface HasBsonType { - BsonType getBsonType(); - } - public static class MongoDBColumn extends AbstractTableColumn { private final boolean isId; diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java index 191d0195e..0645d7f24 100644 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java @@ -105,7 +105,7 @@ public Node generateConstant(MongoDBDataType type) { private String getRandomizedRegexOptions() { List s = Randomly.subset("i", "m", "x", "s"); - return s.stream().reduce("", (current, newVal) -> current + newVal); + return String.join("", s); } @Override From 690ce67913e618aa6872abd64bac936b1a9fcc22 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 27 Mar 2021 18:46:38 +0100 Subject: [PATCH 0400/1284] Downgrade the OS used by MySQL and MariaDB from ubuntu-latest to 18.04 --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8059af59b..aca2c6497 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -161,7 +161,7 @@ jobs: mariadb: name: DBMS Tests (MariaDB) - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 @@ -187,7 +187,7 @@ jobs: mysql: name: DBMS Tests (MySQL) - runs-on: ubuntu-latest + runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 with: From 3000ed13ba40eb090aceea3487589c4024fd646c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 30 Mar 2021 21:23:22 +0000 Subject: [PATCH 0401/1284] Bump spotbugs-maven-plugin from 4.2.0 to 4.2.2 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.2.0 to 4.2.2. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/commits) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2b8783fb8..be5821802 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.2.0 + 4.2.2 spotbugs From 43a37772fbf391e8317a138c0e06e1b235867359 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 30 Mar 2021 23:44:43 +0200 Subject: [PATCH 0402/1284] Temporarily disable the CockroachDB tests See https://github.com/cockroachdb/cockroach/issues/62281. --- .github/workflows/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aca2c6497..91a96365b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -115,8 +115,6 @@ jobs: sleep 10 - name: Create SQLancer user run: cd cockroach-v20.2.6.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - - name: Run Tests - run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test duckdb: name: DBMS Tests (DuckDB) From 34a39b133a7ea4c6440ad57f90cf4bdb2c5ec1d9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 31 Mar 2021 00:10:39 +0200 Subject: [PATCH 0403/1284] Ignore DuckDB error when converting strings with large floating-point numbers --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + src/sqlancer/duckdb/gen/DuckDBTableGenerator.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 8e2e7b698..0cd9f52e3 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -50,6 +50,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { // removed column errors.add("Contents of view were altered: types don't match!"); errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + errors.add("Could not convert string"); // Could not convert string '1.7976931348623157E308' to DOUBLE } private static void addRegexErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index b10151258..de48e5c0f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -58,6 +58,7 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); + DuckDBErrors.addExpressionErrors(errors); sb.append(")"); } } From ced9a843491468e0cb5ac612fe624fbe97ab2865 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 30 Mar 2021 22:30:52 +0000 Subject: [PATCH 0404/1284] Bump postgresql from 42.2.18 to 42.2.19 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.18 to 42.2.19. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/REL42.2.19/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.18...REL42.2.19) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index be5821802..c149f904b 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ org.postgresql postgresql - 42.2.18 + 42.2.19 org.xerial From 25c96091c4b0272769e579b4b3ddc5b6fcb360d2 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 03:31:35 +0000 Subject: [PATCH 0405/1284] Bump junit-jupiter-engine from 5.7.0 to 5.7.1 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.7.0 to 5.7.1. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.7.0...r5.7.1) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c149f904b..496113256 100644 --- a/pom.xml +++ b/pom.xml @@ -248,7 +248,7 @@ org.junit.jupiter junit-jupiter-engine - 5.7.0 + 5.7.1 test From 0b7b40687d1f1cba625c823fcaac285d96dbcdbe Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 08:34:28 +0000 Subject: [PATCH 0406/1284] Bump jcommander from 1.78 to 1.81 Bumps [jcommander](https://github.com/cbeust/jcommander) from 1.78 to 1.81. - [Release notes](https://github.com/cbeust/jcommander/releases) - [Changelog](https://github.com/cbeust/jcommander/blob/master/CHANGELOG.md) - [Commits](https://github.com/cbeust/jcommander/compare/1.78...1.81) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 496113256..21d607cf4 100644 --- a/pom.xml +++ b/pom.xml @@ -218,7 +218,7 @@ com.beust jcommander - 1.78 + 1.81 org.postgresql From 55e737ae49690395b49d45d8d4f2fc1c971cc379 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 08:58:19 +0000 Subject: [PATCH 0407/1284] Bump ecj from 3.24.0 to 3.25.0 Bumps ecj from 3.24.0 to 3.25.0. Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 21d607cf4..59846fa48 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ org.eclipse.jdt ecj - 3.24.0 + 3.25.0 From 56dc64618cc64bd9af6e16d9d40d12fb90dc3b90 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 09:18:31 +0000 Subject: [PATCH 0408/1284] Bump duckdb_jdbc from 0.2.4 to 0.2.5 Bumps [duckdb_jdbc](https://github.com/cwida/duckdb) from 0.2.4 to 0.2.5. - [Release notes](https://github.com/cwida/duckdb/releases) - [Commits](https://github.com/cwida/duckdb/compare/v0.2.4...v0.2.5) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 59846fa48..ba6022d21 100644 --- a/pom.xml +++ b/pom.xml @@ -243,7 +243,7 @@ org.duckdb duckdb_jdbc - 0.2.4 + 0.2.5 org.junit.jupiter From 780137dd7882565832b19c78ae1aa00fed33a624 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 09:51:40 +0000 Subject: [PATCH 0409/1284] Bump formatter-maven-plugin from 2.13.0 to 2.14.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.13.0 to 2.14.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.13.0...formatter-maven-plugin-2.14.0) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ba6022d21..cf197f52c 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.13.0 + 2.14.0 eclipseformat From 1828a825413b1d3485b000557a1249af532f4591 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 6 Apr 2021 17:55:33 +0300 Subject: [PATCH 0410/1284] Update ClickHouse JDBC to 0.3.0 --- pom.xml | 2 +- src/sqlancer/clickhouse/ClickHouseErrors.java | 5 +- .../clickhouse/ast/ClickHouseConstant.java | 500 ++++++++++++++++++ 3 files changed, 504 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index cf197f52c..b74afcb72 100644 --- a/pom.xml +++ b/pom.xml @@ -259,7 +259,7 @@ ru.yandex.clickhouse clickhouse-jdbc - 0.2.5 + 0.3.0 com.h2database diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 6b24d091d..e7f3f8612 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -11,8 +11,6 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Illegal type"); errors.add("Argument at index 1 for function like must be constant"); errors.add("Argument at index 1 for function notLike must be constant"); - errors.add( - "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); errors.add("does not return a value of type UInt8"); errors.add("invalid escape sequence"); errors.add("invalid character class range"); @@ -23,6 +21,8 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Cannot read floating point value"); errors.add("Cannot parse infinity."); errors.add("Attempt to read after eof: while converting"); + errors.add( + "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database errors.add("is not under aggregate function"); errors.add("Invalid type for filter in"); @@ -32,6 +32,7 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("OptimizedRegularExpression: cannot compile re2"); errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation + errors.add("Cannot parse NaN.: while converting 'N' to Float64"); // https://github.com/ClickHouse/ClickHouse/issues/22710 } public static void addExpressionHavingErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java index b70b0841f..048c14296 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java @@ -128,6 +128,14 @@ public ClickHouseConstant cast(ClickHouseDataType type) { return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); case Int64: return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); + case UInt128: + return ClickHouseConstant.createUInt128Constant(BigInteger.valueOf(value)); + case Int128: + return ClickHouseConstant.createInt128Constant(BigInteger.valueOf(value)); + case UInt256: + return ClickHouseConstant.createUInt256Constant(BigInteger.valueOf(value)); + case Int256: + return ClickHouseConstant.createInt256Constant(BigInteger.valueOf(value)); case Float32: return ClickHouseConstant.createFloat32Constant((float) value); case Float64: @@ -921,6 +929,474 @@ public ClickHouseConstant cast(ClickHouseDataType type) { } } + public static class ClickHouseUInt128Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt128Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt128; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() + : ClickHouseConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseConstant.createUInt8Constant(val); + case Int8: + return ClickHouseConstant.createInt8Constant(val); + case UInt16: + return ClickHouseConstant.createUInt16Constant(val); + case Int16: + return ClickHouseConstant.createInt16Constant(val); + case UInt32: + return ClickHouseConstant.createUInt32Constant(val); + case Int32: + return ClickHouseConstant.createInt32Constant(val); + case UInt64: + return ClickHouseConstant.createUInt64Constant(value); + case Int64: + return ClickHouseConstant.createInt64Constant(value); + case UInt128: + return ClickHouseConstant.createUInt128Constant(value); + case Int128: + return ClickHouseConstant.createInt128Constant(value); + case UInt256: + return ClickHouseConstant.createUInt256Constant(value); + case Int256: + return ClickHouseConstant.createInt256Constant(value); + case Float32: + return ClickHouseConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseConstant.createNullConstant(); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case DateTime: + case Enum8: + case Enum16: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal: + case UUID: + case FixedString: + case Nested: + case Tuple: + case Array: + case AggregateFunction: + case Unknown: + default: + throw new AssertionError(type); + } + } + } + + public static class ClickHouseInt128Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt128Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int128; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() + : ClickHouseConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseConstant.createUInt8Constant(val); + case Int8: + return ClickHouseConstant.createInt8Constant(val); + case UInt16: + return ClickHouseConstant.createUInt16Constant(val); + case Int16: + return ClickHouseConstant.createInt16Constant(val); + case UInt32: + return ClickHouseConstant.createUInt32Constant(val); + case Int32: + return ClickHouseConstant.createInt32Constant(val); + case UInt64: + return ClickHouseConstant.createUInt64Constant(value); + case Int64: + return ClickHouseConstant.createInt64Constant(value); + case UInt128: + return ClickHouseConstant.createUInt128Constant(value); + case Int128: + return ClickHouseConstant.createInt128Constant(value); + case UInt256: + return ClickHouseConstant.createUInt256Constant(value); + case Int256: + return ClickHouseConstant.createInt256Constant(value); + case Float32: + return ClickHouseConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseConstant.createNullConstant(); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case DateTime: + case Enum8: + case Enum16: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal: + case UUID: + case FixedString: + case Nested: + case Tuple: + case Array: + case AggregateFunction: + case Unknown: + default: + throw new AssertionError(type); + } + } + } + + public static class ClickHouseUInt256Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt256Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt256; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() + : ClickHouseConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseConstant.createUInt8Constant(val); + case Int8: + return ClickHouseConstant.createInt8Constant(val); + case UInt16: + return ClickHouseConstant.createUInt16Constant(val); + case Int16: + return ClickHouseConstant.createInt16Constant(val); + case UInt32: + return ClickHouseConstant.createUInt32Constant(val); + case Int32: + return ClickHouseConstant.createInt32Constant(val); + case UInt64: + return ClickHouseConstant.createUInt64Constant(value); + case Int64: + return ClickHouseConstant.createInt64Constant(value); + case UInt128: + return ClickHouseConstant.createUInt128Constant(value); + case Int128: + return ClickHouseConstant.createInt128Constant(value); + case UInt256: + return ClickHouseConstant.createUInt256Constant(value); + case Int256: + return ClickHouseConstant.createInt256Constant(value); + case Float32: + return ClickHouseConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseConstant.createNullConstant(); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case DateTime: + case Enum8: + case Enum16: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal: + case UUID: + case FixedString: + case Nested: + case Tuple: + case Array: + case AggregateFunction: + case Unknown: + default: + throw new AssertionError(type); + } + } + } + + public static class ClickHouseInt256Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt256Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int256; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() + : ClickHouseConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseConstant.createUInt8Constant(val); + case Int8: + return ClickHouseConstant.createInt8Constant(val); + case UInt16: + return ClickHouseConstant.createUInt16Constant(val); + case Int16: + return ClickHouseConstant.createInt16Constant(val); + case UInt32: + return ClickHouseConstant.createUInt32Constant(val); + case Int32: + return ClickHouseConstant.createInt32Constant(val); + case UInt64: + return ClickHouseConstant.createUInt64Constant(value); + case Int64: + return ClickHouseConstant.createInt64Constant(value); + case UInt128: + return ClickHouseConstant.createUInt128Constant(value); + case Int128: + return ClickHouseConstant.createInt128Constant(value); + case UInt256: + return ClickHouseConstant.createUInt256Constant(value); + case Int256: + return ClickHouseConstant.createInt256Constant(value); + case Float32: + return ClickHouseConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseConstant.createNullConstant(); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case DateTime: + case Enum8: + case Enum16: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal: + case UUID: + case FixedString: + case Nested: + case Tuple: + case Array: + case AggregateFunction: + case Unknown: + default: + throw new AssertionError(type); + } + } + } + public static class ClickHouseFloat32Constant extends ClickHouseConstant { private final float value; @@ -1284,6 +1760,10 @@ public static ClickHouseConstant createIntConstant(ClickHouseDataType type, long break; case IntervalSecond: break; + case UInt256: + return createUInt256Constant(BigInteger.valueOf(val)); + case UInt128: + return createUInt128Constant(BigInteger.valueOf(val)); case UInt64: return createUInt64Constant(BigInteger.valueOf(val)); case UInt32: @@ -1292,6 +1772,10 @@ public static ClickHouseConstant createIntConstant(ClickHouseDataType type, long return createUInt16Constant(val); case UInt8: return createUInt8Constant(val); + case Int256: + return createInt256Constant(BigInteger.valueOf(val)); + case Int128: + return createInt256Constant(BigInteger.valueOf(val)); case Int64: return createInt64Constant(BigInteger.valueOf(val)); case Int32: @@ -1344,6 +1828,22 @@ public static ClickHouseConstant createIntConstant(ClickHouseDataType type, long throw new AssertionError(type); } + public static ClickHouseConstant createInt256Constant(BigInteger val) { + return new ClickHouseInt256Constant(val); + } + + public static ClickHouseConstant createUInt256Constant(BigInteger val) { + return new ClickHouseUInt256Constant(val); + } + + public static ClickHouseConstant createInt128Constant(BigInteger val) { + return new ClickHouseInt128Constant(val); + } + + public static ClickHouseConstant createUInt128Constant(BigInteger val) { + return new ClickHouseUInt128Constant(val); + } + public static ClickHouseConstant createInt64Constant(BigInteger val) { return new ClickHouseInt64Constant(val); } From 991f841f8f079e1748ab1227cb4f97834bb4e6ec Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 24 Apr 2021 05:32:05 +0300 Subject: [PATCH 0411/1284] Update ClickHouseErrors.java Example: https://clickhouse-test-reports.s3.yandex.net/23561/e9e4ab3813a9a2321ca126c4f313780e5b73abc1/sqlancer_test/TLPHaving.err --- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index e7f3f8612..b69492e85 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -32,7 +32,7 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("OptimizedRegularExpression: cannot compile re2"); errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation - errors.add("Cannot parse NaN.: while converting 'N' to Float64"); // https://github.com/ClickHouse/ClickHouse/issues/22710 + errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 } public static void addExpressionHavingErrors(ExpectedErrors errors) { From de3e1e59020660999a5ebebf1a1370a24f37339a Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 26 Apr 2021 13:49:55 -0700 Subject: [PATCH 0412/1284] Update CockroachDB to v20.2.8 and enable tests CockroachDB tests were disabled after they revealed a bug in v20.2.6, https://github.com/cockroachdb/cockroach/issues/62281. This bug was fixed and v20.2.8. This commit updates the CockroachDB version and enables the tests. --- .github/workflows/main.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91a96365b..61458f894 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,11 +110,13 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.2.6.linux-amd64.tgz | tar xvz - cd cockroach-v20.2.6.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v20.2.8.linux-amd64.tgz | tar xvz + cd cockroach-v20.2.8.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v20.2.6.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v20.2.8.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + - name: Run Tests + run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test duckdb: name: DBMS Tests (DuckDB) From c59b0d69586ab38949481e381e769da42c81a211 Mon Sep 17 00:00:00 2001 From: wanliyi <2608953482@qq.com> Date: Mon, 26 Apr 2021 14:19:49 +0800 Subject: [PATCH 0413/1284] add more tidb error merssage add more tidb error merssage format checkstyle pass Added some expected TiDB error messages format Update CockroachDB to v20.2.8 and enable tests CockroachDB tests were disabled after they revealed a bug in v20.2.6, https://github.com/cockroachdb/cockroach/issues/62281. This bug was fixed and v20.2.8. This commit updates the CockroachDB version and enables the tests. Update CockroachDB to v20.2.8 and enable tests CockroachDB tests were disabled after they revealed a bug in v20.2.6, https://github.com/cockroachdb/cockroach/issues/62281. This bug was fixed and v20.2.8. This commit updates the CockroachDB version and enables the tests. --- src/sqlancer/tidb/TiDBErrors.java | 3 +++ src/sqlancer/tidb/TiDBSchema.java | 3 +++ .../tidb/gen/TiDBAlterTableGenerator.java | 19 +++++++++++++++---- .../tidb/gen/TiDBAnalyzeTableGenerator.java | 13 +++++++------ .../tidb/gen/TiDBDeleteGenerator.java | 2 ++ 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index fa09f11a9..bb3d3ca62 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -22,6 +22,8 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("BIGINT value is out of range"); errors.add("doesn't have a default value"); // default + errors.add("is not valid for CHARACTER SET"); + // known issue: https://github.com/pingcap/tidb/issues/14819 errors.add("Wrong plan type for dataReaderBuilder"); @@ -69,6 +71,7 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Incorrect bigint value"); errors.add("Incorrect decimal value"); errors.add("error parsing regexp"); + errors.add("is not valid for CHARACTER SET"); if (true) { // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/54 diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 4d04369c2..a611543fc 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -196,7 +196,9 @@ private static TiDBCompositeDataType getColumnType(String typeString) { } else { switch (trimmedStringType) { case "text": + case "mediumtext": case "longtext": + case "tinytext": primitiveType = TiDBDataType.TEXT; break; case "float": @@ -223,6 +225,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { break; case "blob": case "longblob": + case "tinyblob": primitiveType = TiDBDataType.BLOB; break; default: diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index e6ebe4dac..3d33a4dbc 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -48,7 +48,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { throw new IgnoreMeException(); } sb.append(column.getName()); - errors.add("with index covered now"); + errors.add("with composite index covered or Primary Key covered now"); errors.add("Unsupported drop integer primary key"); errors.add("has a generated column dependency"); errors.add( @@ -67,15 +67,27 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { } errors.add("Unsupported drop integer primary key"); errors.add("Unsupported drop primary key when alter-primary-key is false"); + errors.add("Unsupported drop primary key when the table's pkIsHandle is true"); + errors.add("Incorrect table definition; there can be only one auto column and it must be defined as a key"); sb.append(" DROP PRIMARY KEY"); break; case ADD_PRIMARY_KEY: sb.append("ADD PRIMARY KEY("); - sb.append(table.getRandomNonEmptyColumnSubset().stream().map(c -> c.getName()) - .collect(Collectors.joining(", "))); + sb.append(table.getRandomNonEmptyColumnSubset().stream().map(c -> { + StringBuilder colName = new StringBuilder(c.getName()); + if (c.getType().getPrimitiveDataType() == TiDBDataType.TEXT + || c.getType().getPrimitiveDataType() == TiDBDataType.BLOB) { + TiDBTableGenerator.appendSpecifiers(colName, c.getType().getPrimitiveDataType()); + } + return colName; + }).collect(Collectors.joining(", "))); sb.append(")"); errors.add("Unsupported add primary key, alter-primary-key is false"); errors.add("Information schema is changed during the execution of the statement"); + errors.add("Multiple primary key defined"); + errors.add("Invalid use of NULL value"); + errors.add("Duplicate entry"); + errors.add("'Defining a virtual generated column as primary key' is not supported for generated columns"); break; case CHANGE: if (TiDBBugs.bug10) { @@ -101,7 +113,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { default: throw new AssertionError(a); } - return new SQLQueryAdapter(sb.toString(), errors, true); } diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 85525b6ff..b52e2cc25 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -31,12 +31,13 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(Randomly.getNotCachedInteger(1, 1024)); sb.append(" BUCKETS"); } - return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("https://github.com/pingcap/tidb/issues/15993", - /* https://github.com/pingcap/tidb/issues/15993 */ "doesn't have a default value" /* - * https://github. - * com/pingcap/tidb/ - * issues/15993 - */)); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("https://github.com/pingcap/tidb/issues/15993", + /* https://github.com/pingcap/tidb/issues/15993 */ "doesn't have a default value", + "Index 'PRIMARY' in field list does not exist in table" /* + * https://github. com/pingcap/tidb/ + * issues/15993 + */)); } } diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index 6c194a010..f6eea4671 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -51,6 +51,8 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); } errors.add("Bad Number"); + errors.add("Truncated incorrect"); // https://github.com/pingcap/tidb/issues/24292 + errors.add("is not valid for CHARACTER SET"); errors.add("Division by 0"); errors.add("error parsing regexp"); return new SQLQueryAdapter(sb.toString(), errors); From e76ad8458e513f152e952c9788a6c9606cb6d5b0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 27 Apr 2021 19:33:31 +0000 Subject: [PATCH 0414/1284] Bump formatter-maven-plugin from 2.14.0 to 2.15.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.14.0 to 2.15.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.14.0...formatter-maven-plugin-2.15.0) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b74afcb72..7749ca4bf 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.14.0 + 2.15.0 eclipseformat From 7fa358c5f35f20065751367ce63d14af333a5a45 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 27 Apr 2021 19:46:44 +0000 Subject: [PATCH 0415/1284] Bump postgresql from 42.2.19 to 42.2.20 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.19 to 42.2.20. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/REL42.2.20/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.19...REL42.2.20) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7749ca4bf..735df29d1 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ org.postgresql postgresql - 42.2.19 + 42.2.20 org.xerial From 443152f3c2a824b3c414ebc147194a8537435aef Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 27 Apr 2021 20:34:29 +0000 Subject: [PATCH 0416/1284] Bump maven-jxr-plugin from 3.0.0 to 3.1.1 Bumps [maven-jxr-plugin](https://github.com/apache/maven-jxr) from 3.0.0 to 3.1.1. - [Release notes](https://github.com/apache/maven-jxr/releases) - [Commits](https://github.com/apache/maven-jxr/compare/jxr-3.0.0...jxr-3.1.1) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 735df29d1..6a6293652 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.apache.maven.plugins maven-jxr-plugin - 3.0.0 + 3.1.1 From 53ac0b438261f8e7d466c3cb4a124e864232e09e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 28 Apr 2021 14:53:51 +0000 Subject: [PATCH 0417/1284] Bump clickhouse-jdbc from 0.3.0 to 0.3.1 Bumps [clickhouse-jdbc](https://github.com/ClickHouse/clickhouse-jdbc) from 0.3.0 to 0.3.1. - [Release notes](https://github.com/ClickHouse/clickhouse-jdbc/releases) - [Changelog](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/ClickHouse/clickhouse-jdbc/compare/v0.3.0...v0.3.1) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6a6293652..0ae278c6b 100644 --- a/pom.xml +++ b/pom.xml @@ -259,7 +259,7 @@ ru.yandex.clickhouse clickhouse-jdbc - 0.3.0 + 0.3.1 com.h2database From 9d632203e767e10495e6d8a1c180710a7073ee77 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 28 Apr 2021 15:15:52 +0000 Subject: [PATCH 0418/1284] Bump mysql-connector-java from 8.0.23 to 8.0.24 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.23 to 8.0.24. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/commits) Signed-off-by: dependabot-preview[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0ae278c6b..d4dc4254d 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ mysql mysql-connector-java - 8.0.23 + 8.0.24 org.mariadb.jdbc From 8b9ac9e548d6589a88867daa8600cbbd7961822f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 29 Apr 2021 15:43:45 +0000 Subject: [PATCH 0419/1284] Upgrade to GitHub-native Dependabot --- .github/dependabot.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..d8484d5ba --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,23 @@ +version: 2 +updates: +- package-ecosystem: maven + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + ignore: + - dependency-name: com.arangodb:arangodb-java-driver + versions: + - 6.10.0 + - 6.11.0 + - 6.11.1 + - dependency-name: org.mongodb:mongodb-driver-sync + versions: + - 4.2.2 + - dependency-name: ru.yandex.clickhouse:clickhouse-jdbc + versions: + - 0.2.6 + - 0.3.0 + - dependency-name: org.postgresql:postgresql + versions: + - 42.2.19 From 793899e1b8b12821770dad9b6fefe6ad70ad3e15 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 3 May 2021 14:25:00 +0200 Subject: [PATCH 0420/1284] Remove the dependabot ignores --- .github/dependabot.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d8484d5ba..a217b347e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,19 +5,3 @@ updates: schedule: interval: daily open-pull-requests-limit: 10 - ignore: - - dependency-name: com.arangodb:arangodb-java-driver - versions: - - 6.10.0 - - 6.11.0 - - 6.11.1 - - dependency-name: org.mongodb:mongodb-driver-sync - versions: - - 4.2.2 - - dependency-name: ru.yandex.clickhouse:clickhouse-jdbc - versions: - - 0.2.6 - - 0.3.0 - - dependency-name: org.postgresql:postgresql - versions: - - 42.2.19 From 75b209edd2db61fb7a5ec0dc392ec38c15615628 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 May 2021 08:22:55 +0000 Subject: [PATCH 0421/1284] Bump jacoco-maven-plugin from 0.8.6 to 0.8.7 Bumps [jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.6 to 0.8.7. - [Release notes](https://github.com/jacoco/jacoco/releases) - [Commits](https://github.com/jacoco/jacoco/compare/v0.8.6...v0.8.7) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d4dc4254d..5b4d1147d 100644 --- a/pom.xml +++ b/pom.xml @@ -49,7 +49,7 @@ org.jacoco jacoco-maven-plugin - 0.8.6 + 0.8.7 From ce238bdc972c640dd45670d18c3c20b3a1e9988b Mon Sep 17 00:00:00 2001 From: Patrick Date: Sat, 8 May 2021 16:26:21 +0200 Subject: [PATCH 0422/1284] Add config tests for ArangoDB With the configuration "withOptimizerRuleTests" it is now possible to test ArangoDB queries further. A randomly generated query is executed with all optimizer flags enabled and with a random number of optimizer flags disabled. --- src/sqlancer/arangodb/ArangoDBOptions.java | 5 +- .../query/ArangoDBOptimizerRules.java | 57 +++++++++++++++++++ .../arangodb/query/ArangoDBSelectQuery.java | 27 ++++++++- .../ArangoDBQueryPartitioningWhereTester.java | 12 +++- 4 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index cdb7ee759..ba06ceac3 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -25,6 +25,10 @@ public class ArangoDBOptions implements DBMSSpecificOptions getTestOracleFactory() { return oracles; @@ -39,6 +43,5 @@ public TestOracle create(ArangoDBProvider.ArangoDBGlobalState globalState) throw return new CompositeTestOracle(oracles, globalState); } } - } } diff --git a/src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java b/src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java new file mode 100644 index 000000000..835849b92 --- /dev/null +++ b/src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java @@ -0,0 +1,57 @@ +package sqlancer.arangodb.query; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; + +public class ArangoDBOptimizerRules { + + private final List allRules = new ArrayList<>(); + + public ArangoDBOptimizerRules() { + // SRC: + // https://www.arangodb.com/docs/stable/aql/execution-and-performance-optimizer.html#list-of-optimizer-rules + // Filtered out irrelevant ones + allRules.add("-fuse-filters"); + // allRules.add("-geo-index-optimizer"); + // allRules.add("-handle-arangosearch-views"); + // allRules.add("-inline-subqueries"); + allRules.add("-interchange-adjacent-enumerations"); + allRules.add("-late-document-materialization"); + // allRules.add("-late-document-materialization-arangosearch"); + allRules.add("-move-calculations-down"); + allRules.add("-move-calculations-up"); + allRules.add("-move-filters-into-enumerate"); + allRules.add("-move-filters-up"); + // allRules.add("-optimize-count"); + // allRules.add("-optimize-subqueries"); + // allRules.add("-optimize-traversals"); + // allRules.add("-patch-update-statements"); + allRules.add("-propagate-constant-attributes"); + allRules.add("-reduce-extraction-to-projection"); + // allRules.add("-remove-collect-variables"); + // allRules.add("-remove-data-modification-out-variables"); + allRules.add("-remove-filter-covered-by-index"); + // allRules.add("-remove-filter-covered-by-traversal"); + allRules.add("-remove-redundant-calculations"); + allRules.add("-remove-redundant-or"); + // allRules.add("-remove-redundant-path-var"); + // allRules.add("-remove-redundant-sorts"); + // allRules.add("-remove-sort-rand"); + allRules.add("-remove-unnecessary-calculations"); + allRules.add("-remove-unnecessary-filters"); + // allRules.add("-replace-function-with-index"); + allRules.add("-replace-or-with-in"); + allRules.add("-simplify-conditions"); + // allRules.add("-sort-in-values"); + // allRules.add("-sort-limit"); + // allRules.add("-splice-subqueries"); + // allRules.add("-use-index-for-sort"); + allRules.add("-use-indexes"); + } + + public List getRandomRules() { + return Randomly.subset(allRules); + } +} diff --git a/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java b/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java index 400585ca2..4725e4178 100644 --- a/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java +++ b/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java @@ -1,10 +1,13 @@ package sqlancer.arangodb.query; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import com.arangodb.ArangoCursor; import com.arangodb.entity.BaseDocument; +import com.arangodb.model.AqlQueryOptions; import sqlancer.GlobalState; import sqlancer.arangodb.ArangoDBConnection; @@ -16,10 +19,13 @@ public class ArangoDBSelectQuery extends ArangoDBQueryAdapter { private final String query; + private List optimizerRules; + private List resultSet; public ArangoDBSelectQuery(String query) { this.query = query; + optimizerRules = new ArrayList<>(); } @Override @@ -40,7 +46,12 @@ public ExpectedErrors getExpectedErrors() { @Override public String getLogString() { - return "db._query(\"" + query + "\")"; + if (optimizerRules.isEmpty()) { + return "db._query(\"" + query + "\")"; + } else { + String rules = optimizerRules.stream().map(Object::toString).collect(Collectors.joining("\",\"")); + return "db._query(\"" + query + "\", null, { optimizer: { rules: [\"" + rules + "\"] } } )"; + } } @Override @@ -54,7 +65,15 @@ public String getLogString() { e.printStackTrace(); } } - ArangoCursor cursor = globalState.getConnection().getDatabase().query(query, BaseDocument.class); + + ArangoCursor cursor; + if (optimizerRules.isEmpty()) { + cursor = globalState.getConnection().getDatabase().query(query, BaseDocument.class); + } else { + AqlQueryOptions options = new AqlQueryOptions(); + cursor = globalState.getConnection().getDatabase().query(query, options.rules(optimizerRules), + BaseDocument.class); + } resultSet = cursor.asListRemaining(); return null; } @@ -62,4 +81,8 @@ public String getLogString() { public List getResultSet() { return resultSet; } + + public void excludeRandomOptRules() { + optimizerRules = new ArangoDBOptimizerRules().getRandomRules(); + } } diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java index 80b7d46bf..4484da8f7 100644 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java @@ -32,7 +32,15 @@ public void check() throws Exception { query = ArangoDBVisitor.asSelectQuery(select); List thirdResultSet = getResultSetAsDocumentList(query, state); - secondResultSet.addAll(thirdResultSet); - assumeResultSetsAreEqual(firstResultSet, secondResultSet, query); + thirdResultSet.addAll(secondResultSet); + assumeResultSetsAreEqual(firstResultSet, thirdResultSet, query); + + if (state.getDmbsSpecificOptions().withOptimizerRuleTests) { + select.setFilterClause(predicate); + query = ArangoDBVisitor.asSelectQuery(select); + query.excludeRandomOptRules(); + List forthResultSet = getResultSetAsDocumentList(query, state); + assumeResultSetsAreEqual(secondResultSet, forthResultSet, query); + } } } From 45249787428833914c700c4b8dca7816d6e618a9 Mon Sep 17 00:00:00 2001 From: Patrick Date: Sat, 8 May 2021 13:16:30 +0200 Subject: [PATCH 0423/1284] Rename novel approach for MongoDB Rename Remove-Reduce to Result-oriented Document Removal --- ...eOracleBase.java => DocumentRemovalOracleBase.java} | 6 +++--- src/sqlancer/mongodb/MongoDBOptions.java | 10 +++++----- ...ReduceBase.java => MongoDBDocumentRemovalBase.java} | 10 +++++----- ...ceTester.java => MongoDBDocumentRemovalTester.java} | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) rename src/sqlancer/common/oracle/{RemoveReduceOracleBase.java => DocumentRemovalOracleBase.java} (71%) rename src/sqlancer/mongodb/test/{MongoDBRemoveReduceBase.java => MongoDBDocumentRemovalBase.java} (90%) rename src/sqlancer/mongodb/test/{MongoDBRemoveReduceTester.java => MongoDBDocumentRemovalTester.java} (90%) diff --git a/src/sqlancer/common/oracle/RemoveReduceOracleBase.java b/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java similarity index 71% rename from src/sqlancer/common/oracle/RemoveReduceOracleBase.java rename to src/sqlancer/common/oracle/DocumentRemovalOracleBase.java index c177f072e..a48a6f17e 100644 --- a/src/sqlancer/common/oracle/RemoveReduceOracleBase.java +++ b/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java @@ -3,17 +3,17 @@ import sqlancer.GlobalState; import sqlancer.common.gen.ExpressionGenerator; -public abstract class RemoveReduceOracleBase> implements TestOracle { +public abstract class DocumentRemovalOracleBase> implements TestOracle { protected E predicate; protected final S state; - protected RemoveReduceOracleBase(S state) { + protected DocumentRemovalOracleBase(S state) { this.state = state; } - protected void initializeRemoveReduceOracle() { + protected void initializeDocumentRemovalOracle() { ExpressionGenerator gen = getGen(); if (gen == null) { throw new IllegalStateException(); diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index cf632085b..77c565a11 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -1,7 +1,7 @@ package sqlancer.mongodb; +import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.DOCUMENT_REMOVAL; import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.QUERY_PARTITIONING; -import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.REMOVE_REDUCE; import java.util.ArrayList; import java.util.Arrays; @@ -13,8 +13,8 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.mongodb.test.MongoDBDocumentRemovalTester; import sqlancer.mongodb.test.MongoDBQueryPartitioningWhereTester; -import sqlancer.mongodb.test.MongoDBRemoveReduceTester; public class MongoDBOptions implements DBMSSpecificOptions { @@ -43,7 +43,7 @@ public class MongoDBOptions implements DBMSSpecificOptions oracles = Arrays.asList(QUERY_PARTITIONING, REMOVE_REDUCE); + public List oracles = Arrays.asList(QUERY_PARTITIONING, DOCUMENT_REMOVAL); @Override public List getTestOracleFactory() { @@ -59,11 +59,11 @@ public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws return new CompositeTestOracle(oracles, globalState); } }, - REMOVE_REDUCE { + DOCUMENT_REMOVAL { @Override public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws Exception { List oracles = new ArrayList<>(); - oracles.add(new MongoDBRemoveReduceTester(globalState)); + oracles.add(new MongoDBDocumentRemovalTester(globalState)); return new CompositeTestOracle(oracles, globalState); } } diff --git a/src/sqlancer/mongodb/test/MongoDBRemoveReduceBase.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java similarity index 90% rename from src/sqlancer/mongodb/test/MongoDBRemoveReduceBase.java rename to src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java index 2a2a54744..e3e3df17c 100644 --- a/src/sqlancer/mongodb/test/MongoDBRemoveReduceBase.java +++ b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java @@ -6,7 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.RemoveReduceOracleBase; +import sqlancer.common.oracle.DocumentRemovalOracleBase; import sqlancer.common.oracle.TestOracle; import sqlancer.mongodb.MongoDBProvider; import sqlancer.mongodb.MongoDBSchema; @@ -15,8 +15,8 @@ import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator; import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; -public class MongoDBRemoveReduceBase extends - RemoveReduceOracleBase, MongoDBProvider.MongoDBGlobalState> implements TestOracle { +public class MongoDBDocumentRemovalBase extends + DocumentRemovalOracleBase, MongoDBProvider.MongoDBGlobalState> implements TestOracle { protected MongoDBSchema schema; protected MongoDBSchema.MongoDBTables targetTables; @@ -25,7 +25,7 @@ public class MongoDBRemoveReduceBase extends protected MongoDBMatchExpressionGenerator expressionGenerator; protected MongoDBSelect select; - protected MongoDBRemoveReduceBase(MongoDBProvider.MongoDBGlobalState state) { + protected MongoDBDocumentRemovalBase(MongoDBProvider.MongoDBGlobalState state) { super(state); } @@ -36,7 +36,7 @@ public void check() throws Exception { mainTable = targetTables.getTables().get(0); generateTargetColumns(); expressionGenerator = new MongoDBMatchExpressionGenerator(state).setColumns(targetColumns); - initializeRemoveReduceOracle(); + initializeDocumentRemovalOracle(); select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); select.setProjectionList(targetColumns); if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/mongodb/test/MongoDBRemoveReduceTester.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java similarity index 90% rename from src/sqlancer/mongodb/test/MongoDBRemoveReduceTester.java rename to src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java index 829f63f23..ece3193d1 100644 --- a/src/sqlancer/mongodb/test/MongoDBRemoveReduceTester.java +++ b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java @@ -13,8 +13,8 @@ import sqlancer.mongodb.query.MongoDBRemoveQuery; import sqlancer.mongodb.query.MongoDBSelectQuery; -public class MongoDBRemoveReduceTester extends MongoDBRemoveReduceBase { - public MongoDBRemoveReduceTester(MongoDBProvider.MongoDBGlobalState state) { +public class MongoDBDocumentRemovalTester extends MongoDBDocumentRemovalBase { + public MongoDBDocumentRemovalTester(MongoDBProvider.MongoDBGlobalState state) { super(state); } From f39982a82f992ff065190df9c2e9c8f13e0dc4d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 May 2021 07:01:49 +0000 Subject: [PATCH 0424/1284] Bump mariadb-java-client from 2.7.2 to 2.7.3 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.7.2 to 2.7.3. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/commits) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5b4d1147d..380324944 100644 --- a/pom.xml +++ b/pom.xml @@ -238,7 +238,7 @@ org.mariadb.jdbc mariadb-java-client - 2.7.2 + 2.7.3 org.duckdb From b253ba7bc2d31b9da939afbd74b8fc3ee4e81523 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 May 2021 08:30:38 +0000 Subject: [PATCH 0425/1284] Bump maven-gpg-plugin from 1.6 to 3.0.1 Bumps [maven-gpg-plugin](https://github.com/apache/maven-gpg-plugin) from 1.6 to 3.0.1. - [Release notes](https://github.com/apache/maven-gpg-plugin/releases) - [Commits](https://github.com/apache/maven-gpg-plugin/compare/maven-gpg-plugin-1.6...maven-gpg-plugin-3.0.1) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 380324944..af378fb6f 100644 --- a/pom.xml +++ b/pom.xml @@ -354,7 +354,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + 3.0.1 sign-artifacts From 30f639e0c5bb95b1fd76329e5d833cb5f74472ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 May 2021 19:33:54 +0000 Subject: [PATCH 0426/1284] Bump mysql-connector-java from 8.0.24 to 8.0.25 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.24 to 8.0.25. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/commits) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index af378fb6f..1848ef107 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ mysql mysql-connector-java - 8.0.24 + 8.0.25 org.mariadb.jdbc From 5e82dbd50227991c8eb819403c2ffe26dd441806 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 31 May 2021 10:10:48 +0300 Subject: [PATCH 0427/1284] more expected errors --- src/sqlancer/clickhouse/ClickHouseErrors.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index b69492e85..08df43612 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -33,6 +33,8 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 + errors.add("Cannot parse number with a sign character but without any numeric character"); + errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); } public static void addExpressionHavingErrors(ExpectedErrors errors) { @@ -57,6 +59,7 @@ public static void addTableManipulationErrors(ExpectedErrors errors) { errors.add("Attempt to read after eof: while converting"); errors.add("Sorting key cannot contain constants"); errors.add("Sampling expression must be present in the primary key"); + errors.add("Cyclic aliases: default expression and column type are incompatible"); } } From 437bcd34705d38823666db615a5f89af1cbc630a Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 1 Jun 2021 12:45:37 +0300 Subject: [PATCH 0428/1284] Fix ClickHouse testing after 0.3.1 ClickHouse JDBC breaking changes --- src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java | 4 +++- .../ast/ClickHouseBinaryComparisonOperationTest.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index 280fade9b..4d6f97008 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -49,7 +49,9 @@ public void start() { if (Randomly.getBoolean()) { sb.append("IF NOT EXISTS "); } - sb.append(tableName); + sb.append(this.globalState.getDatabaseName()); + sb.append("."); + sb.append(this.tableName); sb.append(" ("); int nrColumns = 1 + Randomly.smallNumber(); for (int i = 0; i < nrColumns; i++) { diff --git a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java index a5fafd55a..d9dd89a5f 100644 --- a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java +++ b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java @@ -1,11 +1,12 @@ package sqlancer.clickhouse.ast; import org.junit.jupiter.api.Test; -import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Arrays; import java.util.stream.Collectors; +import ru.yandex.clickhouse.domain.ClickHouseDataType; + import static org.junit.jupiter.api.Assertions.assertEquals; class ClickHouseBinaryComparisonOperationTest { From 17f78c2f778f096353de145b69a3105227061b56 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Mon, 19 Jul 2021 00:26:30 +0800 Subject: [PATCH 0429/1284] Fix PostgresProvider that use wrong database name --- src/sqlancer/postgres/PostgresProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 683ba95d1..c088fa4b7 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -257,7 +257,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE s.execute(createDatabaseCommand); } con.close(); - int databaseIndex = entryURL.indexOf(entryPath) + 1; + int databaseIndex = entryURL.indexOf(entryDatabaseName); String preDatabaseName = entryURL.substring(0, databaseIndex); String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); testURL = preDatabaseName + databaseName + postDatabaseName; From 748ca241024609ac4d6d7ff5d9402af9dca29903 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Mon, 19 Jul 2021 16:01:47 +0800 Subject: [PATCH 0430/1284] Fix compilation failure for some generic parameter types in IDEA --- src/sqlancer/arangodb/ArangoDBOptions.java | 5 +++-- src/sqlancer/clickhouse/ClickHouseOptions.java | 10 +++++----- src/sqlancer/mongodb/MongoDBOptions.java | 7 ++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index ba06ceac3..f1759bccb 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -11,6 +11,7 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.arangodb.test.ArangoDBQueryPartitioningWhereTester; +import sqlancer.arangodb.ArangoDBProvider.ArangoDBGlobalState; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; @@ -34,10 +35,10 @@ public List getTestOracleFactory() { return oracles; } - public enum ArangoDBOracleFactory implements OracleFactory { + public enum ArangoDBOracleFactory implements OracleFactory { QUERY_PARTITIONING { @Override - public TestOracle create(ArangoDBProvider.ArangoDBGlobalState globalState) throws Exception { + public TestOracle create(ArangoDBGlobalState globalState) throws Exception { List oracles = new ArrayList<>(); oracles.add(new ArangoDBQueryPartitioningWhereTester(globalState)); return new CompositeTestOracle(oracles, globalState); diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 5c26cb03b..c25905648 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -33,31 +33,31 @@ public class ClickHouseOptions implements DBMSSpecificOptions { TLPWhere { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPWhereOracle(globalState); } }, TLPDistinct { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPDistinctOracle(globalState); } }, TLPGroupBy { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPGroupByOracle(globalState); } }, TLPAggregate { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPAggregateOracle(globalState); } }, TLPHaving { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPHavingOracle(globalState); } }; diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 77c565a11..5c618f5f7 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -13,6 +13,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; import sqlancer.mongodb.test.MongoDBDocumentRemovalTester; import sqlancer.mongodb.test.MongoDBQueryPartitioningWhereTester; @@ -50,10 +51,10 @@ public List getTestOracleFactory() { return oracles; } - public enum MongoDBOracleFactory implements OracleFactory { + public enum MongoDBOracleFactory implements OracleFactory { QUERY_PARTITIONING { @Override - public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws Exception { + public TestOracle create(MongoDBGlobalState globalState) throws Exception { List oracles = new ArrayList<>(); oracles.add(new MongoDBQueryPartitioningWhereTester(globalState)); return new CompositeTestOracle(oracles, globalState); @@ -61,7 +62,7 @@ public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws }, DOCUMENT_REMOVAL { @Override - public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws Exception { + public TestOracle create(MongoDBGlobalState globalState) throws Exception { List oracles = new ArrayList<>(); oracles.add(new MongoDBDocumentRemovalTester(globalState)); return new CompositeTestOracle(oracles, globalState); From 8c15ec8c116e3ac070bbdac7db7391e80e1916c3 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Wed, 21 Jul 2021 16:18:17 +0800 Subject: [PATCH 0431/1284] Fix typos for dbmsSpecificOptions --- src/sqlancer/GlobalState.java | 10 +++---- src/sqlancer/Main.java | 4 +-- src/sqlancer/ProviderAdapter.java | 2 +- src/sqlancer/arangodb/ArangoDBProvider.java | 2 +- .../arangodb/gen/ArangoDBInsertGenerator.java | 2 +- .../ArangoDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/citus/CitusProvider.java | 4 +-- .../clickhouse/ClickHouseProvider.java | 2 +- .../gen/ClickHouseExpressionGenerator.java | 2 +- .../cockroachdb/CockroachDBProvider.java | 6 ++-- .../gen/CockroachDBIndexGenerator.java | 2 +- .../gen/CockroachDBTableGenerator.java | 4 +-- src/sqlancer/duckdb/DuckDBProvider.java | 8 +++--- .../duckdb/gen/DuckDBExpressionGenerator.java | 28 +++++++++---------- .../duckdb/gen/DuckDBIndexGenerator.java | 2 +- .../duckdb/gen/DuckDBTableGenerator.java | 12 ++++---- src/sqlancer/mongodb/MongoDBProvider.java | 2 +- src/sqlancer/mongodb/MongoDBSchema.java | 2 +- .../mongodb/gen/MongoDBConstantGenerator.java | 2 +- .../mongodb/gen/MongoDBInsertGenerator.java | 2 +- .../gen/MongoDBMatchExpressionGenerator.java | 4 +-- .../mongodb/gen/MongoDBTableGenerator.java | 2 +- .../test/MongoDBDocumentRemovalBase.java | 4 +-- .../test/MongoDBQueryPartitioningBase.java | 4 +-- .../MongoDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/mysql/MySQLGlobalState.java | 2 +- src/sqlancer/postgres/PostgresProvider.java | 6 ++-- .../gen/PostgresExpressionGenerator.java | 2 +- .../postgres/gen/PostgresInsertGenerator.java | 2 +- src/sqlancer/sqlite3/SQLite3Provider.java | 14 +++++----- .../sqlite3/gen/SQLite3ColumnBuilder.java | 6 ++-- .../gen/SQLite3ExpressionGenerator.java | 12 ++++---- .../gen/ddl/SQLite3TableGenerator.java | 12 ++++---- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 2 +- .../sqlite3/oracle/SQLite3Fuzzer.java | 2 +- 35 files changed, 88 insertions(+), 88 deletions(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 642c9da6a..64b5c731e 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -10,7 +10,7 @@ public abstract class GlobalState, S extends Ab protected C databaseConnection; private Randomly r; private MainOptions options; - private O dmbsSpecificOptions; + private O dbmsSpecificOptions; private S schema; private Main.StateLogger logger; private StateToReproduce state; @@ -26,12 +26,12 @@ public C getConnection() { } @SuppressWarnings("unchecked") - public void setDmbsSpecificOptions(Object dmbsSpecificOptions) { - this.dmbsSpecificOptions = (O) dmbsSpecificOptions; + public void setDbmsSpecificOptions(Object dbmsSpecificOptions) { + this.dbmsSpecificOptions = (O) dbmsSpecificOptions; } - public O getDmbsSpecificOptions() { - return dmbsSpecificOptions; + public O getDbmsSpecificOptions() { + return dbmsSpecificOptions; } public void setRandomly(Randomly r) { diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 4c1130f28..20024e437 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -309,7 +309,7 @@ public void run() throws Exception { state.setRandomly(r); state.setDatabaseName(databaseName); state.setMainOptions(options); - state.setDmbsSpecificOptions(command); + state.setDbmsSpecificOptions(command); try (C con = provider.createDatabase(state)) { QueryManager manager = new QueryManager<>(state); try { @@ -343,7 +343,7 @@ private G getInitializedGlobalState(long seed) { state.setRandomly(r); state.setDatabaseName(databaseName); state.setMainOptions(options); - state.setDmbsSpecificOptions(command); + state.setDbmsSpecificOptions(command); return state; } diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 1e3158563..a16f8388c 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -63,7 +63,7 @@ public void generateAndTestDatabase(G globalState) throws Exception { protected abstract void checkViewsAreValid(G globalState); protected TestOracle getTestOracle(G globalState) throws Exception { - List> testOracleFactory = globalState.getDmbsSpecificOptions() + List> testOracleFactory = globalState.getDbmsSpecificOptions() .getTestOracleFactory(); boolean testOracleRequiresMoreThanZeroRows = testOracleFactory.stream() .anyMatch(p -> p.requiresAllTablesToContainRows()); diff --git a/src/sqlancer/arangodb/ArangoDBProvider.java b/src/sqlancer/arangodb/ArangoDBProvider.java index d63d4a1cc..a44c4e422 100644 --- a/src/sqlancer/arangodb/ArangoDBProvider.java +++ b/src/sqlancer/arangodb/ArangoDBProvider.java @@ -47,7 +47,7 @@ private static int mapActions(ArangoDBGlobalState globalState, Action a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case CREATE_INDEX: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumberIndexes); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumberIndexes); default: throw new AssertionError(a); } diff --git a/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java index 9a27ccd57..3cfceeed4 100644 --- a/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java +++ b/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java @@ -26,7 +26,7 @@ private ArangoDBQueryAdapter generate() { ArangoDBConstantGenerator constantGenerator = new ArangoDBConstantGenerator(globalState); for (int i = 0; i < table.getColumns().size(); i++) { - if (!globalState.getDmbsSpecificOptions().testRandomTypeInserts) { + if (!globalState.getDbmsSpecificOptions().testRandomTypeInserts) { constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), table.getColumns().get(i).getType()); } else { diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java index 4484da8f7..6ad19fabf 100644 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java @@ -35,7 +35,7 @@ public void check() throws Exception { thirdResultSet.addAll(secondResultSet); assumeResultSetsAreEqual(firstResultSet, thirdResultSet, query); - if (state.getDmbsSpecificOptions().withOptimizerRuleTests) { + if (state.getDbmsSpecificOptions().withOptimizerRuleTests) { select.setFilterClause(predicate); query = ArangoDBVisitor.asSelectQuery(select); query.excludeRandomOptRules(); diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 49a548298..6d7e5e507 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -300,7 +300,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws Exception { @Override protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { - List oracles = ((CitusOptions) globalState.getDmbsSpecificOptions()).citusOracle.stream().map(o -> { + List oracles = ((CitusOptions) globalState.getDbmsSpecificOptions()).citusOracle.stream().map(o -> { try { return o.create(globalState); } catch (Exception e1) { @@ -415,7 +415,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE // reconnect to coordinator node, test database con = new SQLConnection(DriverManager.getConnection("jdbc:" + testURL, username, password)); ((CitusGlobalState) globalState) - .setRepartition(((CitusOptions) globalState.getDmbsSpecificOptions()).repartition); + .setRepartition(((CitusOptions) globalState.getDbmsSpecificOptions()).repartition); globalState.getState().commentStatements(); return con; } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 297369ded..d936238d4 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -112,7 +112,7 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ port = ClickHouseOptions.DEFAULT_PORT; } - ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); + ClickHouseOptions clickHouseOptions = globalState.getDbmsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); String url = String.format("jdbc:clickhouse://%s:%d/default", host, port); String databaseName = globalState.getDatabaseName(); diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 3bb29e75d..1054055f9 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -91,7 +91,7 @@ protected ClickHouseLancerDataType getRandomType() { public List getRandomJoinClauses( List tables) { List joinStatements = new ArrayList<>(); - if (!globalState.getDmbsSpecificOptions().testJoins) { + if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } if (Randomly.getBoolean() && tables.size() > 1) { diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 8b675459e..ef9e56e98 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -135,10 +135,10 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio standardSettings.add("SET CLUSTER SETTING sql.stats.automatic_collection.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING timeseries.storage.enabled = 'off'"); - if (globalState.getDmbsSpecificOptions().testHashIndexes) { + if (globalState.getDbmsSpecificOptions().testHashIndexes) { standardSettings.add("set experimental_enable_hash_sharded_indexes='on';"); } - if (globalState.getDmbsSpecificOptions().testTempTables) { + if (globalState.getDbmsSpecificOptions().testTempTables) { standardSettings.add("SET experimental_enable_temp_tables = 'on'"); } for (String s : standardSettings) { @@ -241,7 +241,7 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio } total--; } - if (globalState.getDmbsSpecificOptions().makeVectorizationMoreLikely && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().makeVectorizationMoreLikely && Randomly.getBoolean()) { manager.execute(new SQLQueryAdapter("SET vectorize=on;")); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index fd4dfd70d..a37f5a26b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -38,7 +38,7 @@ public void buildStatement() { sb.append(table.getName()); List columns = table.getRandomNonEmptyColumnSubset(); addColumns(sb, columns, true); - boolean hashSharded = globalState.getDmbsSpecificOptions().testHashIndexes + boolean hashSharded = globalState.getDbmsSpecificOptions().testHashIndexes && Randomly.getBooleanWithSmallProbability(); if (hashSharded) { sb.append(" USING HASH WITH BUCKET_COUNT="); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index a33b7521f..d244d2080 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -35,13 +35,13 @@ public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { @Override public void buildStatement() { errors.add("and thus is not indexable"); // array types are not indexable - if (globalState.getDmbsSpecificOptions().testTempTables) { + if (globalState.getDbmsSpecificOptions().testTempTables) { errors.add("constraints on temporary tables may reference only temporary tables"); errors.add("constraints on permanent tables may reference only permanent tables"); } String tableName = globalState.getSchema().getFreeTableName(); sb.append("CREATE "); - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().testTempTables) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testTempTables) { sb.append("TEMP "); } sb.append("TABLE "); diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 3acfc56cb..73adb3cb0 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -65,20 +65,20 @@ private static int mapActions(DuckDBGlobalState globalState, Action a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case CREATE_INDEX: - if (!globalState.getDmbsSpecificOptions().testIndexes) { + if (!globalState.getDbmsSpecificOptions().testIndexes) { return 0; } // fall through case UPDATE: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumUpdates + 1); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); case VACUUM: // seems to be ignored case ANALYZE: // seems to be ignored case EXPLAIN: return r.getInteger(0, 2); case DELETE: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumDeletes + 1); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); case CREATE_VIEW: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumViews + 1); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); } diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 99446ad41..f907e62f2 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -51,28 +51,28 @@ protected Node generateExpression(int depth) { return new NewFunctionNode<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - if (!globalState.getDmbsSpecificOptions().testCollate) { + if (!globalState.getDbmsSpecificOptions().testCollate) { possibleOptions.remove(Expression.COLLATE); } - if (!globalState.getDmbsSpecificOptions().testFunctions) { + if (!globalState.getDbmsSpecificOptions().testFunctions) { possibleOptions.remove(Expression.FUNC); } - if (!globalState.getDmbsSpecificOptions().testCasts) { + if (!globalState.getDbmsSpecificOptions().testCasts) { possibleOptions.remove(Expression.CAST); } - if (!globalState.getDmbsSpecificOptions().testBetween) { + if (!globalState.getDbmsSpecificOptions().testBetween) { possibleOptions.remove(Expression.BETWEEN); } - if (!globalState.getDmbsSpecificOptions().testIn) { + if (!globalState.getDbmsSpecificOptions().testIn) { possibleOptions.remove(Expression.IN); } - if (!globalState.getDmbsSpecificOptions().testCase) { + if (!globalState.getDbmsSpecificOptions().testCase) { possibleOptions.remove(Expression.CASE); } - if (!globalState.getDmbsSpecificOptions().testBinaryComparisons) { + if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { possibleOptions.remove(Expression.BINARY_COMPARISON); } - if (!globalState.getDmbsSpecificOptions().testBinaryLogicals) { + if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { possibleOptions.remove(Expression.BINARY_LOGICAL); } Expression expr = Randomly.fromList(possibleOptions); @@ -135,32 +135,32 @@ public Node generateConstant() { DuckDBDataType type = DuckDBDataType.getRandom(); switch (type) { case INT: - if (!globalState.getDmbsSpecificOptions().testIntConstants) { + if (!globalState.getDbmsSpecificOptions().testIntConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createIntConstant(globalState.getRandomly().getInteger()); case DATE: - if (!globalState.getDmbsSpecificOptions().testDateConstants) { + if (!globalState.getDbmsSpecificOptions().testDateConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createDateConstant(globalState.getRandomly().getInteger()); case TIMESTAMP: - if (!globalState.getDmbsSpecificOptions().testTimestampConstants) { + if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); case VARCHAR: - if (!globalState.getDmbsSpecificOptions().testStringConstants) { + if (!globalState.getDbmsSpecificOptions().testStringConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createStringConstant(globalState.getRandomly().getString()); case BOOLEAN: - if (!globalState.getDmbsSpecificOptions().testBooleanConstants) { + if (!globalState.getDbmsSpecificOptions().testBooleanConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createBooleanConstant(Randomly.getBoolean()); case FLOAT: - if (!globalState.getDmbsSpecificOptions().testFloatConstants) { + if (!globalState.getDbmsSpecificOptions().testFloatConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 4b6b47612..eec5384aa 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -50,7 +50,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(DuckDBToStringVisitor.asString(expr)); } errors.add("already exists!"); - if (globalState.getDmbsSpecificOptions().testRowid) { + if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("Cannot create an index on the rowid!"); } return new SQLQueryAdapter(sb.toString(), errors, true); diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index de48e5c0f..bb8fe2304 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -36,33 +36,33 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType()); - if (globalState.getDmbsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() && columns.get(i).getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { sb.append(" COLLATE "); sb.append(getRandomCollate()); } - if (globalState.getDmbsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" UNIQUE"); } - if (globalState.getDmbsSpecificOptions().testNotNullConstraints + if (globalState.getDbmsSpecificOptions().testNotNullConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); } - if (globalState.getDmbsSpecificOptions().testCheckConstraints + if (globalState.getDbmsSpecificOptions().testCheckConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" CHECK("); sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); DuckDBErrors.addExpressionErrors(errors); sb.append(")"); } - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().testDefaultValues) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); DuckDBErrors.addExpressionErrors(errors); sb.append(")"); } } - if (globalState.getDmbsSpecificOptions().testIndexes && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { errors.add("Invalid type for index"); List primaryKeyColumns = Randomly.nonEmptySubset(columns); sb.append(", PRIMARY KEY("); diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java index 5ff549d32..3b090dade 100644 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ b/src/sqlancer/mongodb/MongoDBProvider.java @@ -49,7 +49,7 @@ public static int mapActions(MongoDBGlobalState globalState, Action a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case CREATE_INDEX: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumberIndexes); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumberIndexes); default: throw new AssertionError(a); } diff --git a/src/sqlancer/mongodb/MongoDBSchema.java b/src/sqlancer/mongodb/MongoDBSchema.java index e9a0afb99..5ae3cdd24 100644 --- a/src/sqlancer/mongodb/MongoDBSchema.java +++ b/src/sqlancer/mongodb/MongoDBSchema.java @@ -36,7 +36,7 @@ public BsonType getBsonType() { public static MongoDBDataType getRandom(MongoDBGlobalState state) { Set valueSet = new HashSet<>(Arrays.asList(values())); - if (state.getDmbsSpecificOptions().nullSafety) { + if (state.getDbmsSpecificOptions().nullSafety) { valueSet.remove(STRING); } MongoDBDataType[] configuredValues = new MongoDBDataType[valueSet.size()]; diff --git a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java index e81291543..2e6b15048 100644 --- a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java @@ -48,7 +48,7 @@ public void addRandomConstant(Document document, String key) { public void addRandomConstantWithType(Document document, String key, MongoDBDataType option) { MongoDBConstant constant; - if (globalState.getDmbsSpecificOptions().testNullInserts && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().testNullInserts && Randomly.getBooleanWithSmallProbability()) { constant = new MongoDBNullConstant(); constant.setValueInDocument(document, key); return; diff --git a/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java b/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java index 4501971b4..f8b8b3ffd 100644 --- a/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java @@ -25,7 +25,7 @@ public MongoDBQueryAdapter generate() { MongoDBConstantGenerator constantGenerator = new MongoDBConstantGenerator(globalState); for (int i = 0; i < table.getColumns().size(); i++) { - if (!globalState.getDmbsSpecificOptions().testRandomTypes) { + if (!globalState.getDbmsSpecificOptions().testRandomTypes) { constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), table.getColumns().get(i).getType()); } else { diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java index 0645d7f24..3e6c833d3 100644 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java @@ -44,7 +44,7 @@ public MongoDBMatchExpressionGenerator(MongoDBGlobalState globalState) { @Override public Node generateLeafNode() { List possibleOptions = new ArrayList<>(Arrays.asList(LeafExpression.values())); - if (!globalState.getDmbsSpecificOptions().testWithRegex) { + if (!globalState.getDbmsSpecificOptions().testWithRegex) { possibleOptions.remove(LeafExpression.REGEX); } LeafExpression expr = Randomly.fromList(possibleOptions); @@ -97,7 +97,7 @@ public Node generateConstant() { public Node generateConstant(MongoDBDataType type) { MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); - if (Randomly.getBooleanWithSmallProbability() && !globalState.getDmbsSpecificOptions().nullSafety) { + if (Randomly.getBooleanWithSmallProbability() && !globalState.getDbmsSpecificOptions().nullSafety) { return MongoDBConstant.createNullConstant(); } return generator.generateConstantWithType(type); diff --git a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java index 6a4f33d38..0153a6334 100644 --- a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java @@ -28,7 +28,7 @@ public MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { for (int i = 0; i < Randomly.smallNumber() + 1; i++) { String columnName = String.format("c%d", i); MongoDBDataType type = createColumn(columnName); - if (globalState.getDmbsSpecificOptions().testValidation) { + if (globalState.getDbmsSpecificOptions().testValidation) { createTableQuery.addValidation(columnName, type.getBsonType()); } } diff --git a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java index e3e3df17c..6faae42bc 100644 --- a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java +++ b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java @@ -44,7 +44,7 @@ public void check() throws Exception { } else { select.setLookupList(Randomly.nonEmptySubset(targetColumns)); } - if (state.getDmbsSpecificOptions().testComputedValues) { + if (state.getDbmsSpecificOptions().testComputedValues) { generateComputedColumns(); } } @@ -55,7 +55,7 @@ private void generateTargetColumns() { targetColumns.add(new MongoDBColumnTestReference(c, true)); } List joinsOtherTables = new ArrayList<>(); - if (!state.getDmbsSpecificOptions().nullSafety) { + if (!state.getDbmsSpecificOptions().nullSafety) { for (int i = 1; i < targetTables.getTables().size(); i++) { MongoDBSchema.MongoDBTable procTable = targetTables.getTables().get(i); for (MongoDBSchema.MongoDBColumn c : procTable.getColumns()) { diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java index b26aa6028..30ebe56bd 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -47,7 +47,7 @@ public void check() throws Exception { } else { select.setLookupList(Randomly.nonEmptySubset(targetColumns)); } - if (state.getDmbsSpecificOptions().testComputedValues) { + if (state.getDbmsSpecificOptions().testComputedValues) { generateComputedColumns(); } } @@ -69,7 +69,7 @@ private void generateTargetColumns() { targetColumns.add(new MongoDBColumnTestReference(c, true)); } List joinsOtherTables = new ArrayList<>(); - if (!state.getDmbsSpecificOptions().nullSafety) { + if (!state.getDbmsSpecificOptions().nullSafety) { for (int i = 1; i < targetTables.getTables().size(); i++) { MongoDBTable procTable = targetTables.getTables().get(i); for (MongoDBColumn c : procTable.getColumns()) { diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java index 5a7507672..53f625d9b 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java @@ -33,7 +33,7 @@ public void check() throws Exception { q = new MongoDBSelectQuery(select); List thirdResultSet = getResultSetAsDocumentList(q, state); - if (state.getDmbsSpecificOptions().testWithCount) { + if (state.getDbmsSpecificOptions().testWithCount) { select.setWithCountClause(true); select.setFilterClause(predicate); q = new MongoDBSelectQuery(select); diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 041ac440b..86cf6dd18 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -14,7 +14,7 @@ protected MySQLSchema readSchema() throws SQLException { } public boolean usesPQS() { - return getDmbsSpecificOptions().oracles.stream().anyMatch(o -> o == MySQLOracleFactory.PQS); + return getDbmsSpecificOptions().oracles.stream().anyMatch(o -> o == MySQLOracleFactory.PQS); } } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 683ba95d1..36dd55465 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -195,7 +195,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws Exception { @Override public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLException { - if (globalState.getDmbsSpecificOptions().getTestOracleFactory().stream() + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch((o) -> o == PostgresOracleFactory.PQS)) { generateOnlyKnown = true; } @@ -205,7 +205,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); entryPath = "/test"; - entryURL = globalState.getDmbsSpecificOptions().connectionURL; + entryURL = globalState.getDbmsSpecificOptions().connectionURL; // trim URL to exclude "jdbc:" if (entryURL.startsWith("jdbc:")) { entryURL = entryURL.substring(5); @@ -305,7 +305,7 @@ protected void prepareTables(PostgresGlobalState globalState) throws Exception { private String getCreateDatabaseCommand(PostgresGlobalState state) { StringBuilder sb = new StringBuilder(); sb.append("CREATE DATABASE " + databaseName + " "); - if (Randomly.getBoolean() && ((PostgresOptions) state.getDmbsSpecificOptions()).testCollations) { + if (Randomly.getBoolean() && ((PostgresOptions) state.getDbmsSpecificOptions()).testCollations) { if (Randomly.getBoolean()) { sb.append("WITH ENCODING '"); sb.append(Randomly.fromOptions("utf8")); diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 010bb8fec..f979e97da 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -379,7 +379,7 @@ private PostgresExpression generateTextExpression(int depth) { if (expectedResult) { validOptions.remove(TextExpression.COLLATE); } - if (!globalState.getDmbsSpecificOptions().testCollations) { + if (!globalState.getDbmsSpecificOptions().testCollations) { validOptions.remove(TextExpression.COLLATE); } option = Randomly.fromList(validOptions); diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index f23387977..41c017b2e 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -49,7 +49,7 @@ public static SQLQueryAdapter insert(PostgresGlobalState globalState) { } sb.append(" VALUES"); - if (globalState.getDmbsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { StringBuilder sbRowValue = new StringBuilder(); sbRowValue.append("("); for (int i = 0; i < columns.size(); i++) { diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 35e9b90cd..54f1a6db4 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -168,7 +168,7 @@ private static int mapActions(SQLite3GlobalState globalState, Action a) { public void generateDatabase(SQLite3GlobalState globalState) throws Exception { Randomly r = new Randomly(SQLite3SpecialStringGenerator::generate); globalState.setRandomly(r); - if (globalState.getDmbsSpecificOptions().generateDatabase) { + if (globalState.getDbmsSpecificOptions().generateDatabase) { addSensiblePragmaDefaults(globalState); int nrTablesToCreate = 1; @@ -186,7 +186,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws Exception { } while (globalState.getSchema().getDatabaseTables().size() < nrTablesToCreate); assert globalState.getSchema().getTables().getTables().size() == nrTablesToCreate; checkTablesForGeneratedColumnLoops(globalState); - if (globalState.getDmbsSpecificOptions().testDBStats && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().testDBStats && Randomly.getBooleanWithSmallProbability()) { SQLQueryAdapter tableQuery = new SQLQueryAdapter( "CREATE VIRTUAL TABLE IF NOT EXISTS stat USING dbstat(main)"); globalState.executeStatement(tableQuery); @@ -224,10 +224,10 @@ private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) private SQLQueryAdapter getTableQuery(SQLite3GlobalState globalState, int i) throws AssertionError { SQLQueryAdapter tableQuery; List options = new ArrayList<>(Arrays.asList(TableType.values())); - if (!globalState.getDmbsSpecificOptions().testFts) { + if (!globalState.getDbmsSpecificOptions().testFts) { options.remove(TableType.FTS); } - if (!globalState.getDmbsSpecificOptions().testRtree) { + if (!globalState.getDbmsSpecificOptions().testRtree) { options.remove(TableType.RTREE); } switch (Randomly.fromList(options)) { @@ -255,11 +255,11 @@ private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws Ex if (!Randomly.getBooleanWithSmallProbability()) { pragmasToExecute.addAll(DEFAULT_PRAGMAS); } - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { // the PQS implementation currently assumes the default behavior of LIKE pragmasToExecute.add("PRAGMA case_sensitive_like=ON;"); } - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { // the encoding has an influence how binary strings are cast pragmasToExecute.add(String.format("PRAGMA encoding = '%s';", Randomly.fromOptions("UTF-8", "UTF-16", "UTF-16le", "UTF-16be"))); @@ -276,7 +276,7 @@ public SQLConnection createDatabase(SQLite3GlobalState globalState) throws SQLEx dir.mkdir(); } File dataBase = new File(dir, globalState.getDatabaseName() + ".db"); - if (dataBase.exists() && ((SQLite3GlobalState) globalState).getDmbsSpecificOptions().deleteIfExists) { + if (dataBase.exists() && ((SQLite3GlobalState) globalState).getDbmsSpecificOptions().deleteIfExists) { dataBase.delete(); } String url = "jdbc:sqlite:" + dataBase.getAbsolutePath(); diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index 41a6fdcef..e07533ed8 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -39,8 +39,8 @@ public boolean isContainsPrimaryKey() { } public String createColumn(String columnName, SQLite3GlobalState globalState, List columns) { - if (globalState.getDmbsSpecificOptions().oracles == SQLite3OracleFactory.PQS - || !globalState.getDmbsSpecificOptions().testCheckConstraints) { + if (globalState.getDbmsSpecificOptions().oracles == SQLite3OracleFactory.PQS + || !globalState.getDbmsSpecificOptions().testCheckConstraints) { allowCheck = false; } sb.append(columnName); @@ -51,7 +51,7 @@ public String createColumn(String columnName, SQLite3GlobalState globalState, Li if (Randomly.getBooleanWithRatherLowProbability()) { List constraints = Randomly.subset(Constraints.values()); if (!Randomly.getBooleanWithSmallProbability() - || globalState.getDmbsSpecificOptions().testGeneratedColumns) { + || globalState.getDbmsSpecificOptions().testGeneratedColumns) { constraints.remove(Constraints.GENERATED_AS); } if (constraints.contains(Constraints.GENERATED_AS)) { diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index edbbc29f6..e7480dc19 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -136,7 +136,7 @@ public List generateOrderBys() { public List getRandomJoinClauses(List tables) { List joinStatements = new ArrayList<>(); - if (!globalState.getDmbsSpecificOptions().testJoins) { + if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } if (Randomly.getBoolean() && tables.size() > 1) { @@ -165,7 +165,7 @@ public SQLite3Expression generateOrderingTerm() { if (Randomly.getBoolean()) { expr = new SQLite3OrderingTerm(expr, Ordering.getRandomValue()); } - if (globalState.getDmbsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { expr = new SQLite3PostfixText(expr, Randomly.fromOptions(" NULLS FIRST", " NULLS LAST"), null /* expr.getExpectedValue() */) { @Override @@ -253,13 +253,13 @@ public SQLite3Expression getRandomExpression(int depth) { if (!allowSubqueries) { list.remove(ExpressionType.RANDOM_QUERY); } - if (!globalState.getDmbsSpecificOptions().testFunctions) { + if (!globalState.getDbmsSpecificOptions().testFunctions) { list.remove(ExpressionType.FUNCTION); } - if (!globalState.getDmbsSpecificOptions().testMatch) { + if (!globalState.getDbmsSpecificOptions().testMatch) { list.remove(ExpressionType.MATCH); } - if (!globalState.getDmbsSpecificOptions().testIn) { + if (!globalState.getDbmsSpecificOptions().testIn) { list.remove(ExpressionType.IN_OPERATOR); } ExpressionType randomExpressionType = Randomly.fromList(list); @@ -504,7 +504,7 @@ static AnyFunction getRandom(SQLite3GlobalState globalState) { private static List getAllFunctions(SQLite3GlobalState globalState) { List functions = new ArrayList<>(Arrays.asList(AnyFunction.values())); - if (!globalState.getDmbsSpecificOptions().testSoundex) { + if (!globalState.getDbmsSpecificOptions().testSoundex) { boolean removed = functions.removeIf(f -> f.name.equals("soundex")); if (!removed) { throw new IllegalStateException(); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index ef7d696cf..cc4731911 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -62,7 +62,7 @@ public static SQLQueryAdapter createTableStatement(String tableName, SQLite3Glob public void start() { sb.append("CREATE "); - if (globalState.getDmbsSpecificOptions().testTempTables && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testTempTables && Randomly.getBoolean()) { tempTable = true; if (Randomly.getBoolean()) { sb.append("TEMP "); @@ -110,12 +110,12 @@ public void start() { } } - if (globalState.getDmbsSpecificOptions().testForeignKeys && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().testForeignKeys && Randomly.getBooleanWithSmallProbability()) { addForeignKey(); } - if (globalState.getDmbsSpecificOptions().testCheckConstraints && globalState - .getDmbsSpecificOptions().oracles != SQLite3OracleFactory.PQS /* + if (globalState.getDbmsSpecificOptions().testCheckConstraints && globalState + .getDbmsSpecificOptions().oracles != SQLite3OracleFactory.PQS /* * we are currently lacking a parser to * read column definitions, and would * interpret a COLLATE in the check @@ -126,7 +126,7 @@ public void start() { } sb.append(")"); - if (globalState.getDmbsSpecificOptions().testWithoutRowids && containsPrimaryKey && !containsAutoIncrement + if (globalState.getDbmsSpecificOptions().testWithoutRowids && containsPrimaryKey && !containsAutoIncrement && Randomly.getBoolean()) { // see https://sqlite.org/withoutrowid.html sb.append(" WITHOUT ROWID"); @@ -151,7 +151,7 @@ private void addColumnConstraints(String s) { * @see https://www.sqlite.org/foreignkeys.html */ private void addForeignKey() { - assert globalState.getDmbsSpecificOptions().testForeignKeys; + assert globalState.getDbmsSpecificOptions().testForeignKeys; List foreignKeyColumns; if (Randomly.getBoolean()) { foreignKeyColumns = Arrays.asList(Randomly.fromList(columnNames)); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 400c66864..ffa3af6cd 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -50,7 +50,7 @@ public static SQLQueryAdapter generate(SQLite3GlobalState globalState) throws SQ SQLite3Expression randomQuery; do { randomQuery = SQLite3RandomQuerySynthesizer.generate(globalState, size); - } while (globalState.getDmbsSpecificOptions().oracles == SQLite3OracleFactory.PQS + } while (globalState.getDbmsSpecificOptions().oracles == SQLite3OracleFactory.PQS && !checkAffinity(randomQuery)); sb.append(SQLite3Visitor.asString(randomQuery)); return new SQLQueryAdapter(sb.toString(), errors, true); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index 429b22dd6..8e868dfd0 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -20,7 +20,7 @@ public void check() throws Exception { String s = SQLite3Visitor .asString(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + ";"; try { - if (globalState.getDmbsSpecificOptions().executeQuery) { + if (globalState.getDbmsSpecificOptions().executeQuery) { globalState.executeStatement(new SQLQueryAdapter(s)); globalState.getManager().incrementSelectQueryCount(); } From 83531015bea6d442650d83039929df10bc9d4db4 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Mon, 26 Jul 2021 09:47:16 +0800 Subject: [PATCH 0432/1284] reorder import packages --- src/sqlancer/arangodb/ArangoDBOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index f1759bccb..6e7a607f0 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -10,8 +10,8 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.arangodb.test.ArangoDBQueryPartitioningWhereTester; import sqlancer.arangodb.ArangoDBProvider.ArangoDBGlobalState; +import sqlancer.arangodb.test.ArangoDBQueryPartitioningWhereTester; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; From b820fcd6b93cf52b5cc369da70b8fa4c1422dad4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 26 Jul 2021 22:21:06 +0200 Subject: [PATCH 0433/1284] Adapt to a recent PostgreSQL change --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 05e1e62ba..243073034 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -235,6 +235,7 @@ private void createColumnConstraint(PostgresDataType type, boolean serial) { switch (c) { case NULL_OR_NOT_NULL: sb.append(Randomly.fromOptions("NOT NULL", "NULL")); + errors.add("conflicting NULL/NOT NULL declarations"); break; case UNIQUE: sb.append("UNIQUE"); From f6ddf1e7d4ac18958dd9ef249da6d3c9309b4794 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 26 Jul 2021 22:35:46 +0200 Subject: [PATCH 0434/1284] Adapt to the removal of the H2 COMPRESS_LOB option --- src/sqlancer/h2/H2SetGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/h2/H2SetGenerator.java b/src/sqlancer/h2/H2SetGenerator.java index 26211a93c..88833e1e9 100644 --- a/src/sqlancer/h2/H2SetGenerator.java +++ b/src/sqlancer/h2/H2SetGenerator.java @@ -21,7 +21,6 @@ private enum Option { BUILTIN_ALIAS_OVERRIDE((r) -> Randomly.fromOptions("TRUE", "FALSE")), // COLLATION((r) -> Randomly.fromOptions("OFF", "ENGLISH", "ENGLISH STRENGTH PRIMARY", "ENGLISH STRENGTH SECONDARY", "ENGLISH STRENGTH TERTIARY", "ENGLISH STRENGTH IDENTICAL")), // - COMPRESS_LOB((r) -> Randomly.fromOptions("NO", "LZF", "DEFLATE")), // DEFAULT_NULL_ORDERING((r) -> Randomly.fromOptions("LOW", "HIGH", "FIRST", "LAST")), // DEFAULT_TABLE_TYPE((r) -> Randomly.fromOptions("MEMORY", "CACHED")), // IGNORECASE((r) -> Randomly.fromOptions("TRUE", "FALSE")), // From 2b3722810a00f4e8ababa26c7627ff8631acab7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 20:49:36 +0000 Subject: [PATCH 0435/1284] Bump slf4j-simple from 1.7.30 to 1.7.32 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.30 to 1.7.32. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/commits) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1848ef107..ae4212606 100644 --- a/pom.xml +++ b/pom.xml @@ -254,7 +254,7 @@ org.slf4j slf4j-simple - 1.7.30 + 1.7.32 ru.yandex.clickhouse From 7adeaabb520d50bda0e04fa4cf33d3dc7d89abf2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 21:02:47 +0000 Subject: [PATCH 0436/1284] Bump clickhouse-jdbc from 0.3.1 to 0.3.1-patch Bumps [clickhouse-jdbc](https://github.com/ClickHouse/clickhouse-jdbc) from 0.3.1 to 0.3.1-patch. - [Release notes](https://github.com/ClickHouse/clickhouse-jdbc/releases) - [Changelog](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/ClickHouse/clickhouse-jdbc/compare/v0.3.1...v0.3.1-patch) --- updated-dependencies: - dependency-name: ru.yandex.clickhouse:clickhouse-jdbc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ae4212606..99e5796a0 100644 --- a/pom.xml +++ b/pom.xml @@ -259,7 +259,7 @@ ru.yandex.clickhouse clickhouse-jdbc - 0.3.1 + 0.3.1-patch com.h2database From 53edb4d639b9c09a2683cb5a279aeace38b8c4fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 21:21:32 +0000 Subject: [PATCH 0437/1284] Bump maven-javadoc-plugin from 3.2.0 to 3.3.0 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.2.0...maven-javadoc-plugin-3.3.0) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99e5796a0..dbeceb225 100644 --- a/pom.xml +++ b/pom.xml @@ -338,7 +338,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.2.0 + 3.3.0 8 From 5bb9de2ca04938654382293118adbd1c2a883301 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 21:53:23 +0000 Subject: [PATCH 0438/1284] Bump postgresql from 42.2.20 to 42.2.23 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.20 to 42.2.23. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/REL42.2.23/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.20...REL42.2.23) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dbeceb225..8f8852a9f 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ org.postgresql postgresql - 42.2.20 + 42.2.23 org.xerial From 2171065193c1c9ff4ad171dd3bfc1cd2cfc7054d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 22:56:21 +0000 Subject: [PATCH 0439/1284] Bump junit-jupiter-engine from 5.7.1 to 5.7.2 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.7.1 to 5.7.2. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.7.1...r5.7.2) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f8852a9f..1f9078b87 100644 --- a/pom.xml +++ b/pom.xml @@ -248,7 +248,7 @@ org.junit.jupiter junit-jupiter-engine - 5.7.1 + 5.7.2 test From 16a372b3b2fe456e514a1152c189f125aa2ff008 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 23:13:33 +0000 Subject: [PATCH 0440/1284] Bump mysql-connector-java from 8.0.25 to 8.0.26 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.25 to 8.0.26. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.25...8.0.26) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1f9078b87..97d742dd2 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ mysql mysql-connector-java - 8.0.25 + 8.0.26 org.mariadb.jdbc From 049d3bc3786e563a7afd1932c14d4570002ebf33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 08:18:21 +0000 Subject: [PATCH 0441/1284] Bump sqlite-jdbc from 3.34.0 to 3.36.0.1 Bumps [sqlite-jdbc](https://github.com/xerial/sqlite-jdbc) from 3.34.0 to 3.36.0.1. - [Release notes](https://github.com/xerial/sqlite-jdbc/releases) - [Changelog](https://github.com/xerial/sqlite-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/xerial/sqlite-jdbc/compare/3.34.0...3.36.0.1) --- updated-dependencies: - dependency-name: org.xerial:sqlite-jdbc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97d742dd2..c860e0690 100644 --- a/pom.xml +++ b/pom.xml @@ -228,7 +228,7 @@ org.xerial sqlite-jdbc - 3.34.0 + 3.36.0.1 mysql From 063a04c56eabce449ed3bdc50d879b332064ca4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 08:42:19 +0000 Subject: [PATCH 0442/1284] Bump formatter-maven-plugin from 2.15.0 to 2.16.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.15.0 to 2.16.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.15.0...formatter-maven-plugin-2.16.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c860e0690..54bd97bdd 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.15.0 + 2.16.0 eclipseformat From 6c08269ff48f4cf667490e8794bdcf830958813b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 28 Jul 2021 10:11:11 +0200 Subject: [PATCH 0443/1284] Fix an expected MySQL 'Row size too large' error --- src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index 408463f24..3f418385d 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -114,6 +114,7 @@ public SQLQueryAdapter create() { errors.add("out of range"); errors.add("Data truncated for functional index"); errors.add("used in key specification without a key length"); + errors.add("Row size too large"); // seems to happen together with MIN_ROWS in the table declaration return new SQLQueryAdapter(string, errors, true); } From 5fa10dfd581719296db0cb9737dbfac1a97f08d5 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Wed, 30 Jun 2021 16:52:39 -0700 Subject: [PATCH 0444/1284] Update CockroachDB to v21.1.7 --- .github/workflows/main.yml | 6 +++--- src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java | 1 + .../gen/CockroachDBSetClusterSettingGenerator.java | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 61458f894..9c4adaa15 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,11 +110,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.2.8.linux-amd64.tgz | tar xvz - cd cockroach-v20.2.8.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v21.1.7.linux-amd64.tgz | tar xvz + cd cockroach-v21.1.7.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v20.2.8.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v21.1.7.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index a37f5a26b..9ded19411 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -27,6 +27,7 @@ public void buildStatement() { errors.add("violates unique constraint"); errors.add("schema change statement cannot follow a statement that has written in the same transaction"); errors.add("and thus is not indexable"); // array types are not indexable + errors.add("the following columns are not indexable due to their type"); // array types are not indexable errors.add("cannot determine type of empty array. Consider annotating with the desired type"); errors.add("incompatible IF expression"); // TODO: investigate; seems to be a bug CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 4fb36a5d1..2d411f26e 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -15,7 +15,6 @@ private CockroachDBSetClusterSettingGenerator() { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBClusterSetting { - COMPATOR_ENABLED("compactor.enabled", CockroachDBSetSessionGenerator::onOff), BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), From 78eaf96c2115d944ed523dfb8ca744a3beba8080 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Wed, 30 Jun 2021 16:53:11 -0700 Subject: [PATCH 0445/1284] Remove interleaved tables and indexes for CockroachDB Interleaved tables are disabled by default in CockroachDB v21.1, and will be permanently removed in a future release. This commit removes generation of interleaved tables and indexes. --- .../cockroachdb/CockroachDBProvider.java | 15 +++------------ .../cockroachdb/gen/CockroachDBGenerator.java | 19 ------------------- .../gen/CockroachDBIndexGenerator.java | 3 --- .../gen/CockroachDBTableGenerator.java | 3 --- .../gen/CockroachDBTruncateGenerator.java | 1 - 5 files changed, 3 insertions(+), 38 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index ef9e56e98..2fcdf1930 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -54,12 +54,9 @@ public enum Action { SHOW(CockroachDBShowGenerator::show), // TRANSACTION((g) -> { String s = Randomly.fromOptions("BEGIN", "ROLLBACK", "COMMIT"); - return new SQLQueryAdapter(s, - ExpectedErrors.from("there is no transaction in progress", - "there is already a transaction in progress", "current transaction is aborted", - "does not exist" /* interleaved indexes */)); - }), // - EXPLAIN((g) -> { + return new SQLQueryAdapter(s, ExpectedErrors.from("there is no transaction in progress", + "there is already a transaction in progress", "current transaction is aborted")); + }), EXPLAIN((g) -> { StringBuilder sb = new StringBuilder("EXPLAIN "); ExpectedErrors errors = new ExpectedErrors(); if (Randomly.getBoolean()) { @@ -267,12 +264,6 @@ public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws S globalState.getState().logStatement("USE " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); - } catch (SQLException e) { - if (e.getMessage().contains("ERROR: invalid interleave backreference")) { - throw new IgnoreMeException(); // TODO: investigate - } else { - throw e; - } } try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java index e06cc8d9d..34d0b9355 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java @@ -1,12 +1,10 @@ package sqlancer.cockroachdb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.common.gen.AbstractGenerator; public abstract class CockroachDBGenerator extends AbstractGenerator { @@ -32,21 +30,4 @@ static void addColumns(StringBuilder sb, List columns, boolea sb.append(")"); } - void generateInterleave() { - // TODO make this more likely to succeed - CockroachDBTable parentTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - List parentColumns = parentTable.getRandomNonEmptyColumnSubset(); - sb.append(" INTERLEAVE IN PARENT "); - sb.append(parentTable.getName()); - sb.append("("); - sb.append(parentColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - errors.add("must refer to a prefix of the primary key column names being interleaved"); - errors.add("must refer to a prefix of the index column names being interleaved"); - errors.add("must match the parent's primary index"); - errors.add("must match type and sort direction of the parent's primary index"); - errors.add("must be a prefix of the index columns being interleaved"); - errors.add("must be a prefix of the primary key columns being interleaved"); - } - } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 9ded19411..66c2936fd 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -53,9 +53,6 @@ public void buildStatement() { sb.append(" "); addColumns(sb, table.getRandomNonEmptyColumnSubset(), false); } - if (!hashSharded /* interleaved indexes cannot also be hash sharded */ && Randomly.getBoolean()) { - generateInterleave(); - } } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index d244d2080..720f2a300 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -167,9 +167,6 @@ public void buildStatement() { errors.add("there is no unique constraint matching given keys for referenced table"); } sb.append(")"); - if (Randomly.getBooleanWithRatherLowProbability() && !globalState.getSchema().getDatabaseTables().isEmpty()) { - generateInterleave(); - } errors.add("collatedstring"); CockroachDBErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index c5bdc2881..02579cf32 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -13,7 +13,6 @@ private CockroachDBTruncateGenerator() { // https://www.cockroachlabs.com/docs/v19.2/truncate.html public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); - errors.add("is interleaved by table"); errors.add("is referenced by foreign key"); // https://github.com/cockroachdb/cockroach/issues/47030 From 60666706e8621bec465410890401604d502ed771 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Aug 2021 11:47:03 +0000 Subject: [PATCH 0446/1284] Bump mariadb-java-client from 2.7.3 to 2.7.4 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.7.3 to 2.7.4. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/2.7.3...2.7.4) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 54bd97bdd..e932cfd93 100644 --- a/pom.xml +++ b/pom.xml @@ -238,7 +238,7 @@ org.mariadb.jdbc mariadb-java-client - 2.7.3 + 2.7.4 org.duckdb From f830e58f70d7de7f4f35c9f99e947766ecbfe5ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 09:13:32 +0000 Subject: [PATCH 0447/1284] Bump sqlite-jdbc from 3.36.0.1 to 3.36.0.3 Bumps [sqlite-jdbc](https://github.com/xerial/sqlite-jdbc) from 3.36.0.1 to 3.36.0.3. - [Release notes](https://github.com/xerial/sqlite-jdbc/releases) - [Changelog](https://github.com/xerial/sqlite-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/xerial/sqlite-jdbc/compare/3.36.0.1...3.36.0.3) --- updated-dependencies: - dependency-name: org.xerial:sqlite-jdbc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e932cfd93..1cfa89d74 100644 --- a/pom.xml +++ b/pom.xml @@ -228,7 +228,7 @@ org.xerial sqlite-jdbc - 3.36.0.1 + 3.36.0.3 mysql From 33b8a7a07aeca158d925ac95c4c92e073e65c447 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Sep 2021 09:08:43 +0000 Subject: [PATCH 0448/1284] Bump maven-javadoc-plugin from 3.3.0 to 3.3.1 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.3.0 to 3.3.1. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.3.0...maven-javadoc-plugin-3.3.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1cfa89d74..20f8088dc 100644 --- a/pom.xml +++ b/pom.xml @@ -338,7 +338,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.3.0 + 3.3.1 8 From 92b75a619352967853f68e5a49b4c740562dd42e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 09:10:44 +0000 Subject: [PATCH 0449/1284] Bump junit-jupiter-engine from 5.7.2 to 5.8.0 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.7.2 to 5.8.0. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.7.2...r5.8.0) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 20f8088dc..c506e2940 100644 --- a/pom.xml +++ b/pom.xml @@ -248,7 +248,7 @@ org.junit.jupiter junit-jupiter-engine - 5.7.2 + 5.8.0 test From 357cd4414185b98c0746459799d652e53ae2186a Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 16 Sep 2021 14:18:50 +0300 Subject: [PATCH 0450/1284] Fix incorrect variable names in PostgresGlobalState --- src/sqlancer/postgres/PostgresGlobalState.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index f36536975..119d9fc24 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -40,16 +40,16 @@ public void setConnection(SQLConnection con) { } private List getCollnames(SQLConnection con) throws SQLException { - List opClasses = new ArrayList<>(); + List collNames = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s .executeQuery("SELECT collname FROM pg_collation WHERE collname LIKE '%utf8' or collname = 'C';")) { while (rs.next()) { - opClasses.add(rs.getString(1)); + collNames.add(rs.getString(1)); } } } - return opClasses; + return collNames; } private List getOpclasses(SQLConnection con) throws SQLException { @@ -65,15 +65,15 @@ private List getOpclasses(SQLConnection con) throws SQLException { } private List getOperators(SQLConnection con) throws SQLException { - List opClasses = new ArrayList<>(); + List operators = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { while (rs.next()) { - opClasses.add(rs.getString(1)); + operators.add(rs.getString(1)); } } } - return opClasses; + return operators; } public List getOperators() { From 749b0fd0345101b557bc0a63911c7dc001738c3a Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 15 Sep 2021 17:01:38 +0300 Subject: [PATCH 0451/1284] Add Citus 10.1 support --- .github/workflows/main.yml | 6 +++--- .travis.yml | 4 ++-- src/sqlancer/citus/CitusProvider.java | 6 +++--- src/sqlancer/citus/CitusSchema.java | 4 ++++ src/sqlancer/postgres/PostgresSchema.java | 1 + 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9c4adaa15..074a8702a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: - name: Set up Citus run: | curl https://install.citusdata.com/community/deb.sh | sudo bash - sudo apt-get -y install postgresql-13-citus-9.5 + sudo apt-get -y install postgresql-13-citus-10.1 sudo chown -R $USER:$USER /var/run/postgresql export PATH=/usr/lib/postgresql/13/bin:$PATH cd ~ @@ -68,8 +68,8 @@ jobs: psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test - psql -c "SELECT * from master_add_node('localhost', 9701);" -p 9700 -U $USER -d test - psql -c "SELECT * from master_add_node('localhost', 9702);" -p 9700 -U $USER -d test + psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test + psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test - name: Run Tests run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test diff --git a/.travis.yml b/.travis.yml index 13cf6704d..318ea6fa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -130,8 +130,8 @@ matrix: - psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test - psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test - psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test - - psql -c "SELECT * from master_add_node('localhost', 9701);" -p 9700 -U $USER -d test - - psql -c "SELECT * from master_add_node('localhost', 9702);" -p 9700 -U $USER -d test + - psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test + - psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test script: - CITUS_AVAILABLE=true mvn -Dtest=TestCitus test - name: ClickHouse diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 6d7e5e507..978168800 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -312,10 +312,10 @@ protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLEx private List readCitusWorkerNodes(PostgresGlobalState globalState, SQLConnection con) throws SQLException { - globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); + globalState.getState().logStatement("SELECT * FROM citus_get_active_worker_nodes()"); List citusWorkerNodes = new ArrayList<>(); try (Statement s = con.createStatement()) { - ResultSet rs = s.executeQuery("SELECT * FROM master_get_active_worker_nodes();"); + ResultSet rs = s.executeQuery("SELECT * FROM citus_get_active_worker_nodes();"); while (rs.next()) { String nodeHost = rs.getString("node_name"); int nodePort = rs.getInt("node_port"); @@ -374,7 +374,7 @@ private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes) throws SQLException { for (CitusWorkerNode w : citusWorkerNodes) { - String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; + String addWorkers = "SELECT * from citus_add_node('" + w.getHost() + "', " + w.getPort() + ");"; globalState.getState().logStatement(addWorkers); try (Statement s = con.createStatement()) { s.execute(addWorkers); diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index 694292a25..defda8804 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -65,6 +65,10 @@ public static CitusSchema fromConnection(SQLConnection con, String databaseName) "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { while (rs.next()) { String tableName = rs.getString("table_name"); + /* citus_tables is a helper view, we don't need to test with it so we let's ignore it */ + if (tableName.equals("citus_tables")) { + continue; + } String distributionColumnName = rs.getString("dist_col_name"); Integer colocationId = rs.getInt("colocationid"); if (rs.wasNull()) { diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 889b2e2a6..c99c8648e 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -124,6 +124,7 @@ public static PostgresDataType getColumnType(String typeString) { case "character": case "character varying": case "name": + case "regclass": return PostgresDataType.TEXT; case "numeric": return PostgresDataType.DECIMAL; From 076ec8e2eff85b448b3ba9db0ddaf0a8f19f76cf Mon Sep 17 00:00:00 2001 From: Neha George Date: Mon, 20 Sep 2021 09:06:42 -0400 Subject: [PATCH 0452/1284] Remove JOIN Oracle in CockroachDB This oracle does not necessarily produce the same results for both queries, thus rendering it an invalid TLP oracle. --- .../cockroachdb/CockroachDBOptions.java | 7 -- .../oracle/tlp/CockroachDBTLPJoinOracle.java | 118 ------------------ 2 files changed, 125 deletions(-) delete mode 100644 src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 2af62ae69..7673c4c0a 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -18,7 +18,6 @@ import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPJoinOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPWhereOracle; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; @@ -77,12 +76,6 @@ public TestOracle create(CockroachDBGlobalState globalState) throws SQLException return new CockroachDBTLPExtendedWhereOracle(globalState); } }, - JOIN { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPJoinOracle(globalState); - } - }, QUERY_PARTITIONING { @Override public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java deleted file mode 100644 index 988f82fba..000000000 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java +++ /dev/null @@ -1,118 +0,0 @@ -package sqlancer.cockroachdb.oracle.tlp; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.cockroachdb.CockroachDBErrors; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; -import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.cockroachdb.ast.CockroachDBColumnReference; -import sqlancer.cockroachdb.ast.CockroachDBConstant; -import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBJoin.OuterType; -import sqlancer.cockroachdb.ast.CockroachDBNotOperation; -import sqlancer.cockroachdb.ast.CockroachDBSelect; -import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; -import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class CockroachDBTLPJoinOracle implements TestOracle { - - final CockroachDBGlobalState state; - final ExpectedErrors errors = new ExpectedErrors(); - - CockroachDBSchema s; - CockroachDBTables targetTables; - CockroachDBExpressionGenerator gen; - CockroachDBSelect select; - CockroachDBExpression predicate; - CockroachDBExpression negatedPredicate; - CockroachDBExpression isNullPredicate; - - public CockroachDBTLPJoinOracle(CockroachDBGlobalState state) { - errors.add("GROUP BY term out of range"); - CockroachDBErrors.addExpressionErrors(errors); - this.state = state; - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new CockroachDBExpressionGenerator(state).setColumns(targetTables.getColumns()); - select = new CockroachDBSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - if (tables.size() < 2) { - return; - } - List tableList = tables.stream().map(t -> new CockroachDBTableReference(t)) - .collect(Collectors.toList()); - CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); - CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); - CockroachDBJoin leftJoinTrue = CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, - CockroachDBConstant.createBooleanConstant(true)); - - select.setJoinList(Arrays.asList(leftJoinTrue)); - select.setFromList(tableList); - select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new CockroachDBNotOperation(predicate); - isNullPredicate = new CockroachDBUnaryPostfixOperation(predicate, CockroachDBUnaryPostfixOperator.IS_NULL); - - String originalQueryString1 = CockroachDBVisitor.asString(select); - - CockroachDBJoin leftJoinFalse = CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, - CockroachDBConstant.createBooleanConstant(false)); - select.setJoinList(Arrays.asList(leftJoinFalse)); - String originalQueryString2 = CockroachDBVisitor.asString(select); - String originalQueryString = originalQueryString1 + " UNION ALL " + originalQueryString2 + " UNION ALL " - + originalQueryString2; - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - // boolean allowOrderBy = Randomly.getBoolean(); - // if (allowOrderBy) { - // select.setOrderByExpressions(gen.getOrderingTerms()); - // } - - select.setJoinList( - Arrays.asList(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, predicate))); - String firstQueryString = CockroachDBVisitor.asString(select); - - select.setJoinList(Arrays - .asList(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, negatedPredicate))); - String secondQueryString = CockroachDBVisitor.asString(select); - - select.setJoinList( - Arrays.asList(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, isNullPredicate))); - String thirdQueryString = CockroachDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - - List generateFetchColumns() { - return Arrays.asList(new CockroachDBColumnReference(targetTables.getColumns().get(0))); - } - - CockroachDBExpression generatePredicate() { - return gen.generateExpression(CockroachDBDataType.BOOL.get()); - } - -} From 17f8e4f770f5e7e5b26493f707ba9f820bc78acb Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 16 Sep 2021 12:32:14 +0300 Subject: [PATCH 0453/1284] Add support for access methods --- src/sqlancer/citus/CitusProvider.java | 16 ++++++++++++ src/sqlancer/citus/gen/CitusCommon.java | 3 +++ .../citus/gen/CitusTableGenerator.java | 6 +++++ .../postgres/PostgresGlobalState.java | 25 +++++++++++++++++++ .../postgres/gen/PostgresTableGenerator.java | 21 +++++++++++++++- 5 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 978168800..172149a52 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -21,8 +21,10 @@ import sqlancer.citus.gen.CitusIndexGenerator; import sqlancer.citus.gen.CitusInsertGenerator; import sqlancer.citus.gen.CitusSetGenerator; +import sqlancer.citus.gen.CitusTableGenerator; import sqlancer.citus.gen.CitusUpdateGenerator; import sqlancer.citus.gen.CitusViewGenerator; +import sqlancer.common.DBMSCommon; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; @@ -271,6 +273,20 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl distributeTable(columns, tableName, globalState); } + @Override + protected void createTables(PostgresGlobalState globalState, int numTables) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = CitusTableGenerator.generate(tableName, globalState.getSchema(), + generateOnlyKnown, globalState); + globalState.executeStatement(createTable); + } catch (IgnoreMeException e) { + + } + } + } + @Override public void generateDatabase(PostgresGlobalState globalState) throws Exception { readFunctions(globalState); diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 2b7bdaea5..f4a121efe 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -47,6 +47,9 @@ public static void addCitusErrors(ExpectedErrors errors) { errors.add("cannot pushdown the subquery"); // see https://github.com/sqlancer/sqlancer/issues/215 errors.add("direct joins between distributed and local tables are not supported"); + errors.add("unlogged columnar tables are not supported"); + errors.add("UPDATE and CTID scans not supported for ColumnarScan"); + errors.add("indexes not supported for columnar tables"); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java index 45d80e6aa..c340400f3 100644 --- a/src/sqlancer/citus/gen/CitusTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -1,5 +1,6 @@ package sqlancer.citus.gen; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.gen.PostgresTableGenerator; @@ -12,4 +13,9 @@ public CitusTableGenerator(String tableName, PostgresSchema newSchema, boolean g CitusCommon.addCitusErrors(errors); } + public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, + PostgresGlobalState globalState) { + return new CitusTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); + } + } diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 119d9fc24..0f2c7ebdb 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -23,6 +23,7 @@ public class PostgresGlobalState extends SQLGlobalState operators = Collections.emptyList(); private List collates = Collections.emptyList(); private List opClasses = Collections.emptyList(); + private List tableAccessMethods = Collections.emptyList(); // store and allow filtering by function volatility classifications private final Map functionsAndTypes = new HashMap<>(); private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); @@ -34,6 +35,7 @@ public void setConnection(SQLConnection con) { this.opClasses = getOpclasses(getConnection()); this.operators = getOperators(getConnection()); this.collates = getCollnames(getConnection()); + this.tableAccessMethods = getTableAccessMethods(getConnection()); } catch (SQLException e) { throw new AssertionError(e); } @@ -76,6 +78,21 @@ private List getOperators(SQLConnection con) throws SQLException { return operators; } + private List getTableAccessMethods(SQLConnection con) throws SQLException { + List tableAccessMethods = new ArrayList<>(); + try (Statement s = con.createStatement()) { + /* + * pg_am includes both index and table access methods so we need to filter with amtype = 't' + */ + try (ResultSet rs = s.executeQuery("SELECT amname FROM pg_am WHERE amtype = 't';")) { + while (rs.next()) { + tableAccessMethods.add(rs.getString(1)); + } + } + } + return tableAccessMethods; + } + public List getOperators() { return operators; } @@ -100,6 +117,14 @@ public String getRandomOpclass() { return Randomly.fromList(opClasses); } + public List getTableAccessMethods() { + return tableAccessMethods; + } + + public String getRandomTableAccessMethod() { + return Randomly.fromList(tableAccessMethods); + } + @Override public PostgresSchema readSchema() throws SQLException { return PostgresSchema.fromConnection(getConnection(), getDatabaseName()); diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 243073034..7d65b8ebd 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -23,6 +23,7 @@ public class PostgresTableGenerator { private boolean columnHasPrimaryKey; private final StringBuilder sb = new StringBuilder(); private boolean isTemporaryTable; + private boolean isPartitionedTable; private final PostgresSchema newSchema; private final List columnsToBeAdded = new ArrayList<>(); protected final ExpectedErrors errors = new ExpectedErrors(); @@ -63,7 +64,7 @@ public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchem return new PostgresTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); } - private SQLQueryAdapter generate() { + protected SQLQueryAdapter generate() { columnCanHavePrimaryKey = true; sb.append("CREATE"); if (Randomly.getBoolean()) { @@ -111,6 +112,7 @@ private void createStandard() throws AssertionError { sb.append(")"); generateInherits(); generatePartitionBy(); + generateUsing(); PostgresCommon.generateWith(sb, globalState, errors); if (Randomly.getBoolean() && isTemporaryTable) { sb.append(" ON COMMIT "); @@ -152,8 +154,10 @@ private void createColumn(String name) throws AssertionError { private void generatePartitionBy() { if (Randomly.getBoolean()) { + isPartitionedTable = false; return; } + isPartitionedTable = true; sb.append(" PARTITION BY "); // TODO "RANGE", String partitionOption = Randomly.fromOptions("RANGE", "LIST", "HASH"); @@ -185,6 +189,21 @@ private void generatePartitionBy() { sb.append(")"); } + private void generateUsing() { + /* + * Postgres does not allow specifying USING clause for partitioned tables since they don't have any storage + * associated with them + */ + if (isPartitionedTable) { + return; + } + if (Randomly.getBoolean()) { + return; + } + sb.append(" USING "); + sb.append(globalState.getRandomTableAccessMethod()); + } + private void generateInherits() { if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { sb.append(" INHERITS("); From 8ec26f7b036e088f8f3dcf08bd3f17f1172315ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Sep 2021 09:10:12 +0000 Subject: [PATCH 0454/1284] Bump junit-jupiter-engine from 5.8.0 to 5.8.1 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.8.0 to 5.8.1. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.8.0...r5.8.1) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c506e2940..0d7b1878c 100644 --- a/pom.xml +++ b/pom.xml @@ -248,7 +248,7 @@ org.junit.jupiter junit-jupiter-engine - 5.8.0 + 5.8.1 test From 448989040cb09fa3d5787ce43e51f8014cf9dbd6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 09:12:33 +0000 Subject: [PATCH 0455/1284] Bump postgresql from 42.2.23 to 42.2.24 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.23 to 42.2.24. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/REL42.2.24/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.23...REL42.2.24) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0d7b1878c..1a943045f 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ org.postgresql postgresql - 42.2.23 + 42.2.24 org.xerial From 2cd5db3c00e103e8f63d9d7a47839e6f927acadc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 10 Nov 2021 19:08:39 +0100 Subject: [PATCH 0456/1284] Update and fix the Citus CI step This step started failing with an error "postgresql-13-citus-10.1 : Depends: postgresql-13 but it is not installable" --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 074a8702a..a6629c889 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,6 +43,7 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up Citus run: | + echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list curl https://install.citusdata.com/community/deb.sh | sudo bash sudo apt-get -y install postgresql-13-citus-10.1 sudo chown -R $USER:$USER /var/run/postgresql From 29d3e88a3a725c6f51ed2584b11701c681901e04 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 10 Nov 2021 20:06:08 +0100 Subject: [PATCH 0457/1284] Temporarily disable warnings due to the newly-appearing xml-apis warning Warning: The artifact xml-apis:xml-apis:jar:2.0.2 has been relocated to xml-apis:xml-apis:jar:1.0.b2 --- .github/workflows/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a6629c889..44ef6aebc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,8 +21,6 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build and check for warnings - run: mvn -B package -DskipTests 2>/dev/null | grep -vz "WARNING" - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests From 799cca36cb2f5142c16a17d82ed1d8438a9f473b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 20:41:11 +0000 Subject: [PATCH 0458/1284] Bump postgresql from 42.2.24 to 42.3.1 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.24 to 42.3.1. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.24...REL42.3.1) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1a943045f..8f231ce72 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ org.postgresql postgresql - 42.2.24 + 42.3.1 org.xerial From 107417e59224c6d5c7ec2f6b09a8c4657bb02897 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 20:53:42 +0000 Subject: [PATCH 0459/1284] Bump mysql-connector-java from 8.0.26 to 8.0.27 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.26 to 8.0.27. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.26...8.0.27) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f231ce72..32ad7cd73 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ mysql mysql-connector-java - 8.0.26 + 8.0.27 org.mariadb.jdbc From bbb4ac677f2ca6d5e97afd8373b087386e5244c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 21:18:02 +0000 Subject: [PATCH 0460/1284] Bump spotbugs-maven-plugin from 4.2.2 to 4.4.2.2 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.2.2 to 4.4.2.2. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/4.2.2...spotbugs-maven-plugin-4.4.2.2) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 32ad7cd73..7ab1d96ab 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.2.2 + 4.4.2.2 spotbugs From 1cabe26e6c969b9eeb075b14e64e57cf3fef51b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 20:53:39 +0000 Subject: [PATCH 0461/1284] Bump ecj from 3.25.0 to 3.27.0 Bumps ecj from 3.25.0 to 3.27.0. --- updated-dependencies: - dependency-name: org.eclipse.jdt:ecj dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7ab1d96ab..a47743728 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ org.eclipse.jdt ecj - 3.25.0 + 3.27.0 From 259bb3e2c87d3ea0dd5bb1e4de0d8fc3212666e1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 10 Nov 2021 22:05:54 +0100 Subject: [PATCH 0462/1284] Discontinue support for Java 1.8 to enable the ecj upgrade --- .github/workflows/main.yml | 62 ++++++++++++++------------------------ 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 44ef6aebc..ac460d4be 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,10 +17,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests @@ -33,10 +33,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up Citus @@ -79,10 +79,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up ClickHouse @@ -101,10 +101,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up CockroachDB @@ -125,10 +125,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build run: mvn -B package -DskipTests=true - name: DuckDB Tests @@ -141,10 +141,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up H2 @@ -166,10 +166,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Install MariaDB @@ -191,10 +191,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up MySQL @@ -226,10 +226,10 @@ jobs: postgresql user: 'sqlancer' postgresql password: 'sqlancer' postgresql db: 'test' - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -243,10 +243,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests @@ -262,10 +262,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up TiDB @@ -278,24 +278,6 @@ jobs: - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test - - java8: - name: Java 8 Compatibility (DuckDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build - run: mvn -B package -DskipTests=true - - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - java13: name: Java 13 Compatibility (DuckDB) runs-on: ubuntu-latest From 9452e22b42299301422c014590bcef8bc3b112da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 09:38:11 +0000 Subject: [PATCH 0463/1284] Bump formatter-maven-plugin from 2.16.0 to 2.17.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.16.0 to 2.17.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.16.0...formatter-maven-plugin-2.17.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a47743728..e75164b9a 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.16.0 + 2.17.0 eclipseformat From 1f781b275af40b9124b12a3557a9a94c147bbf26 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 17 Nov 2021 00:39:47 +0100 Subject: [PATCH 0464/1284] Remove the PostgreSQL vacuum_cleanup_index_scale_factor option. This option was removed with Postgres 14. Fixes https://github.com/sqlancer/sqlancer/issues/400. --- src/sqlancer/postgres/gen/PostgresSetGenerator.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index bfc0bbc04..f600f1f04 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -55,8 +55,6 @@ private enum ConfigurationOption { (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), VACUUM_MULTIXACT_FREEZE_MIN_AGE("vacuum_multixact_freeze_min_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 1000000000)), - VACUUM_CLEANUP_INDEX_SCALE_FACTOR("vacuum_cleanup_index_scale_factor", - (r) -> Randomly.fromOptions(0.0, 0.0000001, 0.00001, 0.01, 0.1, 1, 10, 100, 100000, 10000000000.0)), // TODO others GIN_FUZZY_SEARCH_LIMIT("gin_fuzzy_search_limit", (r) -> r.getInteger(0, 2147483647)), // 19.13. Version and Platform Compatibility From 739b8a64aa7f22a9147b3ab67b0a9a8b6277f2f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Nov 2021 09:12:20 +0000 Subject: [PATCH 0465/1284] Bump spotbugs-maven-plugin from 4.4.2.2 to 4.5.0.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.4.2.2 to 4.5.0.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.4.2.2...spotbugs-maven-plugin-4.5.0.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e75164b9a..9a998533e 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.4.2.2 + 4.5.0.0 spotbugs From 2ed71c3f432000cf7942263db6e5f934fc50dc92 Mon Sep 17 00:00:00 2001 From: entropaos Date: Wed, 8 Dec 2021 04:48:13 -0800 Subject: [PATCH 0466/1284] Fix union operator --- src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java | 2 +- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java index e3e522de1..b2594522a 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java @@ -10,7 +10,7 @@ public class PostgresBinaryRangeOperation extends BinaryNode private final String op; public enum PostgresBinaryRangeOperator implements Operator { - UNION("*"), INTERSECTION("*"), DIFFERENCE("-"); + UNION("+"), INTERSECTION("*"), DIFFERENCE("-"); private final String textRepresentation; diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 62a8a811b..771fc8651 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -114,6 +114,7 @@ public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); } public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { From 3f29b049d784cc3b6491aa7f9f767672f876dd83 Mon Sep 17 00:00:00 2001 From: bashaojing Date: Wed, 15 Dec 2021 11:43:01 +0800 Subject: [PATCH 0467/1284] support oceanbase --- .travis.yml | 6 + src/sqlancer/Main.java | 2 + src/sqlancer/oceanbase/OceanBaseErrors.java | 48 ++ .../OceanBaseExpectedValueVisitor.java | 155 +++++ .../oceanbase/OceanBaseGlobalState.java | 20 + .../oceanbase/OceanBaseHintGenerator.java | 133 ++++ src/sqlancer/oceanbase/OceanBaseOptions.java | 60 ++ src/sqlancer/oceanbase/OceanBaseProvider.java | 161 +++++ src/sqlancer/oceanbase/OceanBaseSchema.java | 291 ++++++++ .../oceanbase/OceanBaseToStringVisitor.java | 306 +++++++++ src/sqlancer/oceanbase/OceanBaseVisitor.java | 110 +++ .../oceanbase/ast/OceanBaseAggregate.java | 26 + .../OceanBaseBinaryComparisonOperation.java | 122 ++++ .../ast/OceanBaseBinaryLogicalOperation.java | 110 +++ .../oceanbase/ast/OceanBaseCastOperation.java | 36 + .../oceanbase/ast/OceanBaseColumnName.java | 17 + .../ast/OceanBaseColumnReference.java | 41 ++ .../ast/OceanBaseComputableFunction.java | 270 ++++++++ .../oceanbase/ast/OceanBaseConstant.java | 628 ++++++++++++++++++ .../oceanbase/ast/OceanBaseExists.java | 30 + .../oceanbase/ast/OceanBaseExpression.java | 9 + .../oceanbase/ast/OceanBaseInOperation.java | 58 ++ src/sqlancer/oceanbase/ast/OceanBaseJoin.java | 10 + .../oceanbase/ast/OceanBaseOrderByTerm.java | 36 + .../oceanbase/ast/OceanBaseSelect.java | 58 ++ .../ast/OceanBaseStringExpression.java | 22 + .../ast/OceanBaseTableReference.java | 17 + src/sqlancer/oceanbase/ast/OceanBaseText.java | 27 + .../ast/OceanBaseUnaryPostfixOperation.java | 58 ++ .../ast/OceanBaseUnaryPrefixOperation.java | 91 +++ .../oceanbase/gen/OceanBaseAlterTable.java | 72 ++ .../gen/OceanBaseDeleteGenerator.java | 52 ++ .../oceanbase/gen/OceanBaseDropIndex.java | 30 + .../gen/OceanBaseExpressionGenerator.java | 212 ++++++ .../gen/OceanBaseInsertGenerator.java | 110 +++ .../gen/OceanBaseTableGenerator.java | 282 ++++++++ .../gen/OceanBaseTruncateTableGenerator.java | 18 + .../gen/OceanBaseUpdateGenerator.java | 54 ++ .../gen/datadef/OceanBaseIndexGenerator.java | 150 +++++ .../oracle/OceanBaseNoRECOracle.java | 204 ++++++ .../OceanBasePivotedQuerySynthesisOracle.java | 159 +++++ .../oceanbase/oracle/OceanBaseTLPBase.java | 62 ++ .../oracle/OceanBaseTLPWhereOracle.java | 44 ++ test/sqlancer/dbms/TestOceanBaseNoREC.java | 29 + test/sqlancer/dbms/TestOceanBasePQS.java | 29 + test/sqlancer/dbms/TestOceanBaseTLP.java | 29 + 46 files changed, 4494 insertions(+) create mode 100644 src/sqlancer/oceanbase/OceanBaseErrors.java create mode 100644 src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java create mode 100644 src/sqlancer/oceanbase/OceanBaseGlobalState.java create mode 100644 src/sqlancer/oceanbase/OceanBaseHintGenerator.java create mode 100644 src/sqlancer/oceanbase/OceanBaseOptions.java create mode 100644 src/sqlancer/oceanbase/OceanBaseProvider.java create mode 100644 src/sqlancer/oceanbase/OceanBaseSchema.java create mode 100644 src/sqlancer/oceanbase/OceanBaseToStringVisitor.java create mode 100644 src/sqlancer/oceanbase/OceanBaseVisitor.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseAggregate.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseColumnName.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseConstant.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseExists.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseExpression.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseInOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseJoin.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseSelect.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseTableReference.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseText.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java create mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java create mode 100644 src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java create mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java create mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java create mode 100644 test/sqlancer/dbms/TestOceanBaseNoREC.java create mode 100644 test/sqlancer/dbms/TestOceanBasePQS.java create mode 100644 test/sqlancer/dbms/TestOceanBaseTLP.java diff --git a/.travis.yml b/.travis.yml index 318ea6fa7..6e5594948 100644 --- a/.travis.yml +++ b/.travis.yml @@ -144,3 +144,9 @@ matrix: - sleep 5 script: - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test + - name: OceanBase + jdk : openjdk8 + script: + - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBaseTLP test + - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBasePQS test + - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBaseNoREC test diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 20024e437..1ab9e29d9 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -37,6 +37,7 @@ import sqlancer.postgres.PostgresProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; +import sqlancer.oceanbase.OceanBaseProvider; public final class Main { @@ -560,6 +561,7 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new MongoDBProvider()); providers.add(new CosmosProvider()); providers.add(new ArangoDBProvider()); + providers.add(new OceanBaseProvider()); return providers; } diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java new file mode 100644 index 000000000..392bd7c5e --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -0,0 +1,48 @@ +package sqlancer.oceanbase; + +import sqlancer.common.query.ExpectedErrors; + +public final class OceanBaseErrors { + + private OceanBaseErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) + errors.add("is not valid for CHARACTER SET"); + errors.add("The observer or zone is not the master"); + errors.add("Incorrect integer value"); + errors.add("Truncated incorrect DOUBLE value"); + errors.add("Invalid numeric"); + errors.add("Data truncated for argument"); + } + public static void addInsertErrors(ExpectedErrors errors) { + errors.add("Duplicate entry"); + errors.add("cannot be null"); + errors.add("doesn't have a default value"); + errors.add("Out of range value"); + errors.add("Incorrect double value"); + errors.add("Incorrect float value"); + errors.add("Incorrect int value"); + errors.add("Incorrect tinyint value"); + errors.add("Data truncation"); + errors.add("Bad Number"); + errors.add("The value specified for generated column"); // TODO: do not insert data into generated columns + errors.add("incorrect utf8 value"); + errors.add("Data truncation: %s value is out of range in '%s'"); + errors.add("Incorrect smallint value"); + errors.add("Incorrect bigint value"); + errors.add("Incorrect decimal value"); + errors.add("error parsing regexp"); + errors.add("The observer or zone is not the master"); + errors.add("Incorrect integer value"); + errors.add("Truncated incorrect DOUBLE value"); + errors.add("Data truncated for argument"); + errors.add("Invalid numeric"); + + + if (true) { + errors.add("Miss column"); + } + } +} diff --git a/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java new file mode 100644 index 000000000..2c03436d1 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java @@ -0,0 +1,155 @@ +package sqlancer.oceanbase; + +import sqlancer.IgnoreMeException; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseColumnName; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; + +public class OceanBaseExpectedValueVisitor implements OceanBaseVisitor { + + private final StringBuilder sb = new StringBuilder(); + private int nrTabs; + + private void print(OceanBaseExpression expr) { + OceanBaseToStringVisitor v = new OceanBaseToStringVisitor(); + v.visit(expr); + for (int i = 0; i < nrTabs; i++) { + sb.append("\t"); + } + sb.append(v.get()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + @Override + public void visit(OceanBaseExpression expr) { + nrTabs++; + try { + OceanBaseVisitor.super.visit(expr); + } catch (IgnoreMeException e) { + + } + nrTabs--; + } + + @Override + public void visit(OceanBaseConstant constant) { + print(constant); + } + + @Override + public void visit(OceanBaseColumnReference column) { + print(column); + } + + @Override + public void visit(OceanBaseUnaryPostfixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(OceanBaseComputableFunction f) { + print(f); + for (OceanBaseExpression expr : f.getArguments()) { + visit(expr); + } + } + + @Override + public void visit(OceanBaseBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public String get() { + return sb.toString(); + } + + @Override + public void visit(OceanBaseSelect select) { + for (OceanBaseExpression j : select.getJoinList()) { + visit(j); + } + if (select.getWhereClause() != null) { + visit(select.getWhereClause()); + } + } + + @Override + public void visit(OceanBaseBinaryComparisonOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(OceanBaseCastOperation op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(OceanBaseInOperation op) { + print(op); + visit(op.getExpr()); + for (OceanBaseExpression right : op.getListElements()) { + visit(right); + } + } + + @Override + public void visit(OceanBaseOrderByTerm op) { + } + + @Override + public void visit(OceanBaseExists op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(OceanBaseStringExpression op) { + print(op); + } + + @Override + public void visit(OceanBaseTableReference ref) { + } + + @Override + public void visit(OceanBaseAggregate aggr) { + } + + @Override + public void visit(OceanBaseColumnName aggr) { + } + + @Override + public void visit(OceanBaseText func) { + } + + @Override + public void visit(OceanBaseUnaryPrefixOperation op){ + print(op); + visit(op.getExpr()); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseGlobalState.java b/src/sqlancer/oceanbase/OceanBaseGlobalState.java new file mode 100644 index 000000000..b8baf08fb --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseGlobalState.java @@ -0,0 +1,20 @@ + +package sqlancer.oceanbase; + +import java.sql.SQLException; + +import sqlancer.SQLGlobalState; +import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; + +public class OceanBaseGlobalState extends SQLGlobalState { + + @Override + protected OceanBaseSchema readSchema() throws SQLException { + return OceanBaseSchema.fromConnection(getConnection(), getDatabaseName()); + } + + public boolean usesPQS() { + return getDbmsSpecificOptions().oracles.stream().anyMatch(o -> o == OceanBaseOracleFactory.PQS); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java new file mode 100644 index 000000000..83accab72 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java @@ -0,0 +1,133 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; + +import java.util.List; +import java.util.stream.Collectors; + +public class OceanBaseHintGenerator { + private OceanBaseSelect select; + private List tables; + private final StringBuilder sb = new StringBuilder(); + private Randomly r = new Randomly(); + + enum IndexHint { + MERGE_JOIN, + INL_JOIN, + INL_HASH_JOIN, + INL_MERGE_JOIN, + HASH_JOIN, + HASH_AGG, + STREAM_AGG, + USE_INDEX, + IGNORE_INDEX, + AGG_TO_COP, + USE_INDEX_MERGE, + NO_INDEX_MERGE, + LEADING, + PredDeduce, + PDML, + USE_TOJA; + } + + public OceanBaseHintGenerator(OceanBaseSelect select, List tables) { + this.select = select; + this.tables = tables; + } + + public static void generateHints(OceanBaseSelect select, List tables) { + new OceanBaseHintGenerator(select, tables).generate(); + + } + + private void generate() { + OceanBaseTable table = Randomly.fromList(tables); + switch (Randomly.fromOptions(IndexHint.values())) { + case PDML: + sb.append(" parallel(" + r.getInteger(0, 10) + "),enable_parallel_dml "); + break; + case PredDeduce: + sb.append("no_pred_deduce"); + break; + case MERGE_JOIN: + tablesHint("USE_MERGE "); + break; + case INL_JOIN: + tablesHint("USE_NL "); + break; + case LEADING: + tablesHint(" LEADING "); + break; + case INL_HASH_JOIN: + tablesHint("USE_HASH "); + break; + case INL_MERGE_JOIN: + tablesHint("USE_BNL "); + break; + case HASH_JOIN: + sb.append(" parallel(1) "); + break; + case HASH_AGG: + sb.append("USE_HASH_AGGREGATION "); + break; + case STREAM_AGG: + sb.append("USE_NL_MATERIALIZATION "); + break; + case USE_INDEX: + indexesHint("INDEX_HINT "); + break; + case IGNORE_INDEX: + sb.append("TOPK (50 50) "); + break; + case AGG_TO_COP: + sb.append("USE_LATE_MATERIALIZATION "); + break; + case USE_INDEX_MERGE: + sb.append("ORDERED "); + break; + case NO_INDEX_MERGE: + tablesHint("NO_MERGE "); + break; + case USE_TOJA: + sb.append("no_rewrite " ); + break; + default: + throw new AssertionError(); + } + + select.setHint(new OceanBaseStringExpression(sb.toString(),new OceanBaseConstant.OceanBaseTextConstant(sb.toString()))); + } + + private void indexesHint(String string) { + sb.append(string); + sb.append("("); + OceanBaseTable table = Randomly.fromList(tables); + List allIndexes = table.getIndexes(); + if (allIndexes.isEmpty()) { + throw new IgnoreMeException(); + } + List indexSubset = Randomly.nonEmptySubset(allIndexes); + sb.append(table.getName()); + sb.append(", "); + sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); + sb.append(")"); + } + + private void tablesHint(String string) { + sb.append(string); + sb.append("("); + appendTables(); + sb.append(")"); + } + + private void appendTables() { + List tableSubset = Randomly.nonEmptySubset(tables); + sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + } +} diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java new file mode 100644 index 000000000..fb529e375 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -0,0 +1,60 @@ +package sqlancer.oceanbase; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; +import sqlancer.oceanbase.oracle.OceanBasePivotedQuerySynthesisOracle; +import sqlancer.oceanbase.oracle.OceanBaseTLPWhereOracle; +import sqlancer.oceanbase.oracle.OceanBaseNoRECOracle; + +@Parameters(separators = "=", commandDescription = "OceanBase (default port: " + OceanBaseOptions.DEFAULT_PORT + + ", default host: " + OceanBaseOptions.DEFAULT_HOST) +public class OceanBaseOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 2883; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(OceanBaseOracleFactory.TLP_WHERE); + + public enum OceanBaseOracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + return new OceanBaseTLPWhereOracle(globalState); + } + }, + NoREC { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + return new OceanBaseNoRECOracle(globalState); + } + }, + PQS { + + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + return new OceanBasePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + } + } + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseProvider.java b/src/sqlancer/oceanbase/OceanBaseProvider.java new file mode 100644 index 000000000..5b57ee3c1 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseProvider.java @@ -0,0 +1,161 @@ +package sqlancer.oceanbase; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import sqlancer.AbstractAction; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.oceanbase.gen.OceanBaseAlterTable; +import sqlancer.oceanbase.gen.OceanBaseDeleteGenerator; +import sqlancer.oceanbase.gen.OceanBaseDropIndex; +import sqlancer.oceanbase.gen.OceanBaseInsertGenerator; +import sqlancer.oceanbase.gen.OceanBaseTableGenerator; +import sqlancer.oceanbase.gen.OceanBaseTruncateTableGenerator; +import sqlancer.oceanbase.gen.OceanBaseUpdateGenerator; +import sqlancer.oceanbase.gen.datadef.OceanBaseIndexGenerator; + +public class OceanBaseProvider extends SQLProviderAdapter { + + public OceanBaseProvider() { + super(OceanBaseGlobalState.class, OceanBaseOptions.class); + } + + enum Action implements AbstractAction { + SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), + INSERT(OceanBaseInsertGenerator::insertRow), + CREATE_INDEX(OceanBaseIndexGenerator::create), + ALTER_TABLE(OceanBaseAlterTable::create), + TRUNCATE_TABLE(OceanBaseTruncateTableGenerator::generate), + SELECT_INFO((g) -> new SQLQueryAdapter( + "select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + g.getDatabaseName() + + "'")), + CREATE_TABLE((g) -> { + String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); + + return OceanBaseTableGenerator.generate(g, tableName); + }), + DELETE(OceanBaseDeleteGenerator::delete), + UPDATE(OceanBaseUpdateGenerator::getQuery), + DROP_INDEX(OceanBaseDropIndex::generate); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(OceanBaseGlobalState globalState) throws Exception { + return sqlQueryProvider.getQuery(globalState); + } + } + + private static int mapActions(OceanBaseGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed = 0; + switch (a) { + case DROP_INDEX: + nrPerformed = r.getInteger(0, 2); + break; + case SHOW_TABLES: + nrPerformed = r.getInteger(0, 1); + break; + case CREATE_TABLE: + nrPerformed = r.getInteger(0, 1); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 5); + break; + case ALTER_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case TRUNCATE_TABLE: + nrPerformed = r.getInteger(0, 2); + break; + case SELECT_INFO: + nrPerformed = r.getInteger(0, 10); + break; + case DELETE: + nrPerformed = r.getInteger(0, 10); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + } + + @Override + public void generateDatabase(OceanBaseGlobalState globalState) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = OceanBaseTableGenerator.generate(globalState, tableName); + globalState.executeStatement(createTable); + } + + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + OceanBaseProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exception,SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = OceanBaseOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = OceanBaseOptions.DEFAULT_PORT; + } + if(username.endsWith("sys")||username.equals("root")) + { + throw new Exception("please don't use sys tenant to test! Firstly create tenant then test"); + } + String databaseName = globalState.getDatabaseName(); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); + + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "oceanbase"; + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseSchema.java b/src/sqlancer/oceanbase/OceanBaseSchema.java new file mode 100644 index 000000000..a16358110 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseSchema.java @@ -0,0 +1,291 @@ +package sqlancer.oceanbase; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.IgnoreMeException; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseConstant; + +public class OceanBaseSchema extends AbstractSchema { + + private static final int NR_SCHEMA_READ_TRIES = 10; + + public enum OceanBaseDataType { + INT, VARCHAR, FLOAT, DOUBLE, DECIMAL; + + public static OceanBaseDataType getRandom(OceanBaseGlobalState globalState) { + if (globalState.usesPQS()) { + return Randomly.fromOptions(OceanBaseDataType.INT, OceanBaseDataType.VARCHAR); + } else { + return Randomly.fromOptions(values()); + } + } + + public boolean isNumeric() { + switch (this) { + case INT: + case DOUBLE: + case FLOAT: + case DECIMAL: + return true; + case VARCHAR: + return false; + default: + throw new AssertionError(this); + } + } + + } + + public static class OceanBaseColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isZeroFill; + private final int precision; + public boolean isPartioned; + + public enum CollateSequence { + NOCASE, RTRIM, BINARY; + + public static CollateSequence random() { + return Randomly.fromOptions(values()); + + } + } + + public OceanBaseColumn(String name, OceanBaseDataType columnType, boolean isPrimaryKey, int precision, boolean isZeroFill) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.precision = precision; + this.isPartioned = true; + this.isZeroFill = isZeroFill; + } + + public int getPrecision() { + return precision; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isZeroFill() { + return isZeroFill; + } + + } + + public static class OceanBaseTables extends AbstractTables { + + public OceanBaseTables(List tables) { + super(tables); + } + + public OceanBaseRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", columnNamesAsString( + c -> c.getType()==OceanBaseDataType.FLOAT || c.isZeroFill() ? ("concat(" + c.getTable().getName() + "." + c.getName() + ",'')" + " AS " + c.getTable().getName() + c.getName()) : (c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName())), + tableNamesAsString()); + //cast float and zerofill as varchar + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet randomRowValues = s.executeQuery(randomRow); + if (!randomRowValues.next()) { + throw new IgnoreMeException(); + } + for (int i = 0; i < getColumns().size(); i++) { + OceanBaseColumn column = getColumns().get(i); + Object value; + int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + OceanBaseConstant constant; + if (randomRowValues.getString(columnIndex) == null) { + if(column.isZeroFill()) + constant= OceanBaseConstant.createStringConstant("null"); + else + constant = OceanBaseConstant.createNullConstant(); + + } else { + switch (column.getType()) { + case INT: + //cast zerofill as varchar + if (column.isZeroFill()){ + value = randomRowValues.getString(columnIndex); + constant = OceanBaseConstant.createStringConstant((String) value); + }else{ + value = randomRowValues.getLong(columnIndex); + constant = OceanBaseConstant.createIntConstant((long) value); + } + break; + case VARCHAR: + value = randomRowValues.getString(columnIndex); + constant = OceanBaseConstant.createStringConstant((String) value); + break; + default: + throw new AssertionError(column.getType()); + } + } + values.put(column, constant); + } + assert !randomRowValues.next(); + return new OceanBaseRowValue(this, values); + } + + } + + } + + private static OceanBaseDataType getColumnType(String typeString) { + switch (typeString) { + case "tinyint": + case "smallint": + case "mediumint": + case "int": + case "bigint": + return OceanBaseDataType.INT; + case "varchar": + case "tinytext": + case "mediumtext": + case "text": + case "longtext": + return OceanBaseDataType.VARCHAR; + case "double": + return OceanBaseDataType.DOUBLE; + case "float": + return OceanBaseDataType.FLOAT; + case "decimal": + return OceanBaseDataType.DECIMAL; + default: + throw new AssertionError(typeString); + } + } + + public static class OceanBaseRowValue extends AbstractRowValue { + + OceanBaseRowValue(OceanBaseTables tables, Map values) { + super(tables, values); + } + + } + + public static class OceanBaseTable extends AbstractRelationalTable { + + public OceanBaseTable(String tableName, List columns, List indexes) { + super(tableName, columns, indexes, false); + } + + public boolean hasPrimaryKey() { + return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); + } + + } + + public static final class OceanBaseIndex extends TableIndex { + + private OceanBaseIndex(String indexName) { + super(indexName); + } + + public static OceanBaseIndex create(String indexName) { + return new OceanBaseIndex(indexName); + } + + @Override + public String getIndexName() { + return super.getIndexName(); + + } + + } + + public static OceanBaseSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + Exception ex = null; + for (int i = 0; i < NR_SCHEMA_READ_TRIES; i++) { + try { + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + "select TABLE_NAME from information_schema.TABLES where table_schema = '" + + databaseName + "';")) { + while (rs.next()) { + String tableName = rs.getString("TABLE_NAME"); + List databaseColumns = getTableColumns(con, tableName, databaseName); + List indexes = getIndexes(con, tableName, databaseName); + OceanBaseTable t = new OceanBaseTable(tableName, databaseColumns, indexes); + for (OceanBaseColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + return new OceanBaseSchema(databaseTables); + } catch (SQLIntegrityConstraintViolationException e) { + ex = e; + } + } + throw new AssertionError(ex); + } + + private static List getIndexes(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME='%s';", + databaseName, tableName))) { + while (rs.next()) { + String indexName = rs.getString("INDEX_NAME"); + if(!indexName.equals("PRIMARY")) + indexes.add(OceanBaseIndex.create(indexName)); + } + } + } + return indexes; + } + + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("select * from information_schema.columns where table_schema = '" + + databaseName + "' AND TABLE_NAME='" + tableName + "'")) { + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + String dataType = rs.getString("DATA_TYPE"); + int precision = rs.getInt("NUMERIC_PRECISION"); + boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); + boolean isZeroFill = rs.getString("COLUMN_TYPE").contains("zerofill"); + + OceanBaseColumn c = new OceanBaseColumn(columnName, getColumnType(dataType), isPrimaryKey, precision, isZeroFill); + columns.add(c); + } + } + } + return columns; + } + + public OceanBaseSchema(List databaseTables) { + super(databaseTables); + } + + public OceanBaseTables getRandomTableNonEmptyTables() { + return new OceanBaseTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java new file mode 100644 index 000000000..053731852 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java @@ -0,0 +1,306 @@ +package sqlancer.oceanbase; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseColumnName; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; + +public class OceanBaseToStringVisitor extends ToStringVisitor implements OceanBaseVisitor { + + int ref; + private Randomly r = new Randomly(); + + @Override + public void visitSpecific(OceanBaseExpression expr) { + OceanBaseVisitor.super.visit(expr); + } + + @Override + public void visit(OceanBaseSelect s) { + sb.append("SELECT "); + if (s.getHint() != null) { + sb.append("/*+ "); + visit(s.getHint(),0); + sb.append(" */ "); + } + + switch (s.getFromOptions()) { + case DISTINCT: + sb.append("DISTINCT "); + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + sb.append(s.getModifiers().stream().collect(Collectors.joining(" "))); + if (s.getModifiers().size() > 0) { + sb.append(" "); + } + if (s.getFetchColumns() == null) { + sb.append("*"); + } else { + for (int i = 0; i < s.getFetchColumns().size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(s.getFetchColumns().get(i)); + } + } + sb.append(" FROM "); + for (int i = 0; i < s.getFromList().size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(s.getFromList().get(i)); + } + for (OceanBaseExpression j : s.getJoinList()) { + visit(j); + } + + if (s.getWhereClause() != null) { + OceanBaseExpression whereClause = s.getWhereClause(); + sb.append(" WHERE "); + visit(whereClause); + } + if (s.getGroupByExpressions() != null && s.getGroupByExpressions().size() > 0) { + sb.append(" "); + sb.append("GROUP BY "); + List groupBys = s.getGroupByExpressions(); + for (int i = 0; i < groupBys.size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(groupBys.get(i)); + } + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + } + if (!s.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + List orderBys = s.getOrderByExpressions(); + for (int i = 0; i < orderBys.size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(s.getOrderByExpressions().get(i)); + } + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(OceanBaseConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(OceanBaseColumnReference column) { + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) + {sb.append("concat(");} + sb.append(column.getColumn().getFullQualifiedName()); + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) + {sb.append(",'')");} + if (column.getRef()){ + sb.append(" AS "); + sb.append(column.getColumn().getTable().getName()); + sb.append(column.getColumn().getName()); + } + } + + @Override + public void visit(OceanBaseUnaryPostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" IS "); + if (op.isNegated()) { + sb.append("NOT "); + } + switch (op.getOperator()) { + case IS_FALSE: + sb.append("FALSE"); + break; + case IS_NULL: + if (Randomly.getBoolean()) { + sb.append("UNKNOWN"); + } else { + sb.append("NULL"); + } + break; + case IS_TRUE: + sb.append("TRUE"); + break; + default: + throw new AssertionError(op); + } + } + + @Override + public void visit(OceanBaseComputableFunction f) { + sb.append(f.getFunction().getName()); + sb.append("("); + for (int i = 0; i < f.getArguments().length; i++) { + if (i != 0) { + sb.append(", "); + } + visit(f.getArguments()[i]); + } + sb.append(")"); + } + + @Override + public void visit(OceanBaseBinaryLogicalOperation op) { + sb.append("("); + visit(op.getLeft()); + sb.append(")"); + sb.append(" "); + sb.append(op.getTextRepresentation()); + sb.append(" "); + sb.append("("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseBinaryComparisonOperation op) { + sb.append("("); + visit(op.getLeft()); + sb.append(") "); + sb.append(op.getOp().getTextRepresentation()); + sb.append(" ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseCastOperation op) { + sb.append("CAST("); + visit(op.getExpr()); + sb.append(" AS "); + sb.append(op.getType()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN "); + sb.append("("); + for (int i = 0; i < op.getListElements().size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(op.getListElements().get(i)); + } + sb.append(")"); + } + + @Override + public void visit(OceanBaseOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder() == OceanBaseOrder.ASC ? "ASC" : "DESC"); + } + + @Override + public void visit(OceanBaseExists op) { + sb.append(" EXISTS ("); + visit(op.getExpr()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseStringExpression op) { + if(op.getStr().contains("SELECT")){ + sb.append(op.getStr()); + } + else{ + String str = op.getStr(); + if(str.length() > 0){ + sb.append(r.getInteger(0, 100000)); + } + else{ + sb.append(r.getInteger(0, 1000000)); + } + } + } + + public void visit(OceanBaseStringExpression op,int type) { + sb.append(op.getStr()); + } + + @Override + public void visit(OceanBaseTableReference ref) { + sb.append(ref.getTable().getName()); + } + + @Override + public void visit(OceanBaseAggregate aggr) { + sb.append(aggr.getAggr()); + sb.append("("); + visit(aggr.getExpr()); + sb.append(")"); + } + @Override + public void visit(OceanBaseColumnName c) { + sb.append(c.getColumn().getName()); + } + @Override + public void visit(OceanBaseText func) { + visit(func.getExpr()); + sb.append(func.getText()); + } + + @Override + public void visit(OceanBaseUnaryPrefixOperation op) { + sb.append("("); + sb.append(op.getOp().getTextRepresentation()); + sb.append(" "); + visit(op.getExpr()); + sb.append(")"); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseVisitor.java b/src/sqlancer/oceanbase/OceanBaseVisitor.java new file mode 100644 index 000000000..5babfd7df --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseVisitor.java @@ -0,0 +1,110 @@ +package sqlancer.oceanbase; + +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseColumnName; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; + +public interface OceanBaseVisitor { + + void visit(OceanBaseTableReference ref); + + void visit(OceanBaseConstant constant); + + void visit(OceanBaseColumnReference column); + + void visit(OceanBaseUnaryPostfixOperation column); + + void visit(OceanBaseComputableFunction f); + + void visit(OceanBaseBinaryLogicalOperation op); + + void visit(OceanBaseSelect select); + + void visit(OceanBaseBinaryComparisonOperation op); + + void visit(OceanBaseCastOperation op); + + void visit(OceanBaseInOperation op); + + void visit(OceanBaseOrderByTerm op); + + void visit(OceanBaseExists op); + + void visit(OceanBaseStringExpression op); + + void visit(OceanBaseAggregate aggr); + + void visit(OceanBaseColumnName c); + + void visit(OceanBaseText fun); + + void visit(OceanBaseUnaryPrefixOperation op); + + default void visit(OceanBaseExpression expr) { + if (expr instanceof OceanBaseConstant) { + visit((OceanBaseConstant) expr); + } else if (expr instanceof OceanBaseColumnReference) { + visit((OceanBaseColumnReference) expr); + } else if (expr instanceof OceanBaseUnaryPostfixOperation) { + visit((OceanBaseUnaryPostfixOperation) expr); + } else if (expr instanceof OceanBaseComputableFunction) { + visit((OceanBaseComputableFunction) expr); + } else if (expr instanceof OceanBaseBinaryLogicalOperation) { + visit((OceanBaseBinaryLogicalOperation) expr); + } else if (expr instanceof OceanBaseSelect) { + visit((OceanBaseSelect) expr); + } else if (expr instanceof OceanBaseBinaryComparisonOperation) { + visit((OceanBaseBinaryComparisonOperation) expr); + } else if (expr instanceof OceanBaseCastOperation) { + visit((OceanBaseCastOperation) expr); + } else if (expr instanceof OceanBaseInOperation) { + visit((OceanBaseInOperation) expr); + } else if (expr instanceof OceanBaseOrderByTerm) { + visit((OceanBaseOrderByTerm) expr); + } else if (expr instanceof OceanBaseExists) { + visit((OceanBaseExists) expr); + } else if (expr instanceof OceanBaseStringExpression) { + visit((OceanBaseStringExpression) expr); + } else if (expr instanceof OceanBaseTableReference) { + visit((OceanBaseTableReference) expr); + } else if (expr instanceof OceanBaseAggregate) { + visit((OceanBaseAggregate) expr); + } else if (expr instanceof OceanBaseColumnName) { + visit((OceanBaseColumnName) expr); + } else if (expr instanceof OceanBaseText) { + visit((OceanBaseText) expr); + } else if (expr instanceof OceanBaseUnaryPrefixOperation) { + visit((OceanBaseUnaryPrefixOperation) expr); + } else { + throw new AssertionError(expr); + } + } + + static String asString(OceanBaseExpression expr) { + OceanBaseToStringVisitor visitor = new OceanBaseToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + static String asExpectedValues(OceanBaseExpression expr) { + OceanBaseExpectedValueVisitor visitor = new OceanBaseExpectedValueVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java new file mode 100644 index 000000000..ddf50b745 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java @@ -0,0 +1,26 @@ +package sqlancer.oceanbase.ast; + + +public class OceanBaseAggregate implements OceanBaseExpression { + + private OceanBaseExpression expr; + private OceanBaseAggregateFunction aggr; + + public OceanBaseAggregate(OceanBaseExpression expr, OceanBaseAggregateFunction aggr) { + this.expr = expr; + this.aggr = aggr; + } + + public enum OceanBaseAggregateFunction { + COUNT + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public OceanBaseAggregateFunction getAggr() { + return aggr; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java new file mode 100644 index 000000000..3fe3bd32b --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java @@ -0,0 +1,122 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.LikeImplementationHelper; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; + +public class OceanBaseBinaryComparisonOperation implements OceanBaseExpression { + + public enum BinaryComparisonOperator { + EQUALS("=") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + return leftVal.isEquals(rightVal); + } + }, + NOT_EQUALS("!=") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant isEquals = leftVal.isEquals(rightVal); + if (isEquals.getType() == OceanBaseDataType.INT) { + return OceanBaseConstant.createIntConstant(1 - isEquals.getInt()); + } + return isEquals; + } + }, + LESS("<") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + return leftVal.isLessThan(rightVal); + } + }, + LESS_EQUALS("<=") { + + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant lessThan = leftVal.isLessThan(rightVal); + if (lessThan == null) { + return null; + } + if (lessThan.getType() == OceanBaseDataType.INT && lessThan.getInt() == 0) { + return leftVal.isEquals(rightVal); + } else { + return lessThan; + } + } + }, + GREATER(">") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant equals = leftVal.isEquals(rightVal); + if (equals.getType() == OceanBaseDataType.INT && equals.getInt() == 1) { + return OceanBaseConstant.createFalse(); + } else { + OceanBaseConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + return OceanBaseUnaryPrefixOperator.NOT.applyNotNull(applyLess); + } + } + }, + GREATER_EQUALS(">=") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant equals = leftVal.isEquals(rightVal); + if (equals.getType() == OceanBaseDataType.INT && equals.getInt() == 1) { + return OceanBaseConstant.createTrue(); + } else { + OceanBaseConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + return OceanBaseUnaryPrefixOperator.NOT.applyNotNull(applyLess); + } + } + }; + private final String textRepresentation; + + public String getTextRepresentation() { + return textRepresentation; + } + + BinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public abstract OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal); + + public static BinaryComparisonOperator getRandom() { + return Randomly.fromOptions(BinaryComparisonOperator.values()); + } + } + + private final OceanBaseExpression left; + private final OceanBaseExpression right; + private final BinaryComparisonOperator op; + + public OceanBaseBinaryComparisonOperation(OceanBaseExpression left, OceanBaseExpression right, BinaryComparisonOperator op) { + this.left = left; + this.right = right; + this.op = op; + } + + public OceanBaseExpression getLeft() { + return left; + } + + public BinaryComparisonOperator getOp() { + return op; + } + + public OceanBaseExpression getRight() { + return right; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return op.getExpectedValue(left.getExpectedValue(), right.getExpectedValue()); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java new file mode 100644 index 000000000..0cb9ded11 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java @@ -0,0 +1,110 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.Randomly; + +public class OceanBaseBinaryLogicalOperation implements OceanBaseExpression { + + private final OceanBaseExpression left; + private final OceanBaseExpression right; + private final OceanBaseBinaryLogicalOperator op; + private final String textRepresentation; + + public enum OceanBaseBinaryLogicalOperator { + AND("AND", "&&") { + @Override + public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) { + if (left.isNull() && right.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (left.isNull()) { + if (right.asBooleanNotNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createFalse(); + } + } else if (right.isNull()) { + if (left.asBooleanNotNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createFalse(); + } + } else { + return OceanBaseConstant.createBoolean(left.asBooleanNotNull() && right.asBooleanNotNull()); + } + } + }, + OR("OR", "||") { + @Override + public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) { + if (!left.isNull() && left.asBooleanNotNull()) { + return OceanBaseConstant.createTrue(); + } else if (!right.isNull() && right.asBooleanNotNull()) { + return OceanBaseConstant.createTrue(); + } else if (left.isNull() || right.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createFalse(); + } + } + }, + XOR("XOR") { + @Override + public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) { + if (left.isNull() || right.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + boolean xorVal = left.asBooleanNotNull() ^ right.asBooleanNotNull(); + return OceanBaseConstant.createBoolean(xorVal); + } + }; + + private final String[] textRepresentations; + + OceanBaseBinaryLogicalOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + + public abstract OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right); + + public static OceanBaseBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public OceanBaseBinaryLogicalOperation(OceanBaseExpression left, OceanBaseExpression right, OceanBaseBinaryLogicalOperator op) { + this.left = left; + this.right = right; + this.op = op; + this.textRepresentation = op.getTextRepresentation(); + } + + public OceanBaseExpression getLeft() { + return left; + } + + public OceanBaseBinaryLogicalOperator getOp() { + return op; + } + + public OceanBaseExpression getRight() { + return right; + } + + public String getTextRepresentation() { + return textRepresentation; + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant leftExpected = left.getExpectedValue(); + OceanBaseConstant rightExpected = right.getExpectedValue(); + if (left.getExpectedValue() == null || right.getExpectedValue() == null) { + return null; + } + return op.apply(leftExpected, rightExpected); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java new file mode 100644 index 000000000..3b5ade9a3 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java @@ -0,0 +1,36 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseCastOperation implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final CastType type; + + public enum CastType { + SIGNED, UNSIGNED; + + public static CastType getRandom() { + return SIGNED; + // return Randomly.fromOptions(CastType.values()); + } + + } + + public OceanBaseCastOperation(OceanBaseExpression expr, CastType type) { + this.expr = expr; + this.type = type; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public CastType getType() { + return type; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return expr.getExpectedValue().castAs(type); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java new file mode 100644 index 000000000..c6ee0f350 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java @@ -0,0 +1,17 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.oceanbase.OceanBaseSchema; + +public class OceanBaseColumnName implements OceanBaseExpression { + + private OceanBaseSchema.OceanBaseColumn column; + + public OceanBaseColumnName(OceanBaseSchema.OceanBaseColumn column) { + this.column = column; + } + + public OceanBaseSchema.OceanBaseColumn getColumn() { + return column; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java new file mode 100644 index 000000000..64f903267 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java @@ -0,0 +1,41 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; + +public class OceanBaseColumnReference implements OceanBaseExpression { + + private final OceanBaseColumn column; + private final OceanBaseConstant value; + private boolean isRef = false; + + public OceanBaseColumnReference(OceanBaseColumn column, OceanBaseConstant value) { + this.column = column; + this.value = value; + } + + public static OceanBaseColumnReference create(OceanBaseColumn column, OceanBaseConstant value) { + return new OceanBaseColumnReference(column, value); + } + + public OceanBaseColumn getColumn() { + return column; + } + + public OceanBaseConstant getValue() { + return value; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return value; + } + + public OceanBaseColumnReference setRef(boolean isRef){ + this.isRef = isRef; + return this; + } + public boolean getRef(){ + return isRef; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java new file mode 100644 index 000000000..ee78dd584 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java @@ -0,0 +1,270 @@ +package sqlancer.oceanbase.ast; + +import java.util.function.BinaryOperator; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseCastOperation.CastType; + +public class OceanBaseComputableFunction implements OceanBaseExpression { + + private final OceanBaseFunction func; + private final OceanBaseExpression[] args; + + public OceanBaseComputableFunction(OceanBaseFunction func, OceanBaseExpression... args) { + this.func = func; + this.args = args; + } + + public OceanBaseFunction getFunction() { + return func; + } + + public OceanBaseExpression[] getArguments() { + return args; + } + + public enum OceanBaseFunction { + + BIT_COUNT(1, "BIT_COUNT") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + OceanBaseConstant arg = evaluatedArgs[0]; + if (arg.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + long val = arg.castAs(CastType.SIGNED).getInt(); + return OceanBaseConstant.createIntConstant(Long.bitCount(val)); + } + } + + }, + COALESCE(2, "COALESCE") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + OceanBaseConstant result = OceanBaseConstant.createNullConstant(); + for (OceanBaseConstant arg : args) { + if (!arg.isNull()) { + result = arg; + break; + } + } + return castToMostGeneralType(result, origArgs); + } + + @Override + public boolean isVariadic() { + return true; + } + + }, + IF(3, "IF") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + OceanBaseConstant cond = args[0]; + OceanBaseConstant left = args[1]; + OceanBaseConstant right = args[2]; + OceanBaseConstant result; + if (cond.isNull() || !cond.asBooleanNotNull()) { + result = right; + } else { + result = left; + } + return castToMostGeneralType(result, new OceanBaseExpression[] { origArgs[1], origArgs[2] }); + } + }, + + IFNULL(2, "IFNULL") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + OceanBaseConstant result; + if (args[0].isNull()) { + result = args[1]; + } else { + result = args[0]; + }//args[0] and args[1] both null, if type is varchar, return null of varchar + return castToMostGeneralType(result, origArgs); + } + + }, + LEAST(2, "LEAST", true) { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + return aggregate(evaluatedArgs, args, (min, cur) -> cur.isLessThan(min).asBooleanNotNull() ? cur : min); + } + + }, + GREATEST(2, "GREATEST", true) { + @Override + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + return aggregate(evaluatedArgs, args, (max, cur) -> cur.isLessThan(max).asBooleanNotNull() ? max : cur); + } + }; + + private String functionName; + final int nrArgs; + private final boolean variadic; + + private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] typeExpressions, BinaryOperator op) { + boolean containsNull = Stream.of(evaluatedArgs).anyMatch(arg -> arg.isNull()); + if (containsNull) { + //IFNULL(GREATEST('aa',NULL), 0) -> '0' + // case1:c1 is float，value is NULL;select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' + // select COALESCE(GREATEST(1, concat(t1.c1)), 1) from t1;->1 + // select COALESCE(GREATEST('0', 1, concat(t1.c1)), 1) from t1;->1 + // select COALESCE(GREATEST('0', concat(t1.c1)), 1) from t1;->'1' + // select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' + // case2: c0 is decimal,value is NULL + // select IFNULL(GREATEST("iffI|2&nBJLQQ", c0, '0'), 1) from t0;->1 + // select IFNULL(GREATEST("iffI|2&nBJLQQ", NULL, '0'), 1) from t0;->'1' + OceanBaseDataType type; + boolean allVarchar = true; + for (OceanBaseExpression expr : typeExpressions) { + if (expr instanceof OceanBaseColumnReference){ + type = ((OceanBaseColumnReference) expr).getColumn().getType(); + if(type == OceanBaseDataType.FLOAT) + type = OceanBaseDataType.VARCHAR; + }else + type = expr.getExpectedValue().getType(); + if(type != null && type.isNumeric()){ + allVarchar = false; + break; + } + } + if (allVarchar) + return OceanBaseConstant.createStringConstant("null"); + else + return OceanBaseConstant.createNullConstant(); + } + OceanBaseConstant least = evaluatedArgs[1]; + /*select least(1,'H8*GPLuBjDj#Xem]W'); -> 0 + select least('1','H8*GPLuBjDj#Xem]W'); ->1 + select LEAST('000000000001', 'b', 1);->0*/ + OceanBaseDataType dataType = evaluatedArgs[0].getType(); + boolean sameDataType = true; + for (OceanBaseConstant arg : evaluatedArgs) { + if (arg.getType() != dataType){ + sameDataType = false; + break; + } + } + for (OceanBaseConstant arg : evaluatedArgs) { + OceanBaseConstant left; + OceanBaseConstant right; + if(sameDataType){ + left = least; + right = arg; + }else{ + //select GREATEST('1.47529e18', -1188315266);->1.47529e18 + if(least.getType() == OceanBaseDataType.VARCHAR) + left = least.castAsDouble(); + else + left = least; + if(arg.getType() == OceanBaseDataType.VARCHAR) + right = arg.castAsDouble(); + else + right = arg; + } + least = op.apply(right, left); + } + return least; + } + + OceanBaseFunction(int nrArgs, String functionName) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = false; + } + + OceanBaseFunction(int nrArgs, String functionName, boolean variadic) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = variadic; + } + + public int getNrArgs() { + return nrArgs; + } + + public abstract OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args); + + public static OceanBaseFunction getRandomFunction() { + return Randomly.fromOptions(values()); + } + + @Override + public String toString() { + return functionName; + } + + public boolean isVariadic() { + return variadic; + } + + public String getName() { + return functionName; + } + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant[] constants = new OceanBaseConstant[args.length]; + for (int i = 0; i < constants.length; i++) { + constants[i] = args[i].getExpectedValue(); + } + return func.apply(constants, args); + } + + public static OceanBaseConstant castToMostGeneralType(OceanBaseConstant cons, OceanBaseExpression... typeExpressions) { + OceanBaseDataType type = getMostGeneralType(typeExpressions); + if (cons.isNull()) { + if (type == OceanBaseDataType.FLOAT || type == OceanBaseDataType.VARCHAR) + return OceanBaseConstant.createStringConstant("null"); + else + return cons; + }else{ + switch (type) { + case INT: + if (cons.isInt()) { + return cons; + } else { + return OceanBaseConstant.createIntConstant(cons.castAs(CastType.SIGNED).getInt()); + } + case VARCHAR: + return OceanBaseConstant.createStringConstant(cons.castAsString()); + default: + return cons; + } + } + } + + public static OceanBaseDataType getMostGeneralType(OceanBaseExpression... expressions) { + OceanBaseDataType type = null; + for (OceanBaseExpression expr : expressions) { + OceanBaseDataType exprType; + if (expr instanceof OceanBaseColumnReference) { + exprType = ((OceanBaseColumnReference) expr).getColumn().getType(); + if(((OceanBaseColumnReference) expr).getColumn().isZeroFill()) + exprType = OceanBaseDataType.VARCHAR; + } else { + exprType = expr.getExpectedValue().getType(); + } + if (type == null) { + type = exprType; + if (exprType == OceanBaseDataType.FLOAT) + type = OceanBaseDataType.VARCHAR; + } else if (exprType == OceanBaseDataType.VARCHAR || exprType == OceanBaseDataType.FLOAT) { + type = OceanBaseDataType.VARCHAR; + } + + } + return type; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java new file mode 100644 index 000000000..d99447fb6 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -0,0 +1,628 @@ +package sqlancer.oceanbase.ast; + +import java.math.BigInteger; +import java.math.BigDecimal; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseCastOperation.CastType; + +public abstract class OceanBaseConstant implements OceanBaseExpression { + + public boolean isInt() { + return false; + } + + public boolean isNull() { + return false; + } + + public boolean isDouble() { + return false; + } + + public boolean isEmpty() { + return false; + } + + public abstract static class OceanBaseNoPQSConstant extends OceanBaseConstant { + + @Override + public boolean asBooleanNotNull() { + throw throwException(); + } + + private RuntimeException throwException() { + throw new UnsupportedOperationException("not applicable for PQS evaluation!"); + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + return null; + } + + @Override + public OceanBaseConstant castAs(CastType type) { + throw throwException(); + } + + @Override + public String castAsString() { + throw throwException(); + + } + + @Override + public OceanBaseConstant castAsDouble() { + throw throwException(); + } + + @Override + public OceanBaseDataType getType() { + throw throwException(); + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + throw throwException(); + } + + } + + public static class OceanBaseDoubleConstant extends OceanBaseNoPQSConstant { + + private final double val; + + public OceanBaseDoubleConstant(double val) { + this.val = val; + if (Double.isInfinite(val) || Double.isNaN(val)) { + throw new IgnoreMeException(); + } + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public double getDouble() { + return this.val; + } + + @Override + public long getInt() { + return new Double(val).longValue(); + } + + @Override + public boolean asBooleanNotNull() { + return Double.compare(Math.abs(val), 0.0) != 0; + } + + @Override + public OceanBaseConstant castAs(CastType type) { + if (type == CastType.SIGNED) { + long value = new Double(val).longValue(); + if (val - value >=0.5) + value = value +1; + return new OceanBaseIntConstant(value, true); + } else if (type == CastType.UNSIGNED) { + long value = new Double(val).longValue(); + if (val - value >=0.5) + value = value +1; + return new OceanBaseIntConstant(value, false); + } else { + throw new AssertionError(); + } + } + + @Override + public String castAsString() { + return String.valueOf(new BigDecimal(val));//select IFNULL(1.713591018E9, '11') -> 1713591018 + } + + @Override + public boolean isDouble() { + return true; + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal instanceof OceanBaseIntConstant){ + return OceanBaseConstant.createBoolean(val < rightVal.getInt()); + } else if (rightVal instanceof OceanBaseDoubleConstant){ + return OceanBaseConstant.createBoolean(val < rightVal.getDouble()); + } else if (rightVal instanceof OceanBaseTextConstant){ + return isLessThan(rightVal.castAsDouble()); + } else{ + throw new AssertionError(rightVal); + } + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal instanceof OceanBaseIntConstant){ + return OceanBaseConstant.createBoolean(val == rightVal.getInt()); + } else if (rightVal instanceof OceanBaseDoubleConstant){ + return OceanBaseConstant.createBoolean(val == rightVal.getDouble()); + } else if (rightVal instanceof OceanBaseTextConstant){ + return isEquals(rightVal.castAsDouble()); + } else { + throw new AssertionError(rightVal); + } + } + @Override + public OceanBaseDataType getType() { + return OceanBaseDataType.DOUBLE; + } + + } + + public static class OceanBaseTextConstant extends OceanBaseConstant { + + private final String value; + private final boolean singleQuotes; + + public OceanBaseTextConstant(String value) { + this.value = value; + singleQuotes = Randomly.getBoolean(); + + } + + private void checkIfSmallFloatingPointText() { + boolean isSmallFloatingPointText = isString() && asBooleanNotNull() + && castAs(CastType.SIGNED).getInt() == 0; + if (isSmallFloatingPointText) { + throw new IgnoreMeException(); + } + } + @Override + public boolean isNull() { + if(value.equalsIgnoreCase("NULL")) + return true; + else + return false; + } + + @Override + public boolean isEmpty(){ + //"" " " + if (value.length() ==0 ){ + return true; + }else{ + for(int i =0;i< value.length(); i++){ + String sub =value.substring(i, i+1); + if(!sub.equals(" ")) + return false; + } + return true; + } + } + + @Override + public boolean asBooleanNotNull() { + for (int i = value.length(); i >= 1; i--) { + try { + char currentChar = value.charAt(i-1); + int currentVal= Integer.valueOf(currentChar); + if (currentVal < 48||currentVal > 57) + continue; + String substring = value.substring(0, i); + Double val = Double.valueOf(substring); + return val != 0 && !Double.isNaN(val); + } catch (NumberFormatException e) { + // ignore + } + } + return false; + } + + @Override + public String getTextRepresentation() { + StringBuilder sb = new StringBuilder(); + String quotes = singleQuotes ? "'" : "\""; + sb.append(quotes); + String text = value.replace(quotes, quotes + quotes).replace("\\", "\\\\"); + sb.append(text); + sb.append(quotes); + return sb.toString(); + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + if (isNull() || rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isInt()) { + checkIfSmallFloatingPointText(); + if (asBooleanNotNull()) { + throw new IgnoreMeException(); + } + return castAs(CastType.SIGNED).isEquals(rightVal); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return castAsDouble().isEquals(rightVal); + } else if (rightVal.isString()) { + if (isEmpty() && rightVal.isEmpty()) + return OceanBaseConstant.createBoolean(true); + else + return OceanBaseConstant.createBoolean(value.equalsIgnoreCase(rightVal.getString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public String getString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public OceanBaseConstant castAs(CastType type) { + if(isNull()) + return OceanBaseConstant.createNullConstant(); + if (type == CastType.SIGNED || type == CastType.UNSIGNED) { + String value = this.value; + while (value.startsWith(" ") || value.startsWith("\t") || value.startsWith("\n")) { + if (value.startsWith("\n")) { + throw new IgnoreMeException(); + } + value = value.substring(1); + } + for (int i = value.length(); i >= 1; i--) { + try { + //select CAST("꯵z)" AS SIGNED); ->0 + char currentChar = value.charAt(i-1); + int currentVal= Integer.valueOf(currentChar); + if (currentVal < 48||currentVal > 57) + throw new IgnoreMeException(); + String substring = value.substring(0, i); + long val = Long.parseLong(substring); + return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); + } catch (NumberFormatException e) { + throw new IgnoreMeException(); + } + } + return OceanBaseConstant.createIntConstant(0, type == CastType.SIGNED ? true : false); + } else { + throw new AssertionError(); + } + } + + @Override + public OceanBaseConstant castAsDouble() { + String value = this.value; + while (value.startsWith(" ") || value.startsWith("\t") || value.startsWith("\n")) { + if (value.startsWith("\n")) { + throw new IgnoreMeException(); + } + value = value.substring(1); + } + for (int i = value.length(); i >= 1; i--) { + try { + char currentChar = value.charAt(i-1); + int currentVal= Integer.valueOf(currentChar); + if (currentVal < 48||currentVal > 57) + //ignore special char + throw new IgnoreMeException(); + String substring = value.substring(0, i); + double val = Double.parseDouble(substring); + return OceanBaseConstant.createDoubleConstant(val); + } catch (NumberFormatException e) { + throw new IgnoreMeException(); + } + } + return OceanBaseConstant.createIntConstant(0); + } + + @Override + public String castAsString() { + return value; + } + + @Override + public OceanBaseDataType getType() { + return OceanBaseDataType.VARCHAR; + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + if (isNull() || rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isInt()) { + if (asBooleanNotNull()) { + throw new IgnoreMeException(); + } + checkIfSmallFloatingPointText(); + return castAs(rightVal.isSigned() ? CastType.SIGNED : CastType.UNSIGNED).isLessThan(rightVal); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return castAsDouble().isLessThan(rightVal); + } else if (rightVal.isString()) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(rightVal); + } + } + } + + public static class OceanBaseIntConstant extends OceanBaseConstant { + + private final long value; + private final String stringRepresentation; + private final boolean isSigned; + + public OceanBaseIntConstant(long value, boolean isSigned) { + this.value = value; + this.isSigned = isSigned; + if (value == 0 && Randomly.getBoolean()) { + stringRepresentation = "FALSE"; + } else if (value == 1 && Randomly.getBoolean()) { + stringRepresentation = "TRUE"; + } else { + if (isSigned) { + stringRepresentation = String.valueOf(value); + } else { + stringRepresentation = Long.toUnsignedString(value); + } + } + } + + public OceanBaseIntConstant(long value, String stringRepresentation) { + this.value = value; + this.stringRepresentation = stringRepresentation; + isSigned = true; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public long getInt() { + return value; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public String getTextRepresentation() { + return stringRepresentation; + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + if (rightVal.isInt()) { + return OceanBaseConstant.createBoolean(new BigInteger(getStringRepr()) + .compareTo(new BigInteger(((OceanBaseIntConstant) rightVal).getStringRepr())) == 0); + } else if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isString()) { + if (rightVal.asBooleanNotNull()) { + throw new IgnoreMeException(); + } + return isEquals(rightVal.castAs(CastType.SIGNED)); + } else if (rightVal instanceof OceanBaseDoubleConstant){ + return OceanBaseConstant.createBoolean(value == rightVal.getDouble()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public OceanBaseConstant castAs(CastType type) { + if (type == CastType.SIGNED) { + return new OceanBaseIntConstant(value, true); + } else if (type == CastType.UNSIGNED) { + return new OceanBaseIntConstant(value, false); + } else { + throw new AssertionError(); + } + } + + @Override + public String castAsString() { + if (isSigned) { + return String.valueOf(value); + } else { + return Long.toUnsignedString(value); + } + } + + @Override + public OceanBaseConstant castAsDouble() { + return this; + } + + @Override + public OceanBaseDataType getType() { + return OceanBaseDataType.INT; + } + + @Override + public boolean isSigned() { + return isSigned; + } + + private String getStringRepr() { + if (isSigned) { + return String.valueOf(value); + } else { + return Long.toUnsignedString(value); + } + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + if (rightVal.isInt()) { + long intVal = rightVal.getInt(); + if (isSigned && rightVal.isSigned()) { + return OceanBaseConstant.createBoolean(value < intVal); + } else { + return OceanBaseConstant.createBoolean(new BigInteger(getStringRepr()) + .compareTo(new BigInteger(((OceanBaseIntConstant) rightVal).getStringRepr())) < 0); + } + } else if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isString()) { + if (rightVal.asBooleanNotNull()) { + throw new IgnoreMeException(); + } + return isLessThan(rightVal.castAs(isSigned ? CastType.SIGNED : CastType.UNSIGNED)); + } else if (rightVal instanceof OceanBaseDoubleConstant){ + return OceanBaseConstant.createBoolean(value < rightVal.getDouble()); + } else { + throw new AssertionError(rightVal); + } + } + + } + + public static class OceanBaseNullConstant extends OceanBaseConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public boolean asBooleanNotNull() { + throw new UnsupportedOperationException(this.toString()); + } + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + return OceanBaseConstant.createNullConstant(); + } + + @Override + public OceanBaseConstant castAs(CastType type) { + return this; + } + + @Override + public String castAsString() { + return "NULL"; + } + + @Override + public OceanBaseConstant castAsDouble() { + return this; + } + + @Override + public OceanBaseDataType getType() { + return null; + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + return this; + } + + } + + public long getInt() { + throw new UnsupportedOperationException(); + } + + public double getDouble() { + throw new UnsupportedOperationException(); + } + + public boolean isSigned() { + return false; + } + + public String getString() { + throw new UnsupportedOperationException(); + } + + public boolean isString() { + return false; + } + + public static OceanBaseConstant createNullConstant() { + return new OceanBaseNullConstant(); + } + + public static OceanBaseConstant createIntConstant(long value) { + return new OceanBaseIntConstant(value, true); + } + + public static OceanBaseConstant createIntConstant(long value, boolean signed) { + return new OceanBaseIntConstant(value, signed); + } + + public static OceanBaseConstant createUnsignedIntConstant(long value) { + return new OceanBaseIntConstant(value, false); + } + + public static OceanBaseConstant createIntConstantNotAsBoolean(long value) { + return new OceanBaseIntConstant(value, String.valueOf(value)); + } + + public static OceanBaseConstant createDoubleConstant(double value) { + return new OceanBaseDoubleConstant(value); + } + + @Override + public OceanBaseConstant getExpectedValue() { + return this; + } + + public abstract boolean asBooleanNotNull(); + + public abstract String getTextRepresentation(); + + public static OceanBaseConstant createFalse() { + return OceanBaseConstant.createIntConstant(0); + } + + public static OceanBaseConstant createBoolean(boolean isTrue) { + return OceanBaseConstant.createIntConstant(isTrue ? 1 : 0); + } + + public static OceanBaseConstant createTrue() { + return OceanBaseConstant.createIntConstant(1); + } + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract OceanBaseConstant isEquals(OceanBaseConstant rightVal); + + public abstract OceanBaseConstant castAs(CastType type); + + public abstract String castAsString(); + + public abstract OceanBaseConstant castAsDouble(); + + public static OceanBaseConstant createStringConstant(String string) { + return new OceanBaseTextConstant(string); + } + + public abstract OceanBaseDataType getType(); + + protected abstract OceanBaseConstant isLessThan(OceanBaseConstant rightVal); + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseExists.java b/src/sqlancer/oceanbase/ast/OceanBaseExists.java new file mode 100644 index 000000000..18595d261 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseExists.java @@ -0,0 +1,30 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseExists implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final OceanBaseConstant expected; + + public OceanBaseExists(OceanBaseExpression expr, OceanBaseConstant expectedValue) { + this.expr = expr; + this.expected = expectedValue; + } + + public OceanBaseExists(OceanBaseExpression expr) { + this.expr = expr; + this.expected = expr.getExpectedValue(); + if (expected == null) { + throw new AssertionError(); + } + } + + public OceanBaseExpression getExpr() { + return expr; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return expected; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseExpression.java b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java new file mode 100644 index 000000000..195a05967 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java @@ -0,0 +1,9 @@ +package sqlancer.oceanbase.ast; + +public interface OceanBaseExpression { + + default OceanBaseConstant getExpectedValue() { + throw new AssertionError("PQS not supported for this operator"); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java new file mode 100644 index 000000000..e5c482a56 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java @@ -0,0 +1,58 @@ +package sqlancer.oceanbase.ast; + +import java.util.List; + +import sqlancer.IgnoreMeException; + +public class OceanBaseInOperation implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final List listElements; + private final boolean isTrue; + + public OceanBaseInOperation(OceanBaseExpression expr, List listElements, boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant leftVal = expr.getExpectedValue(); + if (leftVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + boolean isNull = false; + for (OceanBaseExpression rightExpr : listElements) { + OceanBaseConstant rightVal = rightExpr.getExpectedValue(); + + OceanBaseConstant convertedRightVal = rightVal; + OceanBaseConstant isEquals = leftVal.isEquals(convertedRightVal); + if (isEquals.isNull()) { + isNull = true; + } else { + if (isEquals.getInt() == 1) { + return OceanBaseConstant.createBoolean(isTrue); + } + } + } + if (isNull) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createBoolean(!isTrue); + } + + } + + public boolean isTrue() { + return isTrue; + } +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseJoin.java b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java new file mode 100644 index 000000000..9e8271bbf --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java @@ -0,0 +1,10 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseJoin implements OceanBaseExpression { + + @Override + public OceanBaseConstant getExpectedValue() { + throw new UnsupportedOperationException(); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java b/src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java new file mode 100644 index 000000000..b8f2d029a --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java @@ -0,0 +1,36 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.Randomly; + +public class OceanBaseOrderByTerm implements OceanBaseExpression { + + private final OceanBaseOrder order; + private final OceanBaseExpression expr; + + public enum OceanBaseOrder { + ASC, DESC; + + public static OceanBaseOrder getRandomOrder() { + return Randomly.fromOptions(OceanBaseOrder.values()); + } + } + + public OceanBaseOrderByTerm(OceanBaseExpression expr, OceanBaseOrder order) { + this.expr = expr; + this.order = order; + } + + public OceanBaseOrder getOrder() { + return order; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + @Override + public OceanBaseConstant getExpectedValue() { + throw new AssertionError(this); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java new file mode 100644 index 000000000..642553173 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java @@ -0,0 +1,58 @@ +package sqlancer.oceanbase.ast; + +import java.util.Collections; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.ast.SelectBase; + +public class OceanBaseSelect extends SelectBase implements OceanBaseExpression { + + private SelectType fromOptions = SelectType.ALL; + private List modifiers = Collections.emptyList(); + private List groupBys = new ArrayList<>(); + private OceanBaseStringExpression hint; + + public enum SelectType { + DISTINCT, ALL; + } + + public void setSelectType(SelectType fromOptions) { + this.setFromOptions(fromOptions); + } + + public SelectType getFromOptions() { + return fromOptions; + } + + public void setFromOptions(SelectType fromOptions) { + this.fromOptions = fromOptions; + } + public void setGroupByClause(List groupBys){ + this.groupBys = groupBys; + } + public List getGroupByClause(){ + return this.groupBys; + } + public void setModifiers(List modifiers) { + this.modifiers = modifiers; + } + + public List getModifiers() { + return modifiers; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return null; + } + + public void setHint(OceanBaseStringExpression hint) { + this.hint = hint; + } + + public OceanBaseStringExpression getHint() { + return hint; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java b/src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java new file mode 100644 index 000000000..3c1a55176 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java @@ -0,0 +1,22 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseStringExpression implements OceanBaseExpression { + + private final String str; + private final OceanBaseConstant expectedValue; + + public OceanBaseStringExpression(String str, OceanBaseConstant expectedValue) { + this.str = str; + this.expectedValue = expectedValue; + } + + public String getStr() { + return str; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return expectedValue; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseTableReference.java b/src/sqlancer/oceanbase/ast/OceanBaseTableReference.java new file mode 100644 index 000000000..8102ca1cc --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseTableReference.java @@ -0,0 +1,17 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseTableReference implements OceanBaseExpression { + + private final OceanBaseTable table; + + public OceanBaseTableReference(OceanBaseTable table) { + this.table = table; + } + + public OceanBaseTable getTable() { + return table; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseText.java b/src/sqlancer/oceanbase/ast/OceanBaseText.java new file mode 100644 index 000000000..566effde4 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseText.java @@ -0,0 +1,27 @@ +package sqlancer.oceanbase.ast; + + +public class OceanBaseText implements OceanBaseExpression { + + private OceanBaseExpression expr; + private String text; + private boolean prefix; + + public OceanBaseText(OceanBaseExpression expr, String text, boolean prefix) { + this.expr = expr; + this.text = text; + this.prefix = prefix; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + public boolean isPrefix() { + return prefix; + } +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java new file mode 100644 index 000000000..a2c3b6a57 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java @@ -0,0 +1,58 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseUnaryPostfixOperation implements OceanBaseExpression { + + private final OceanBaseExpression expression; + private final UnaryPostfixOperator operator; + private boolean negate; + + public enum UnaryPostfixOperator { + IS_NULL, IS_TRUE, IS_FALSE; + } + + public OceanBaseUnaryPostfixOperation(OceanBaseExpression expr, UnaryPostfixOperator op, boolean negate) { + this.expression = expr; + this.operator = op; + this.setNegate(negate); + } + + public OceanBaseExpression getExpression() { + return expression; + } + + public UnaryPostfixOperator getOperator() { + return operator; + } + + public boolean isNegated() { + return negate; + } + + public void setNegate(boolean negate) { + this.negate = negate; + } + + @Override + public OceanBaseConstant getExpectedValue() { + boolean val; + OceanBaseConstant expectedValue = expression.getExpectedValue(); + switch (operator) { + case IS_NULL: + val = expectedValue.isNull(); + break; + case IS_FALSE: + val = !expectedValue.isNull() && !expectedValue.asBooleanNotNull(); + break; + case IS_TRUE: + val = !expectedValue.isNull() && expectedValue.asBooleanNotNull(); + break; + default: + throw new AssertionError(operator); + } + if (negate) { + val = !val; + } + return OceanBaseConstant.createIntConstant(val ? 1 : 0); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java new file mode 100644 index 000000000..c598e3bd0 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java @@ -0,0 +1,91 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; + +public class OceanBaseUnaryPrefixOperation extends UnaryOperatorNode + implements OceanBaseExpression { + + public enum OceanBaseUnaryPrefixOperator implements Operator { + NOT("!", "NOT") { + @Override + public OceanBaseConstant applyNotNull(OceanBaseConstant expr) { + return OceanBaseConstant.createIntConstant(expr.asBooleanNotNull() ? 0 : 1); + } + }, + PLUS("+") { + @Override + public OceanBaseConstant applyNotNull(OceanBaseConstant expr) { + return expr; + } + }, + MINUS("-") { + @Override + public OceanBaseConstant applyNotNull(OceanBaseConstant expr) { + if (expr.isString()) { + throw new IgnoreMeException(); + } else if (expr.isInt()) { + if (!expr.isSigned()) { + throw new IgnoreMeException(); + } + return OceanBaseConstant.createIntConstant(-expr.getInt()); + } else if (expr.isDouble()) { + return OceanBaseConstant.createDoubleConstant(-expr.getDouble()); + } else { + throw new AssertionError(expr); + } + } + }; + + private String[] textRepresentations; + + OceanBaseUnaryPrefixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public abstract OceanBaseConstant applyNotNull(OceanBaseConstant expr); + + public static OceanBaseUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + } + + public OceanBaseUnaryPrefixOperation(OceanBaseExpression expr, OceanBaseUnaryPrefixOperator op) { + super(expr, op); + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant subExprVal = expr.getExpectedValue(); + if (op == OceanBaseUnaryPrefixOperator.PLUS){ + if (subExprVal.isNull() && subExprVal.getType() == null) + return OceanBaseConstant.createNullConstant(); + else + return subExprVal; + } + if (subExprVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return op.applyNotNull(subExprVal); + } + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.PREFIX; + } + public OceanBaseExpression getExpr() { + return expr; + } + public OceanBaseUnaryPrefixOperator getOp() { + return op; + } +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java new file mode 100644 index 000000000..b3f7de7bb --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java @@ -0,0 +1,72 @@ +package sqlancer.oceanbase.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseAlterTable { + + private final OceanBaseSchema schema; + private final StringBuilder sb = new StringBuilder(); + boolean couldAffectSchema; + private List selectedActions; + + public OceanBaseAlterTable(OceanBaseSchema newSchema) { + this.schema = newSchema; + } + + public static SQLQueryAdapter create(OceanBaseGlobalState globalState) { + return new OceanBaseAlterTable(globalState.getSchema()).create(); + } + + private enum Action { + COMPRESSION; + private String[] potentialErrors; + + Action(String... couldCauseErrors) { + this.potentialErrors = couldCauseErrors.clone(); + } + + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = ExpectedErrors.from("does not support the create option", "doesn't have this option", + "is not supported for this operation", "Data truncation", "Specified key was too long"); + errors.add("Data truncated for functional index "); + sb.append("ALTER TABLE "); + OceanBaseTable table = schema.getRandomTable(); + sb.append(table.getName()); + sb.append(" "); + List list = new ArrayList<>(Arrays.asList(Action.values())); + selectedActions = Randomly.subset(list); + int i = 0; + for (Action a : selectedActions) { + if (i++ != 0) { + sb.append(", "); + } + switch (a) { + case COMPRESSION: + sb.append("COMPRESSION "); + sb.append("'"); + sb.append(Randomly.fromOptions("ZLIB_1.0", "LZ4_1.0", "NONE")); + sb.append("'"); + break; + } + } + for (Action a : selectedActions) { + for (String error : a.potentialErrors) { + errors.add(error); + } + } + return new SQLQueryAdapter(sb.toString(), errors, couldAffectSchema); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java new file mode 100644 index 000000000..de76917e8 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -0,0 +1,52 @@ +package sqlancer.oceanbase.gen; + +import java.util.Arrays; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseDeleteGenerator { + + private final StringBuilder sb = new StringBuilder(); + private final OceanBaseGlobalState globalState; + private final Randomly r; + public OceanBaseDeleteGenerator(OceanBaseGlobalState globalState) { + this.globalState = globalState; + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter delete(OceanBaseGlobalState globalState) { + return new OceanBaseDeleteGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + OceanBaseTable randomTable = globalState.getSchema().getRandomTable(); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(randomTable.getColumns()); + ExpectedErrors errors = new ExpectedErrors(); + sb.append("DELETE"); + if (Randomly.getBoolean()) { + sb.append(" /*+parallel("+r.getLong(0, 10)+") enable_parallel_dml*/ "); + } + sb.append(" FROM "); + sb.append(randomTable.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + OceanBaseErrors.addExpressionErrors(errors); + } + errors.addAll(Arrays.asList("doesn't have this option", + "Truncated incorrect DOUBLE value", + "Truncated incorrect INTEGER value", + "Truncated incorrect DECIMAL value", + "Data truncated for functional index", + "Incorrect value", "Out of range value for column", + "Data truncation: %s value is out of range in '%s'")); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java new file mode 100644 index 000000000..5e7c06fb2 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java @@ -0,0 +1,30 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public final class OceanBaseDropIndex { + + private OceanBaseDropIndex() { + } + + public static SQLQueryAdapter generate(OceanBaseGlobalState globalState) { + OceanBaseTable table = globalState.getSchema().getRandomTable(); + if (!table.hasIndexes()) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder(); + sb.append("DROP INDEX "); + sb.append(table.getRandomIndex().getIndexName()); + sb.append(" ON "); + sb.append(table.getName()); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("LOCK=NONE is not supported", "ALGORITHM=INPLACE is not supported", + "Data truncation", "Data truncated for functional index")); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java new file mode 100644 index 000000000..279223d73 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java @@ -0,0 +1,212 @@ +package sqlancer.oceanbase.gen; + +import java.util.ArrayList; +import java.util.List; + +import java.sql.Connection; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseRowValue; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation.BinaryComparisonOperator; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction.OceanBaseFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseConstant.OceanBaseDoubleConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; + +public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator { + + private OceanBaseGlobalState state; + private OceanBaseRowValue rowVal; + private Randomly r; + private List OceanBaseColumns = new ArrayList<>(); + + public OceanBaseExpressionGenerator(OceanBaseGlobalState state) { + this.state = state; + } + public OceanBaseExpressionGenerator setCon(Connection con) { + return this; + } + public OceanBaseExpressionGenerator setState(OceanBaseGlobalState state) { + this.state = state; + return this; + } + + public OceanBaseExpressionGenerator setOceanBaseColumns(List columns) { + this.OceanBaseColumns = columns; + return this; + } + public OceanBaseExpressionGenerator(Randomly r) { + this.r = r; + } + + public OceanBaseExpressionGenerator setRowVal(OceanBaseRowValue rowVal) { + this.rowVal = rowVal; + return this; + } + + private enum Actions { + COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, + BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, EXISTS; + } + + @Override + public OceanBaseExpression generateExpression(int depth) { + if (depth >= state.getOptions().getMaxExpressionDepth()) { + return generateLeafNode(); + } + switch (Randomly.fromOptions(Actions.values())) { + case COLUMN: + return generateColumn(); + case LITERAL: + return generateConstant(); + case UNARY_PREFIX_OPERATION: + OceanBaseExpression subExpr = generateExpression(depth + 1); + OceanBaseUnaryPrefixOperator random = OceanBaseUnaryPrefixOperator.getRandom(); + return new OceanBaseUnaryPrefixOperation(subExpr, random); + case UNARY_POSTFIX: + return new OceanBaseUnaryPostfixOperation(generateExpression(depth + 1), + Randomly.fromOptions(OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.values()), + Randomly.getBoolean()); + case COMPUTABLE_FUNCTION: + return getComputableFunction(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return new OceanBaseBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), + OceanBaseBinaryLogicalOperator.getRandom()); + case BINARY_COMPARISON_OPERATION: + return new OceanBaseBinaryComparisonOperation(generateExpression(depth + 1), generateExpression(depth + 1), + BinaryComparisonOperator.getRandom()); + case CAST: + return new OceanBaseCastOperation(generateExpression(depth + 1), OceanBaseCastOperation.CastType.getRandom()); + case IN_OPERATION: + OceanBaseExpression expr = generateExpression(depth + 1); + List rightList = new ArrayList<>(); + for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { + rightList.add(generateExpression(depth + 1)); + } + return new OceanBaseInOperation(expr, rightList, Randomly.getBoolean()); + case EXISTS: + return getExists(); + default: + throw new AssertionError(); + } + } + + private OceanBaseExpression getExists() { + if (Randomly.getBoolean()) { + return new OceanBaseExists(new OceanBaseStringExpression("SELECT 1", OceanBaseConstant.createTrue())); + } else { + return new OceanBaseExists(new OceanBaseStringExpression("SELECT 1 from dual wHERE FALSE", OceanBaseConstant.createFalse())); + } + } + + private OceanBaseExpression getComputableFunction(int depth) { + OceanBaseFunction func = OceanBaseFunction.getRandomFunction(); + int nrArgs = func.getNrArgs(); + if (func.isVariadic()) { + nrArgs += Randomly.smallNumber(); + } + OceanBaseExpression[] args = new OceanBaseExpression[nrArgs]; + for (int i = 0; i < args.length; i++) { + args[i] = generateExpression(depth + 1); + } + return new OceanBaseComputableFunction(func, args); + } + + private enum ConstantType { + INT, NULL, STRING, DOUBLE; + public static ConstantType[] valuesPQS() { + return new ConstantType[] { INT, NULL, STRING }; + } + } + + @Override + public OceanBaseExpression generateConstant() { + ConstantType[] values; + if (state.usesPQS()) { + values = ConstantType.valuesPQS(); + } else { + values = ConstantType.values(); + } + switch (Randomly.fromOptions(values)) { + case INT: + return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); + case NULL: + return OceanBaseConstant.createNullConstant(); + case STRING: + String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); + OceanBaseConstant createStringConstant = OceanBaseConstant.createStringConstant(string); + return createStringConstant; + case DOUBLE: + double val = state.getRandomly().getDouble(); + return new OceanBaseDoubleConstant(val); + default: + throw new AssertionError(); + } + } + + @Override + public OceanBaseExpression generateColumn() { + OceanBaseColumn c = Randomly.fromList(columns); + OceanBaseConstant val; + if (rowVal == null) { + val = OceanBaseConstant.createNullConstant(); + } else { + val = rowVal.getValues().get(c); + } + return OceanBaseColumnReference.create(c, val); + } + + public OceanBaseExpression generateConstant(OceanBaseColumn col) { + switch (col.getType().name()) { + case "INT": + return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); + case "NULL": + return OceanBaseConstant.createNullConstant(); + case "VARCHAR": + String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); + OceanBaseConstant createStringConstant = OceanBaseConstant.createStringConstant(string); + return createStringConstant; + case "DOUBLE": + double val = state.getRandomly().getDouble(); + return new OceanBaseDoubleConstant(val); + case "FLOAT": + val = state.getRandomly().getDouble(); + return new OceanBaseDoubleConstant(val); + case "DECIMAL": + val = state.getRandomly().getDouble(); + return new OceanBaseDoubleConstant(val); + default: + throw new AssertionError(); + } + } + + @Override + public OceanBaseExpression negatePredicate(OceanBaseExpression predicate) { + return new OceanBaseUnaryPrefixOperation(predicate, OceanBaseUnaryPrefixOperator.NOT); + } + + @Override + public OceanBaseExpression isNull(OceanBaseExpression expr) { + return new OceanBaseUnaryPostfixOperation(expr, OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + } +} + diff --git a/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java new file mode 100644 index 000000000..aca0511ce --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java @@ -0,0 +1,110 @@ +package sqlancer.oceanbase.gen; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseInsertGenerator { + + private final OceanBaseTable table; + private final StringBuilder sb = new StringBuilder(); + private final ExpectedErrors errors = new ExpectedErrors(); + private final OceanBaseGlobalState globalState; + private final Randomly r; + private int type = 0; + + public OceanBaseInsertGenerator(OceanBaseGlobalState globalState) { + this.globalState = globalState; + table = globalState.getSchema().getRandomTable(); + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter insertRow(OceanBaseGlobalState globalState) throws SQLException { + if (Randomly.getBoolean()) { + return new OceanBaseInsertGenerator(globalState).generateInsert(); + } else { + return new OceanBaseInsertGenerator(globalState).generateReplace(); + } + } + + private SQLQueryAdapter generateReplace() { + sb.append("REPLACE"); + type = 1; + return generateInto(); + + } + + private SQLQueryAdapter generateInsert() { + sb.append("INSERT"); + if (Randomly.getBoolean()) { + sb.append(" /*+parallel("+r.getLong(0, 10)+") enable_parallel_dml*/ "); + } + if (Randomly.getBoolean()) { + sb.append(" "); + } + return generateInto(); + } + + private SQLQueryAdapter generateInto() { + sb.append(" INTO "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(") "); + sb.append("VALUES"); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + int nrRows; + if (Randomly.getBoolean()) { + nrRows = 1; + } else { + nrRows = 1 + Randomly.smallNumber(); + } + for (int row = 0; row < nrRows; row++) { + if (row != 0) { + sb.append(", "); + } + sb.append("("); + for (int c = 0; c < columns.size(); c++) { + if (c != 0) { + sb.append(", "); + } + OceanBaseColumn col = columns.get(c); + sb.append(OceanBaseVisitor.asString(gen.generateConstant(col))); + + } + sb.append(")"); + } + if(Randomly.getBoolean() && type == 0){ + + List upcolumns = table.getRandomNonEmptyColumnSubset(); + if (upcolumns.size() > 0 ){ + sb.append(" ON DUPLICATE KEY UPDATE "); + + sb.append(upcolumns.get(0).getName()); + sb.append("="); + sb.append(gen.generateConstant(upcolumns.get(0))); + } + } + errors.add("doesn't have a default value"); + errors.add("Data truncation"); + errors.add("Incorrect integer value"); + errors.add("Duplicate entry"); + errors.add("Data truncated for functional index"); + errors.add("Data truncated for column"); + errors.add("cannot be null"); + errors.add("Incorrect decimal value"); + errors.add("Duplicated primary key"); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java new file mode 100644 index 000000000..bf4f679a9 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java @@ -0,0 +1,282 @@ +package sqlancer.oceanbase.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; + +public class OceanBaseTableGenerator { + + private final StringBuilder sb = new StringBuilder(); + private final boolean allowPrimaryKey; + private boolean setPrimaryKey; + private final String tableName; + private final Randomly r; + private int columnId; + private boolean tableHasNullableColumn; + private int keysSpecified; + private final List columns = new ArrayList<>(); + private final OceanBaseSchema schema; + private final OceanBaseGlobalState globalState; + + public OceanBaseTableGenerator(OceanBaseGlobalState globalState, String tableName) { + this.tableName = tableName; + this.r = globalState.getRandomly(); + this.schema = globalState.getSchema(); + allowPrimaryKey = Randomly.getBoolean(); + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(OceanBaseGlobalState globalState, String tableName) { + return new OceanBaseTableGenerator(globalState, tableName).create(); + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + + sb.append("CREATE"); + sb.append(" TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { + sb.append(" LIKE "); + sb.append(schema.getRandomTable().getName()); + return new SQLQueryAdapter(sb.toString(), true); + } else { + sb.append("("); + for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(); + } + sb.append(")"); + sb.append(" "); + appendTableOptions(); + appendPartitionOptions(); + addCommonErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + } + + private void addCommonErrors(ExpectedErrors list) { + list.add("doesn't have this option"); + list.add("must include all columns"); + list.add("not allowed type for this type of partitioning"); + list.add("doesn't support BLOB/TEXT columns"); + list.add("A BLOB field is not allowed in partition function"); + list.add("Too many keys specified; max 1 keys allowed"); + list.add("The total length of the partitioning fields is too large"); + } + + private enum PartitionOptions { + HASH, KEY + } + + private void appendPartitionOptions() { + sb.append(" PARTITION BY"); + switch (Randomly.fromOptions(PartitionOptions.values())) { + case HASH: + sb.append(" HASH("); + sb.append(Randomly.fromList(columns)); + sb.append(")"); + sb.append(" partitions "); + sb.append(r.getInteger(1, 20)); + break; + case KEY: + sb.append(" KEY"); + sb.append(" ("); + sb.append(Randomly.nonEmptySubset(columns).stream().collect(Collectors.joining(", "))); + sb.append(")"); + break; + default: + throw new AssertionError(); + } + } + + private enum TableOptions { + BS,BLOOM,AUTO_INCREMENT; + + public static List getRandomTableOptions() { + List options; + if (Randomly.getBooleanWithSmallProbability()) { + options = Randomly.subset(TableOptions.values()); + } else { + if (Randomly.getBoolean()) { + options = Collections.emptyList(); + } else { + options = Randomly.nonEmptySubset(Arrays.asList(TableOptions.values()), 0); + } + } + return options; + } + } + + private void appendTableOptions() { + List tableOptions = TableOptions.getRandomTableOptions(); + int i = 0; + for (TableOptions o : tableOptions) { + if (i++ != 0) { + sb.append(", "); + } + switch (o) { + case AUTO_INCREMENT: + sb.append("AUTO_INCREMENT = " + r.getPositiveInteger()); + break; + case BLOOM: + sb.append("USE_BLOOM_FILTER = "); + if(Randomly.getBoolean()) + sb.append(" FALSE "); + else + sb.append(" true "); + break; + case BS: + sb.append(" BLOCK_SIZE = "); + if(Randomly.getBoolean()) + sb.append(" 16384 "); + else + sb.append(" 32768 "); + break; + default: + throw new AssertionError(o); + } + } + } + + private void appendColumn() { + String columnName = DBMSCommon.createColumnName(columnId); + columns.add(columnName); + sb.append(columnName); + appendColumnDefinition(); + columnId++; + } + + private enum ColumnOptions { + NULL_OR_NOT_NULL, UNIQUE, COMMENT, PRIMARY_KEY + } + + private void appendColumnDefinition() { + sb.append(" "); + + OceanBaseDataType randomType = OceanBaseDataType.getRandom(globalState); + boolean isTextType = randomType == OceanBaseDataType.VARCHAR; + appendTypeString(randomType); + sb.append(" "); + boolean isNull = false; + boolean columnHasPrimaryKey = false; + + List columnOptions = Randomly.subset(ColumnOptions.values()); + if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { + tableHasNullableColumn = true; + } + if (isTextType) { + columnOptions.remove(ColumnOptions.PRIMARY_KEY); + columnOptions.remove(ColumnOptions.UNIQUE); + } + for (ColumnOptions o : columnOptions) { + sb.append(" "); + switch (o) { + case NULL_OR_NOT_NULL: + // PRIMARY KEYs cannot be NULL + if (!columnHasPrimaryKey) { + if (Randomly.getBoolean()) { + sb.append("NULL"); + } + tableHasNullableColumn = true; + isNull = true; + } else { + sb.append("NOT NULL"); + } + break; + case UNIQUE: + sb.append("UNIQUE"); + keysSpecified++; + if (Randomly.getBoolean()) { + sb.append(" KEY"); + } + break; + case COMMENT: + // TODO: generate randomly + sb.append(String.format("COMMENT '%s' ", "asdf")); + break; + case PRIMARY_KEY: + // PRIMARY KEYs cannot be NULL + if (allowPrimaryKey && !setPrimaryKey && !isNull) { + sb.append("PRIMARY KEY"); + setPrimaryKey = true; + columnHasPrimaryKey = true; + } + break; + default: + throw new AssertionError(); + } + } + + } + + private void appendTypeString(OceanBaseDataType randomType) { + switch (randomType) { + case DECIMAL: + sb.append("DECIMAL"); + optionallyAddPrecisionAndScale(sb); + break; + case INT: + sb.append(Randomly.fromOptions("TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT")); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(Randomly.getNotCachedInteger(0, 255)); + sb.append(")"); + } + break; + case VARCHAR: + sb.append(Randomly.fromOptions("VARCHAR(500)")); + break; + case FLOAT: + sb.append("FLOAT"); + optionallyAddPrecisionAndScale(sb); + break; + case DOUBLE: + sb.append(Randomly.fromOptions("DOUBLE", "FLOAT")); + optionallyAddPrecisionAndScale(sb); + break; + default: + throw new AssertionError(); + } + if (randomType.isNumeric()) { + if (Randomly.getBoolean() && randomType != OceanBaseDataType.INT) { + sb.append(" UNSIGNED"); + } + if (Randomly.getBoolean()) { + sb.append(" ZEROFILL"); + } + } + } + + public static void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean()) { + sb.append("("); + long m = Randomly.getNotCachedInteger(1, 53); + sb.append(m); + sb.append(", "); + long nCandidate = Randomly.getNotCachedInteger(1, 30); + long n = Math.min(nCandidate, m); + sb.append(n); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java new file mode 100644 index 000000000..04f5d3613 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java @@ -0,0 +1,18 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; + +public final class OceanBaseTruncateTableGenerator { + + private OceanBaseTruncateTableGenerator() { + } + + public static SQLQueryAdapter generate(OceanBaseGlobalState globalState) { + StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); + sb.append(globalState.getSchema().getRandomTable().getName()); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("doesn't have this option")); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java new file mode 100644 index 000000000..ff20328d5 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -0,0 +1,54 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.Randomly; +import sqlancer.oceanbase.*; + + +import java.sql.SQLException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class OceanBaseUpdateGenerator { + private OceanBaseUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(OceanBaseGlobalState globalState) throws SQLException { + Randomly r = globalState.getRandomly(); + ExpectedErrors errors = new ExpectedErrors(); + OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); + StringBuilder sb = new StringBuilder("UPDATE "); + if (Randomly.getBoolean()) { + sb.append(" /*+parallel("+r.getInteger(0, 10)+") enable_parallel_dml*/ "); + } + sb.append(table.getName()); + sb.append(" SET "); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append("="); + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant(columns.get(i))); + } else { + sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + OceanBaseErrors.addExpressionErrors(errors); + } + } + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + OceanBaseErrors.addExpressionErrors(errors); + sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + errors.add("Data Too Long"); + } + errors.add("Duplicated primary key"); + OceanBaseErrors.addInsertErrors(errors); + + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java new file mode 100644 index 000000000..09daea34a --- /dev/null +++ b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java @@ -0,0 +1,150 @@ +package sqlancer.oceanbase.gen.datadef; + +import java.util.List; +import java.util.Set; +import java.util.HashSet; +import java.util.ArrayList; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; + +public class OceanBaseIndexGenerator { + + private final Randomly r; + private StringBuilder sb = new StringBuilder(); + private boolean columnIsPrimaryKey; + private boolean containsInPlace; + private OceanBaseSchema schema; + private List columns = new ArrayList<>(); + private final OceanBaseGlobalState globalState; + + public OceanBaseIndexGenerator(OceanBaseSchema schema, Randomly r, OceanBaseGlobalState globalState) { + this.schema = schema; + this.r = r; + this.globalState = globalState; + } + + public static SQLQueryAdapter create(OceanBaseGlobalState globalState) { + return new OceanBaseIndexGenerator(globalState.getSchema(), globalState.getRandomly(), globalState).create(); + } + + public SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + OceanBaseErrors.addExpressionErrors(errors); + sb.append("CREATE "); + sb.append("INDEX "); + sb.append(globalState.getSchema().getFreeIndexName()); + indexType(); + sb.append(" ON "); + OceanBaseTable table = schema.getRandomTable(); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(table.getName()); + sb.append("("); + List randomColumn = table.getRandomNonEmptyColumnSubset(); + int i = 0; + for (OceanBaseColumn c : randomColumn) { + if (i++ != 0) { + sb.append(", "); + } + if (c.isPrimaryKey()) { + columnIsPrimaryKey = true; + } + c.isPartioned = true; + columns.add(c); + sb.append(c.getName()); + if (Randomly.getBoolean() && c.getType() == OceanBaseDataType.VARCHAR) { + sb.append("("); + sb.append(r.getInteger(1, 5)); + sb.append(")"); + c.isPartioned = false; + } + } + sb.append(")"); + appendPartitionOptions(); + indexOption(); + String string = sb.toString(); + sb = new StringBuilder(); + errors.add("A primary key index cannot be invisible"); + errors.add("Functional index on a column is not supported. Consider using a regular index instead."); + errors.add("Incorrect usage of spatial/fulltext/hash index and explicit index order"); + errors.add("must include all columns"); + errors.add("cannot index the expression"); + errors.add("Data truncation: Truncated incorrect"); + errors.add("a disallowed function."); + errors.add("Data truncation"); + errors.add("Cannot create a functional index on an expression that returns a BLOB or TEXT."); + errors.add("used in key specification without a key length"); + errors.add("can't be used in key specification with the used table type"); + errors.add("Specified key was too long"); + errors.add("out of range"); + errors.add("Data truncated for functional index"); + errors.add("used in key specification without a key length"); + errors.add("Row size too large"); // seems to happen together with MIN_ROWS in the table declaration + return new SQLQueryAdapter(string, errors, true); + } + + private enum PartitionOptions { + HASH + } + private void appendPartitionOptions() { + if (Randomly.getBoolean()) { + return; + } + + OceanBaseColumn colIndex = Randomly.fromList(columns); + + if(colIndex.isPartioned == false){ + return; + } + + if(colIndex.getType() == OceanBaseDataType.VARCHAR){ + sb.append(" PARTITION BY"); + sb.append(" KEY"); + sb.append(" ("); + String name = colIndex.getName(); + sb.append(name); + sb.append(")"); + sb.append(" partitions "); + sb.append(r.getInteger(1, 20)); + } + else if(OceanBaseDataType.INT == colIndex.getType()){ + sb.append(" PARTITION BY"); + sb.append(" HASH("); + String name = colIndex.getName(); + sb.append(name); + sb.append(") "); + sb.append(" partitions "); + sb.append(r.getInteger(1, 20)); + } + else{ + return; + } + } + + private void indexOption() { + if (Randomly.getBoolean()) { + sb.append(" "); + } + } + + private void indexType() { + if (Randomly.getBoolean()) { + sb.append(" USING "); + sb.append(Randomly.fromOptions("BTREE", "HASH")); + } + } + + public void setNewSchema(OceanBaseSchema schema) { + this.schema = schema; + } +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java new file mode 100644 index 000000000..85748870e --- /dev/null +++ b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java @@ -0,0 +1,204 @@ +package sqlancer.oceanbase.oracle; + +import sqlancer.Randomly; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseVisitor; +import sqlancer.oceanbase.ast.*; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction.OceanBaseFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.Random; + +public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { + + // SELECT COUNT(*) FROM t0 WHERE ; + // SELECT SUM(count) FROM (SELECT IS TRUE as count FROM t0); + // SELECT (SELECT COUNT(*) FROM t0 WHERE c0 IS NOT 0) = (SELECT COUNT(*) FROM + // (SELECT c0 is NOT 0 FROM t0)); + private OceanBaseSchema s; + private Randomly r; + private String firstQueryString; + private String secondQueryString; + private static final int NOT_FOUND = -1; + + public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + errors.add("is out of range"); + // regex + errors.add("unmatched parentheses"); + errors.add("nothing to repeat at offset"); + errors.add("missing )"); + errors.add("missing terminating ]"); + errors.add("range out of order in character class"); + errors.add("unrecognized character after "); + errors.add("Got error '(*VERB) not recognized or malformed"); + errors.add("must be followed by"); + errors.add("malformed number or name after"); + errors.add("digit expected after"); + } + + public void check() throws SQLException { + OceanBaseSchema.OceanBaseTable randomTable = s.getRandomTable(); + List columns = randomTable.getColumns(); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state).setColumns(columns); + OceanBaseExpression randomWhereCondition = gen.generateExpression(); + List groupBys = Collections.emptyList(); // getRandomExpressions(columns); + List tableList = (Arrays.asList(randomTable)).stream().map(t -> new OceanBaseTableReference(t)) + .collect(Collectors.toList()); + int firstCount = getFirstQueryCount(tableList, randomWhereCondition, groupBys); + int secondCount = getSecondQuery(tableList, randomWhereCondition, groupBys); + if (firstCount != secondCount && firstCount != NOT_FOUND && secondCount != NOT_FOUND) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, + firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + private int getSecondQuery(List tableList, OceanBaseExpression randomWhereCondition, + List groupBys) throws SQLException { + OceanBaseSelect select = new OceanBaseSelect(); + select.setGroupByClause(groupBys); + OceanBaseExpression expr = getTrueExpr(randomWhereCondition); + + OceanBaseText asText = new OceanBaseText(expr, " as count", false); + select.setFetchColumns(Arrays.asList(asText)); + select.setFromList(tableList); + select.setSelectType(OceanBaseSelect.SelectType.ALL); + int secondCount = 0; + + unoptimizedQueryString = "SELECT SUM(count) FROM (" + OceanBaseVisitor.asString(select) + ") as asdf"; + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + if (options.logEachSelect()) { + logger.writeCurrent(unoptimizedQueryString); + } + try { + rs = q.executeAndGet(state); + } catch (Exception e) { + throw new AssertionError(optimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(List tableList, OceanBaseExpression randomWhereCondition, + List groupBys) throws SQLException { + OceanBaseSelect select = new OceanBaseSelect(); + select.setGroupByClause(groupBys); + // SELECT COUNT(t1.c3) FROM t1 WHERE (- (t1.c2)); + // SELECT SUM(count) FROM (SELECT ((- (t1.c2)) IS TRUE) as count FROM t1);; + OceanBaseAggregate aggr = new OceanBaseAggregate( + new OceanBaseColumnName(new OceanBaseSchema.OceanBaseColumn("*", OceanBaseSchema.OceanBaseDataType.INT, false, 0, false)), + OceanBaseAggregate.OceanBaseAggregateFunction.COUNT); + select.setFetchColumns(Arrays.asList(aggr)); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + select.setSelectType(OceanBaseSelect.SelectType.ALL); + int firstCount = 0; + optimizedQueryString = OceanBaseVisitor.asString(select); + SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); + SQLancerResultSet rs; + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try { + rs = q.executeAndGet(state); + } catch (Exception e) { + throw new AssertionError(firstQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + firstCount += rs.getLong(1); + } + rs.close(); + return firstCount; + } + + private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition){ + int i = new Random().nextInt(8); + OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE,false); + + OceanBaseUnaryPostfixOperation isNotTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE,true); + + OceanBaseUnaryPostfixOperation isFalse= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE,false); + + OceanBaseUnaryPostfixOperation isNotFalse= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE,true); + + OceanBaseUnaryPostfixOperation isNULL= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL,false); + + OceanBaseUnaryPostfixOperation isNotNULL= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL,true); + + OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); + switch(i){ + case 0: + expr = isTrue; + break; + case 1: + expr = new OceanBaseUnaryPrefixOperation(new OceanBaseBinaryLogicalOperation(isFalse, isNULL, OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case 2: + expr = new OceanBaseUnaryPrefixOperation(new OceanBaseUnaryPrefixOperation(isTrue, OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case 3: + expr = new OceanBaseUnaryPrefixOperation(isNotTrue, OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case 4: + expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); + break; + case 5: + OceanBaseExpression[] args = new OceanBaseExpression[3]; + args[0] = OceanBaseConstant.createIntConstant(1); + args[1] = isTrue; + args[2] = OceanBaseConstant.createIntConstant(0); + expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); + break; + case 6: + OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; + ifArgs[0] = OceanBaseConstant.createNullConstant(); + ifArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); + break; + case 7: + OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; + coalesceArgs[0] = OceanBaseConstant.createNullConstant(); + coalesceArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); + break; + } + return expr; + } + + +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..0b9c04040 --- /dev/null +++ b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java @@ -0,0 +1,159 @@ +package sqlancer.oceanbase.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.ArrayList; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseRowValue; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTables; +import sqlancer.oceanbase.OceanBaseVisitor; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation.UnaryPostfixOperator; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; + +public class OceanBasePivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + private List columns; + + public OceanBasePivotedQuerySynthesisOracle(OceanBaseGlobalState globalState) throws SQLException { + super(globalState); + OceanBaseErrors.addExpressionErrors(errors); + errors.add("in 'order clause'"); + errors.add("value is out of range"); + } + + @Override + public Query getRectifiedQuery() throws SQLException { + OceanBaseTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); + List tables = randomFromTables.getTables(); + + OceanBaseSelect selectStatement = new OceanBaseSelect(); + selectStatement.setSelectType(Randomly.fromOptions(OceanBaseSelect.SelectType.values())); + columns = randomFromTables.getColumns(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + + selectStatement.setFromList(tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList())); + + fetchColumns = columns.stream().map(c -> new OceanBaseColumnReference(c, null)).map(d -> d.setRef(true)).collect(Collectors.toList()); + selectStatement.setFetchColumns(fetchColumns); + OceanBaseExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + OceanBaseExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + OceanBaseExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + List orderBy = generateOrderBy(columns); + selectStatement.setOrderByExpressions(orderBy); + + return new SQLQueryAdapter(OceanBaseVisitor.asString(selectStatement), errors); + } + + private List generateGroupByClause(List columns, OceanBaseRowValue rw) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> OceanBaseColumnReference.create(c, rw.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + public List generateOrderBy(List columns) { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new OceanBaseOrderByTerm(OceanBaseColumnReference.create(Randomly.fromList(columns), null), + OceanBaseOrder.getRandomOrder())); + } + return orderBys; + } + + private OceanBaseConstant generateLimit() { + if (Randomly.getBoolean()) { + return OceanBaseConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private OceanBaseExpression generateOffset() { + if (Randomly.getBoolean()) { + return OceanBaseConstant.createIntConstantNotAsBoolean(0); + } else { + return null; + } + } + + private OceanBaseExpression generateRectifiedExpression(List columns, OceanBaseRowValue rw) { + OceanBaseExpression expression = new OceanBaseExpressionGenerator(globalState).setRowVal(rw).setColumns(columns) + .generateExpression(); + OceanBaseConstant expectedValue = expression.getExpectedValue(); + OceanBaseExpression result; + if (expectedValue.isNull()) { + result = new OceanBaseUnaryPostfixOperation(expression, UnaryPostfixOperator.IS_NULL, false); + } else if (expectedValue.asBooleanNotNull()) { + result = expression; + } else { + result = new OceanBaseUnaryPrefixOperation(expression, OceanBaseUnaryPrefixOperator.NOT); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions + sb.append(query.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (OceanBaseColumn c : columns) { + if (i++ != 0) { + sb.append(" AND "); + } + if(pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) + sb.append("concat("); + sb.append("result." + c.getTable().getName() + c.getName()); + if(pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) + sb.append(",'')"); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); + } + } + + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, query.getExpectedErrors()); + } + + @Override + protected String getExpectedValues(OceanBaseExpression expr) { + return OceanBaseVisitor.asExpectedValues(expr); + } +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java new file mode 100644 index 000000000..3e05a305b --- /dev/null +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java @@ -0,0 +1,62 @@ +package sqlancer.oceanbase.oracle; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTables; +import sqlancer.oceanbase.ast.*; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; +import sqlancer.oceanbase.gen.OceanBaseHintGenerator; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; + +public abstract class OceanBaseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { + + OceanBaseSchema s; + OceanBaseTables targetTables; + OceanBaseExpressionGenerator gen; + OceanBaseSelect select; + + public OceanBaseTLPBase(OceanBaseGlobalState state) { + super(state); + OceanBaseErrors.addExpressionErrors(errors); + errors.add("value is out of range"); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new OceanBaseExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new OceanBaseSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + OceanBaseHintGenerator.generateHints(select, tables); + List tableList = tables.stream().map(t -> new OceanBaseTableReference(t)) + .collect(Collectors.toList()); + select.setFromList(tableList); + select.setWhereClause(null); + } + + List generateFetchColumns() { + return Arrays.asList(OceanBaseColumnReference.create(targetTables.getColumns().get(0), null)); + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java new file mode 100644 index 000000000..73a0c3a6f --- /dev/null +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java @@ -0,0 +1,44 @@ +package sqlancer.oceanbase.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseTLPWhereOracle extends OceanBaseTLPBase { + + public OceanBaseTLPWhereOracle(OceanBaseGlobalState state) { + super(state); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = OceanBaseVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setOrderByExpressions(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = OceanBaseVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = OceanBaseVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = OceanBaseVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + +} diff --git a/test/sqlancer/dbms/TestOceanBaseNoREC.java b/test/sqlancer/dbms/TestOceanBaseNoREC.java new file mode 100644 index 000000000..3101c63b3 --- /dev/null +++ b/test/sqlancer/dbms/TestOceanBaseNoREC.java @@ -0,0 +1,29 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestOceanBaseNoREC { + + String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); + boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); + + @Test + public void testNoREC() { + assumeTrue(oceanBaseIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--database-prefix", + "pqsdb", "--num-queries", + TestConfig.NUM_QUERIES, + "--username","sqlancer@test", "--password", "sqlancer", + //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: + //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "oceanbase", "--oracle", "NoREC" })); + } + +} diff --git a/test/sqlancer/dbms/TestOceanBasePQS.java b/test/sqlancer/dbms/TestOceanBasePQS.java new file mode 100644 index 000000000..52dcb2eb5 --- /dev/null +++ b/test/sqlancer/dbms/TestOceanBasePQS.java @@ -0,0 +1,29 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestOceanBasePQS { + + String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); + boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); + + @Test + public void testPQS() { + assumeTrue(oceanBaseIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", + "pqsdb", "--num-queries", + TestConfig.NUM_QUERIES, + "--username","sqlancer@test", "--password", "sqlancer", + //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: + //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "oceanbase", "--oracle", "PQS" })); + } + +} diff --git a/test/sqlancer/dbms/TestOceanBaseTLP.java b/test/sqlancer/dbms/TestOceanBaseTLP.java new file mode 100644 index 000000000..e172c21fc --- /dev/null +++ b/test/sqlancer/dbms/TestOceanBaseTLP.java @@ -0,0 +1,29 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestOceanBaseTLP { + + String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); + boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); + + @Test + public void testTLP() { + assumeTrue(oceanBaseIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--database-prefix", + "pqsdb", "--num-queries", + TestConfig.NUM_QUERIES, + "--username","sqlancer@test", "--password", "sqlancer", + //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: + //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "oceanbase", "--oracle", "TLP_WHERE" })); + } + +} From 6b7c6f62caf10fab4a6195ef6367ea9ebe6f4f45 Mon Sep 17 00:00:00 2001 From: bashaojing Date: Wed, 15 Dec 2021 20:51:01 +0800 Subject: [PATCH 0468/1284] fix for hint,update default port,add readme for install oceanbase --- README.md | 1 + .../oceanbase/OceanBaseHintGenerator.java | 78 ++++++++++--------- src/sqlancer/oceanbase/OceanBaseOptions.java | 6 +- src/sqlancer/oceanbase/OceanBaseProvider.java | 6 ++ src/sqlancer/oceanbase/README.md | 21 +++++ test/sqlancer/dbms/TestOceanBaseNoREC.java | 2 +- test/sqlancer/dbms/TestOceanBasePQS.java | 2 +- test/sqlancer/dbms/TestOceanBaseTLP.java | 2 +- 8 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 src/sqlancer/oceanbase/README.md diff --git a/README.md b/README.md index 9559ac048..7008c1d92 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | DuckDB | Working | Untyped, Generic | | | ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | | TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | +| OceanBase | Working | Untyped | | # Using SQLancer diff --git a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java index 83accab72..b3f9d5412 100644 --- a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java +++ b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java @@ -18,22 +18,24 @@ public class OceanBaseHintGenerator { private Randomly r = new Randomly(); enum IndexHint { + PDML, + NO_PRED_DEDUCE, MERGE_JOIN, - INL_JOIN, - INL_HASH_JOIN, - INL_MERGE_JOIN, - HASH_JOIN, - HASH_AGG, - STREAM_AGG, - USE_INDEX, - IGNORE_INDEX, - AGG_TO_COP, - USE_INDEX_MERGE, - NO_INDEX_MERGE, + HASH_JOIN, + NL_JOIN, + BNL_JOIN, + NO_MERGE_JOIN, + NO_HASH_JOIN, + NO_NL_JOIN, + NO_BNL_JOIN, + HASH_AGG, + NL_MATERIALIZATION, + LATE_MATERIALIZATION, + USE_INDEX, + TOPK, LEADING, - PredDeduce, - PDML, - USE_TOJA; + ORDERED, + NO_REWRITE; } public OceanBaseHintGenerator(OceanBaseSelect select, List tables) { @@ -52,50 +54,56 @@ private void generate() { case PDML: sb.append(" parallel(" + r.getInteger(0, 10) + "),enable_parallel_dml "); break; - case PredDeduce: - sb.append("no_pred_deduce"); + case NO_PRED_DEDUCE: + sb.append("NO_PRED_DEDUCE"); break; case MERGE_JOIN: tablesHint("USE_MERGE "); break; - case INL_JOIN: + case HASH_JOIN: + tablesHint("USE_HASH "); + break; + case NL_JOIN: tablesHint("USE_NL "); break; - case LEADING: - tablesHint(" LEADING "); + case BNL_JOIN: + tablesHint("USE_BNL "); break; - case INL_HASH_JOIN: - tablesHint("USE_HASH "); + case NO_MERGE_JOIN: + sb.append(" NO_USE_MERGE "); break; - case INL_MERGE_JOIN: - tablesHint("USE_BNL "); + case NO_HASH_JOIN: + sb.append(" NO_USE_HASH "); break; - case HASH_JOIN: - sb.append(" parallel(1) "); + case NO_NL_JOIN: + sb.append(" NO_USE_NL "); + break; + case NO_BNL_JOIN: + sb.append(" NO_USE_BNL "); break; case HASH_AGG: sb.append("USE_HASH_AGGREGATION "); break; - case STREAM_AGG: + case NL_MATERIALIZATION: sb.append("USE_NL_MATERIALIZATION "); break; + case LATE_MATERIALIZATION: + sb.append("USE_LATE_MATERIALIZATION "); + break; case USE_INDEX: indexesHint("INDEX_HINT "); break; - case IGNORE_INDEX: + case TOPK: sb.append("TOPK (50 50) "); break; - case AGG_TO_COP: - sb.append("USE_LATE_MATERIALIZATION "); + case LEADING: + tablesHint(" LEADING "); break; - case USE_INDEX_MERGE: + case ORDERED: sb.append("ORDERED "); break; - case NO_INDEX_MERGE: - tablesHint("NO_MERGE "); - break; - case USE_TOJA: - sb.append("no_rewrite " ); + case NO_REWRITE: + sb.append("NO_REWRITE " ); break; default: throw new AssertionError(); diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index fb529e375..bd4c20f46 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -19,7 +19,7 @@ + ", default host: " + OceanBaseOptions.DEFAULT_HOST) public class OceanBaseOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; - public static final int DEFAULT_PORT = 2883; + public static final int DEFAULT_PORT = 2881; @Parameter(names = "--oracle") public List oracles = Arrays.asList(OceanBaseOracleFactory.TLP_WHERE); @@ -51,6 +51,10 @@ public boolean requiresAllTablesToContainRows() { } } } + @Parameter(names = { "--query-timeout" }, description = "Query timeout") + public int queryTimeout = 1000000000; + @Parameter(names = { "--transaction-timeout" }, description = "Transaction timeout") + public int trxTimeout = 1000000000; @Override public List getTestOracleFactory() { diff --git a/src/sqlancer/oceanbase/OceanBaseProvider.java b/src/sqlancer/oceanbase/OceanBaseProvider.java index 5b57ee3c1..186109a70 100644 --- a/src/sqlancer/oceanbase/OceanBaseProvider.java +++ b/src/sqlancer/oceanbase/OceanBaseProvider.java @@ -140,6 +140,12 @@ public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exc String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", host, port); Connection con = DriverManager.getConnection(url, username, password); + try(Statement s = con.createStatement()){ + s.execute("set ob_query_timeout=" + globalState.getDbmsSpecificOptions().queryTimeout); + } + try(Statement s = con.createStatement()){ + s.execute("set ob_trx_timeout=" + globalState.getDbmsSpecificOptions().trxTimeout); + } try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); diff --git a/src/sqlancer/oceanbase/README.md b/src/sqlancer/oceanbase/README.md new file mode 100644 index 000000000..17940c678 --- /dev/null +++ b/src/sqlancer/oceanbase/README.md @@ -0,0 +1,21 @@ +## Install Oceanbase +There are some methods to install OceanBase. +A method to install a local single-node OceanBase cluster: +```shell +sudo yum install -y yum-utils +sudo yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo +sudo yum install -y ob-deploy +source /etc/profile.d/obd.sh + +git clone https://github.com/oceanbase/obdeploy.git +cd obdeploy +sudo obd cluster deploy c1 -c ./example/mini-local-example.yaml -A +sudo obd cluster start c1 +``` +> **NOTE:** We do not recommend that you use sys tenant to test. So please deploy clutser with optition -A, will create the test tenant during the bootstrap by using all available resources of the cluster. +> **NOTE:** Then you can create user. + +```shell +mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" +``` +Other methods, see [OceanBase Deployer](https://github.com/oceanbase/obdeploy/blob/master/README.md). diff --git a/test/sqlancer/dbms/TestOceanBaseNoREC.java b/test/sqlancer/dbms/TestOceanBaseNoREC.java index 3101c63b3..ddf6bcda0 100644 --- a/test/sqlancer/dbms/TestOceanBaseNoREC.java +++ b/test/sqlancer/dbms/TestOceanBaseNoREC.java @@ -22,7 +22,7 @@ public void testNoREC() { TestConfig.NUM_QUERIES, "--username","sqlancer@test", "--password", "sqlancer", //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "NoREC" })); } diff --git a/test/sqlancer/dbms/TestOceanBasePQS.java b/test/sqlancer/dbms/TestOceanBasePQS.java index 52dcb2eb5..4123e76c0 100644 --- a/test/sqlancer/dbms/TestOceanBasePQS.java +++ b/test/sqlancer/dbms/TestOceanBasePQS.java @@ -22,7 +22,7 @@ public void testPQS() { TestConfig.NUM_QUERIES, "--username","sqlancer@test", "--password", "sqlancer", //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "PQS" })); } diff --git a/test/sqlancer/dbms/TestOceanBaseTLP.java b/test/sqlancer/dbms/TestOceanBaseTLP.java index e172c21fc..6eef3fc91 100644 --- a/test/sqlancer/dbms/TestOceanBaseTLP.java +++ b/test/sqlancer/dbms/TestOceanBaseTLP.java @@ -22,7 +22,7 @@ public void testTLP() { TestConfig.NUM_QUERIES, "--username","sqlancer@test", "--password", "sqlancer", //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "TLP_WHERE" })); } From 9b02f1324f07c6b8a82c73f93f7940d65208e78b Mon Sep 17 00:00:00 2001 From: bashaojing Date: Thu, 16 Dec 2021 19:49:32 +0800 Subject: [PATCH 0469/1284] fix for format check and some comments --- .travis.yml | 6 - src/sqlancer/Main.java | 2 +- src/sqlancer/oceanbase/OceanBaseErrors.java | 6 +- .../OceanBaseExpectedValueVisitor.java | 8 +- .../oceanbase/OceanBaseHintGenerator.java | 141 --------------- src/sqlancer/oceanbase/OceanBaseOptions.java | 3 +- src/sqlancer/oceanbase/OceanBaseProvider.java | 30 ++-- src/sqlancer/oceanbase/OceanBaseSchema.java | 61 +++---- .../oceanbase/OceanBaseToStringVisitor.java | 42 ++--- .../OceanBaseUserCheckException.java | 8 + src/sqlancer/oceanbase/OceanBaseVisitor.java | 8 +- src/sqlancer/oceanbase/README.md | 18 +- .../oceanbase/ast/OceanBaseAggregate.java | 5 +- .../OceanBaseBinaryComparisonOperation.java | 5 +- .../ast/OceanBaseBinaryLogicalOperation.java | 5 +- .../oceanbase/ast/OceanBaseColumnName.java | 2 +- .../ast/OceanBaseColumnReference.java | 15 +- .../ast/OceanBaseComputableFunction.java | 95 +++++----- .../oceanbase/ast/OceanBaseConstant.java | 90 +++++----- .../oceanbase/ast/OceanBaseInOperation.java | 2 - .../oceanbase/ast/OceanBaseSelect.java | 9 +- src/sqlancer/oceanbase/ast/OceanBaseText.java | 7 +- .../ast/OceanBaseUnaryPrefixOperation.java | 9 +- .../oceanbase/gen/OceanBaseAlterTable.java | 4 +- .../gen/OceanBaseDeleteGenerator.java | 15 +- .../oceanbase/gen/OceanBaseDropIndex.java | 6 +- .../gen/OceanBaseExpressionGenerator.java | 54 +++--- .../oceanbase/gen/OceanBaseHintGenerator.java | 125 +++++++++++++ .../gen/OceanBaseInsertGenerator.java | 13 +- .../gen/OceanBaseTableGenerator.java | 57 +++--- .../gen/OceanBaseUpdateGenerator.java | 35 ++-- .../gen/datadef/OceanBaseIndexGenerator.java | 28 +-- .../oracle/OceanBaseNoRECOracle.java | 170 ++++++++++-------- .../OceanBasePivotedQuerySynthesisOracle.java | 24 +-- .../oceanbase/oracle/OceanBaseTLPBase.java | 20 +-- test/sqlancer/dbms/TestOceanBaseNoREC.java | 12 +- test/sqlancer/dbms/TestOceanBasePQS.java | 13 +- test/sqlancer/dbms/TestOceanBaseTLP.java | 12 +- 38 files changed, 590 insertions(+), 575 deletions(-) delete mode 100644 src/sqlancer/oceanbase/OceanBaseHintGenerator.java create mode 100644 src/sqlancer/oceanbase/OceanBaseUserCheckException.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java diff --git a/.travis.yml b/.travis.yml index 6e5594948..318ea6fa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -144,9 +144,3 @@ matrix: - sleep 5 script: - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test - - name: OceanBase - jdk : openjdk8 - script: - - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBaseTLP test - - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBasePQS test - - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBaseNoREC test diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 1ab9e29d9..a3ff789e4 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -34,10 +34,10 @@ import sqlancer.mariadb.MariaDBProvider; import sqlancer.mongodb.MongoDBProvider; import sqlancer.mysql.MySQLProvider; +import sqlancer.oceanbase.OceanBaseProvider; import sqlancer.postgres.PostgresProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; -import sqlancer.oceanbase.OceanBaseProvider; public final class Main { diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index 392bd7c5e..893490c34 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -16,6 +16,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Invalid numeric"); errors.add("Data truncated for argument"); } + public static void addInsertErrors(ExpectedErrors errors) { errors.add("Duplicate entry"); errors.add("cannot be null"); @@ -39,10 +40,7 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Truncated incorrect DOUBLE value"); errors.add("Data truncated for argument"); errors.add("Invalid numeric"); + errors.add("Miss column"); - - if (true) { - errors.add("Miss column"); - } } } diff --git a/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java index 2c03436d1..f9e57bd24 100644 --- a/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java @@ -1,9 +1,11 @@ package sqlancer.oceanbase; import sqlancer.IgnoreMeException; +import sqlancer.oceanbase.ast.OceanBaseAggregate; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseColumnReference; import sqlancer.oceanbase.ast.OceanBaseComputableFunction; import sqlancer.oceanbase.ast.OceanBaseConstant; @@ -14,10 +16,8 @@ import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseStringExpression; import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; -import sqlancer.oceanbase.ast.OceanBaseAggregate; -import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; public class OceanBaseExpectedValueVisitor implements OceanBaseVisitor { @@ -147,7 +147,7 @@ public void visit(OceanBaseText func) { } @Override - public void visit(OceanBaseUnaryPrefixOperation op){ + public void visit(OceanBaseUnaryPrefixOperation op) { print(op); visit(op.getExpr()); } diff --git a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java deleted file mode 100644 index b3f9d5412..000000000 --- a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java +++ /dev/null @@ -1,141 +0,0 @@ -package sqlancer.oceanbase.gen; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.ast.OceanBaseConstant; -import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.ast.OceanBaseStringExpression; - -import java.util.List; -import java.util.stream.Collectors; - -public class OceanBaseHintGenerator { - private OceanBaseSelect select; - private List tables; - private final StringBuilder sb = new StringBuilder(); - private Randomly r = new Randomly(); - - enum IndexHint { - PDML, - NO_PRED_DEDUCE, - MERGE_JOIN, - HASH_JOIN, - NL_JOIN, - BNL_JOIN, - NO_MERGE_JOIN, - NO_HASH_JOIN, - NO_NL_JOIN, - NO_BNL_JOIN, - HASH_AGG, - NL_MATERIALIZATION, - LATE_MATERIALIZATION, - USE_INDEX, - TOPK, - LEADING, - ORDERED, - NO_REWRITE; - } - - public OceanBaseHintGenerator(OceanBaseSelect select, List tables) { - this.select = select; - this.tables = tables; - } - - public static void generateHints(OceanBaseSelect select, List tables) { - new OceanBaseHintGenerator(select, tables).generate(); - - } - - private void generate() { - OceanBaseTable table = Randomly.fromList(tables); - switch (Randomly.fromOptions(IndexHint.values())) { - case PDML: - sb.append(" parallel(" + r.getInteger(0, 10) + "),enable_parallel_dml "); - break; - case NO_PRED_DEDUCE: - sb.append("NO_PRED_DEDUCE"); - break; - case MERGE_JOIN: - tablesHint("USE_MERGE "); - break; - case HASH_JOIN: - tablesHint("USE_HASH "); - break; - case NL_JOIN: - tablesHint("USE_NL "); - break; - case BNL_JOIN: - tablesHint("USE_BNL "); - break; - case NO_MERGE_JOIN: - sb.append(" NO_USE_MERGE "); - break; - case NO_HASH_JOIN: - sb.append(" NO_USE_HASH "); - break; - case NO_NL_JOIN: - sb.append(" NO_USE_NL "); - break; - case NO_BNL_JOIN: - sb.append(" NO_USE_BNL "); - break; - case HASH_AGG: - sb.append("USE_HASH_AGGREGATION "); - break; - case NL_MATERIALIZATION: - sb.append("USE_NL_MATERIALIZATION "); - break; - case LATE_MATERIALIZATION: - sb.append("USE_LATE_MATERIALIZATION "); - break; - case USE_INDEX: - indexesHint("INDEX_HINT "); - break; - case TOPK: - sb.append("TOPK (50 50) "); - break; - case LEADING: - tablesHint(" LEADING "); - break; - case ORDERED: - sb.append("ORDERED "); - break; - case NO_REWRITE: - sb.append("NO_REWRITE " ); - break; - default: - throw new AssertionError(); - } - - select.setHint(new OceanBaseStringExpression(sb.toString(),new OceanBaseConstant.OceanBaseTextConstant(sb.toString()))); - } - - private void indexesHint(String string) { - sb.append(string); - sb.append("("); - OceanBaseTable table = Randomly.fromList(tables); - List allIndexes = table.getIndexes(); - if (allIndexes.isEmpty()) { - throw new IgnoreMeException(); - } - List indexSubset = Randomly.nonEmptySubset(allIndexes); - sb.append(table.getName()); - sb.append(", "); - sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); - sb.append(")"); - } - - private void tablesHint(String string) { - sb.append(string); - sb.append("("); - appendTables(); - sb.append(")"); - } - - private void appendTables() { - List tableSubset = Randomly.nonEmptySubset(tables); - sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); - } -} diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index bd4c20f46..eb7200437 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -11,9 +11,9 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; +import sqlancer.oceanbase.oracle.OceanBaseNoRECOracle; import sqlancer.oceanbase.oracle.OceanBasePivotedQuerySynthesisOracle; import sqlancer.oceanbase.oracle.OceanBaseTLPWhereOracle; -import sqlancer.oceanbase.oracle.OceanBaseNoRECOracle; @Parameters(separators = "=", commandDescription = "OceanBase (default port: " + OceanBaseOptions.DEFAULT_PORT + ", default host: " + OceanBaseOptions.DEFAULT_HOST) @@ -51,6 +51,7 @@ public boolean requiresAllTablesToContainRows() { } } } + @Parameter(names = { "--query-timeout" }, description = "Query timeout") public int queryTimeout = 1000000000; @Parameter(names = { "--transaction-timeout" }, description = "Transaction timeout") diff --git a/src/sqlancer/oceanbase/OceanBaseProvider.java b/src/sqlancer/oceanbase/OceanBaseProvider.java index 186109a70..d7308ab63 100644 --- a/src/sqlancer/oceanbase/OceanBaseProvider.java +++ b/src/sqlancer/oceanbase/OceanBaseProvider.java @@ -31,21 +31,17 @@ public OceanBaseProvider() { } enum Action implements AbstractAction { - SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), - INSERT(OceanBaseInsertGenerator::insertRow), - CREATE_INDEX(OceanBaseIndexGenerator::create), - ALTER_TABLE(OceanBaseAlterTable::create), - TRUNCATE_TABLE(OceanBaseTruncateTableGenerator::generate), + SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), INSERT(OceanBaseInsertGenerator::insertRow), + CREATE_INDEX(OceanBaseIndexGenerator::create), ALTER_TABLE(OceanBaseAlterTable::create), + TRUNCATE_TABLE(OceanBaseTruncateTableGenerator::generate), SELECT_INFO((g) -> new SQLQueryAdapter( "select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + g.getDatabaseName() - + "'")), + + "'")), CREATE_TABLE((g) -> { String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); - + return OceanBaseTableGenerator.generate(g, tableName); - }), - DELETE(OceanBaseDeleteGenerator::delete), - UPDATE(OceanBaseUpdateGenerator::getQuery), + }), DELETE(OceanBaseDeleteGenerator::delete), UPDATE(OceanBaseUpdateGenerator::update), DROP_INDEX(OceanBaseDropIndex::generate); private final SQLQueryProvider sqlQueryProvider; @@ -118,7 +114,7 @@ public void generateDatabase(OceanBaseGlobalState globalState) throws Exception } @Override - public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exception,SQLException { + public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exception, SQLException { String username = globalState.getOptions().getUserName(); String password = globalState.getOptions().getPassword(); String host = globalState.getOptions().getHost(); @@ -129,9 +125,9 @@ public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exc if (port == MainOptions.NO_SET_PORT) { port = OceanBaseOptions.DEFAULT_PORT; } - if(username.endsWith("sys")||username.equals("root")) - { - throw new Exception("please don't use sys tenant to test! Firstly create tenant then test"); + if (username.endsWith("sys") || username.equals("root")) { + throw new OceanBaseUserCheckException( + "please don't use sys tenant to test! Firstly create tenant then test"); } String databaseName = globalState.getDatabaseName(); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); @@ -140,13 +136,13 @@ public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exc String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", host, port); Connection con = DriverManager.getConnection(url, username, password); - try(Statement s = con.createStatement()){ + + try (Statement s = con.createStatement()) { s.execute("set ob_query_timeout=" + globalState.getDbmsSpecificOptions().queryTimeout); } - try(Statement s = con.createStatement()){ + try (Statement s = con.createStatement()) { s.execute("set ob_trx_timeout=" + globalState.getDbmsSpecificOptions().trxTimeout); } - try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); } diff --git a/src/sqlancer/oceanbase/OceanBaseSchema.java b/src/sqlancer/oceanbase/OceanBaseSchema.java index a16358110..7de2457ee 100644 --- a/src/sqlancer/oceanbase/OceanBaseSchema.java +++ b/src/sqlancer/oceanbase/OceanBaseSchema.java @@ -8,11 +8,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Stream; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.IgnoreMeException; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; @@ -65,11 +64,12 @@ public enum CollateSequence { public static CollateSequence random() { return Randomly.fromOptions(values()); - + } } - public OceanBaseColumn(String name, OceanBaseDataType columnType, boolean isPrimaryKey, int precision, boolean isZeroFill) { + public OceanBaseColumn(String name, OceanBaseDataType columnType, boolean isPrimaryKey, int precision, + boolean isZeroFill) { super(name, null, columnType); this.isPrimaryKey = isPrimaryKey; this.precision = precision; @@ -84,7 +84,7 @@ public int getPrecision() { public boolean isPrimaryKey() { return isPrimaryKey; } - + public boolean isZeroFill() { return isZeroFill; } @@ -98,10 +98,14 @@ public OceanBaseTables(List tables) { } public OceanBaseRowValue getRandomRowValue(SQLConnection con) throws SQLException { - String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", columnNamesAsString( - c -> c.getType()==OceanBaseDataType.FLOAT || c.isZeroFill() ? ("concat(" + c.getTable().getName() + "." + c.getName() + ",'')" + " AS " + c.getTable().getName() + c.getName()) : (c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName())), + String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", + columnNamesAsString(c -> c.getType() == OceanBaseDataType.FLOAT || c.isZeroFill() + ? "concat(" + c.getTable().getName() + "." + c.getName() + ",'')" + " AS " + + c.getTable().getName() + c.getName() + : c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + + c.getName()), tableNamesAsString()); - //cast float and zerofill as varchar + // cast float and zerofill as varchar Map values = new HashMap<>(); try (Statement s = con.createStatement()) { ResultSet randomRowValues = s.executeQuery(randomRow); @@ -114,20 +118,20 @@ public OceanBaseRowValue getRandomRowValue(SQLConnection con) throws SQLExceptio int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); assert columnIndex == i + 1; OceanBaseConstant constant; - if (randomRowValues.getString(columnIndex) == null) { - if(column.isZeroFill()) - constant= OceanBaseConstant.createStringConstant("null"); - else - constant = OceanBaseConstant.createNullConstant(); - + if (randomRowValues.getString(columnIndex) == null) { + if (column.isZeroFill()) { + constant = OceanBaseConstant.createStringConstant("null"); + } else { + constant = OceanBaseConstant.createNullConstant(); + } } else { switch (column.getType()) { case INT: - //cast zerofill as varchar - if (column.isZeroFill()){ + // cast zerofill as varchar + if (column.isZeroFill()) { value = randomRowValues.getString(columnIndex); constant = OceanBaseConstant.createStringConstant((String) value); - }else{ + } else { value = randomRowValues.getLong(columnIndex); constant = OceanBaseConstant.createIntConstant((long) value); } @@ -175,7 +179,8 @@ private static OceanBaseDataType getColumnType(String typeString) { } } - public static class OceanBaseRowValue extends AbstractRowValue { + public static class OceanBaseRowValue + extends AbstractRowValue { OceanBaseRowValue(OceanBaseTables tables, Map values) { super(tables, values); @@ -183,7 +188,8 @@ public static class OceanBaseRowValue extends AbstractRowValue { + public static class OceanBaseTable + extends AbstractRelationalTable { public OceanBaseTable(String tableName, List columns, List indexes) { super(tableName, columns, indexes, false); @@ -204,13 +210,6 @@ private OceanBaseIndex(String indexName) { public static OceanBaseIndex create(String indexName) { return new OceanBaseIndex(indexName); } - - @Override - public String getIndexName() { - return super.getIndexName(); - - } - } public static OceanBaseSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { @@ -219,8 +218,8 @@ public static OceanBaseSchema fromConnection(SQLConnection con, String databaseN try { List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery( - "select TABLE_NAME from information_schema.TABLES where table_schema = '" + try (ResultSet rs = s + .executeQuery("select TABLE_NAME from information_schema.TABLES where table_schema = '" + databaseName + "';")) { while (rs.next()) { String tableName = rs.getString("TABLE_NAME"); @@ -251,8 +250,9 @@ private static List getIndexes(SQLConnection con, String tableNa databaseName, tableName))) { while (rs.next()) { String indexName = rs.getString("INDEX_NAME"); - if(!indexName.equals("PRIMARY")) + if (!indexName.equals("PRIMARY")) { indexes.add(OceanBaseIndex.create(indexName)); + } } } } @@ -272,7 +272,8 @@ private static List getTableColumns(SQLConnection con, String t boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); boolean isZeroFill = rs.getString("COLUMN_TYPE").contains("zerofill"); - OceanBaseColumn c = new OceanBaseColumn(columnName, getColumnType(dataType), isPrimaryKey, precision, isZeroFill); + OceanBaseColumn c = new OceanBaseColumn(columnName, getColumnType(dataType), isPrimaryKey, + precision, isZeroFill); columns.add(c); } } diff --git a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java index 053731852..56fef98f1 100644 --- a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java @@ -5,9 +5,12 @@ import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseAggregate; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseColumnReference; import sqlancer.oceanbase.ast.OceanBaseComputableFunction; import sqlancer.oceanbase.ast.OceanBaseConstant; @@ -19,17 +22,14 @@ import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseStringExpression; import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; -import sqlancer.oceanbase.ast.OceanBaseAggregate; -import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; public class OceanBaseToStringVisitor extends ToStringVisitor implements OceanBaseVisitor { int ref; - private Randomly r = new Randomly(); + private final Randomly r = new Randomly(); @Override public void visitSpecific(OceanBaseExpression expr) { @@ -41,7 +41,7 @@ public void visit(OceanBaseSelect s) { sb.append("SELECT "); if (s.getHint() != null) { sb.append("/*+ "); - visit(s.getHint(),0); + visit(s.getHint(), 0); sb.append(" */ "); } @@ -133,12 +133,14 @@ public String get() { @Override public void visit(OceanBaseColumnReference column) { - if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) - {sb.append("concat(");} + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) { + sb.append("concat("); + } sb.append(column.getColumn().getFullQualifiedName()); - if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) - {sb.append(",'')");} - if (column.getRef()){ + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) { + sb.append(",'')"); + } + if (column.getRef()) { sb.append(" AS "); sb.append(column.getColumn().getTable().getName()); sb.append(column.getColumn().getName()); @@ -254,21 +256,19 @@ public void visit(OceanBaseExists op) { @Override public void visit(OceanBaseStringExpression op) { - if(op.getStr().contains("SELECT")){ - sb.append(op.getStr()); - } - else{ + if (op.getStr().contains("SELECT")) { + sb.append(op.getStr()); + } else { String str = op.getStr(); - if(str.length() > 0){ + if (str.length() > 0) { sb.append(r.getInteger(0, 100000)); - } - else{ + } else { sb.append(r.getInteger(0, 1000000)); } } } - public void visit(OceanBaseStringExpression op,int type) { + public void visit(OceanBaseStringExpression op, int type) { sb.append(op.getStr()); } @@ -283,11 +283,13 @@ public void visit(OceanBaseAggregate aggr) { sb.append("("); visit(aggr.getExpr()); sb.append(")"); - } + } + @Override public void visit(OceanBaseColumnName c) { sb.append(c.getColumn().getName()); } + @Override public void visit(OceanBaseText func) { visit(func.getExpr()); diff --git a/src/sqlancer/oceanbase/OceanBaseUserCheckException.java b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java new file mode 100644 index 000000000..3aedc6f48 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java @@ -0,0 +1,8 @@ +package sqlancer.oceanbase; + +public class OceanBaseUserCheckException extends RuntimeException { + + public OceanBaseUserCheckException(String s) { + super(s); + } +} diff --git a/src/sqlancer/oceanbase/OceanBaseVisitor.java b/src/sqlancer/oceanbase/OceanBaseVisitor.java index 5babfd7df..91f3549f7 100644 --- a/src/sqlancer/oceanbase/OceanBaseVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseVisitor.java @@ -1,8 +1,10 @@ package sqlancer.oceanbase; +import sqlancer.oceanbase.ast.OceanBaseAggregate; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseColumnReference; import sqlancer.oceanbase.ast.OceanBaseComputableFunction; import sqlancer.oceanbase.ast.OceanBaseConstant; @@ -13,10 +15,8 @@ import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseStringExpression; import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; -import sqlancer.oceanbase.ast.OceanBaseAggregate; -import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; public interface OceanBaseVisitor { @@ -52,7 +52,7 @@ public interface OceanBaseVisitor { void visit(OceanBaseColumnName c); void visit(OceanBaseText fun); - + void visit(OceanBaseUnaryPrefixOperation op); default void visit(OceanBaseExpression expr) { diff --git a/src/sqlancer/oceanbase/README.md b/src/sqlancer/oceanbase/README.md index 17940c678..ddac91aed 100644 --- a/src/sqlancer/oceanbase/README.md +++ b/src/sqlancer/oceanbase/README.md @@ -1,19 +1,33 @@ ## Install Oceanbase There are some methods to install OceanBase. A method to install a local single-node OceanBase cluster: +### Install OBD by using RPM packages (only for CentOS 7 or later) ```shell sudo yum install -y yum-utils sudo yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo sudo yum install -y ob-deploy source /etc/profile.d/obd.sh - +``` +### Start an OceanBase cluster +```shell git clone https://github.com/oceanbase/obdeploy.git cd obdeploy sudo obd cluster deploy c1 -c ./example/mini-local-example.yaml -A sudo obd cluster start c1 ``` +After you install OBD, you can run these commands as the root user to start a local single-node OceanBase cluster. +Before you run the commands, make sure that these conditions are met: + +- You have logged on as the root user. +- Ports `2881` and `2882` are available. +- Your server has at least 8 GB of memory. +- Your server has at least 2 CPU cores. + +> **NOTE:** If the preceding conditions are not met, see [OceanBase Deployer](https://github.com/oceanbase/obdeploy/blob/master/README.md). + > **NOTE:** We do not recommend that you use sys tenant to test. So please deploy clutser with optition -A, will create the test tenant during the bootstrap by using all available resources of the cluster. -> **NOTE:** Then you can create user. + +### Then create user for test. ```shell mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" diff --git a/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java index ddf50b745..13a0306bb 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java @@ -1,10 +1,9 @@ package sqlancer.oceanbase.ast; - public class OceanBaseAggregate implements OceanBaseExpression { - private OceanBaseExpression expr; - private OceanBaseAggregateFunction aggr; + private final OceanBaseExpression expr; + private final OceanBaseAggregateFunction aggr; public OceanBaseAggregate(OceanBaseExpression expr, OceanBaseAggregateFunction aggr) { this.expr = expr; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java index 3fe3bd32b..6edaa9357 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java @@ -1,6 +1,5 @@ package sqlancer.oceanbase.ast; -import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; @@ -75,6 +74,7 @@ public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseCo } } }; + private final String textRepresentation; public String getTextRepresentation() { @@ -96,7 +96,8 @@ public static BinaryComparisonOperator getRandom() { private final OceanBaseExpression right; private final BinaryComparisonOperator op; - public OceanBaseBinaryComparisonOperation(OceanBaseExpression left, OceanBaseExpression right, BinaryComparisonOperator op) { + public OceanBaseBinaryComparisonOperation(OceanBaseExpression left, OceanBaseExpression right, + BinaryComparisonOperator op) { this.left = left; this.right = right; this.op = op; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java index 0cb9ded11..2f9f69a87 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java @@ -22,7 +22,7 @@ public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) return OceanBaseConstant.createFalse(); } } else if (right.isNull()) { - if (left.asBooleanNotNull()) { + if (left.asBooleanNotNull()) { return OceanBaseConstant.createNullConstant(); } else { return OceanBaseConstant.createFalse(); @@ -74,7 +74,8 @@ public static OceanBaseBinaryLogicalOperator getRandom() { } } - public OceanBaseBinaryLogicalOperation(OceanBaseExpression left, OceanBaseExpression right, OceanBaseBinaryLogicalOperator op) { + public OceanBaseBinaryLogicalOperation(OceanBaseExpression left, OceanBaseExpression right, + OceanBaseBinaryLogicalOperator op) { this.left = left; this.right = right; this.op = op; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java index c6ee0f350..ae1a2c001 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java @@ -4,7 +4,7 @@ public class OceanBaseColumnName implements OceanBaseExpression { - private OceanBaseSchema.OceanBaseColumn column; + private final OceanBaseSchema.OceanBaseColumn column; public OceanBaseColumnName(OceanBaseSchema.OceanBaseColumn column) { this.column = column; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java index 64f903267..19753f813 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java @@ -6,7 +6,7 @@ public class OceanBaseColumnReference implements OceanBaseExpression { private final OceanBaseColumn column; private final OceanBaseConstant value; - private boolean isRef = false; + private boolean isRef; public OceanBaseColumnReference(OceanBaseColumn column, OceanBaseConstant value) { this.column = column; @@ -30,12 +30,13 @@ public OceanBaseConstant getExpectedValue() { return value; } - public OceanBaseColumnReference setRef(boolean isRef){ - this.isRef = isRef; - return this; - } - public boolean getRef(){ - return isRef; + public OceanBaseColumnReference setRef(boolean isRef) { + this.isRef = isRef; + return this; + } + + public boolean getRef() { + return isRef; } } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java index ee78dd584..e9dd03ebe 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java @@ -14,7 +14,7 @@ public class OceanBaseComputableFunction implements OceanBaseExpression { public OceanBaseComputableFunction(OceanBaseFunction func, OceanBaseExpression... args) { this.func = func; - this.args = args; + this.args = args.clone(); } public OceanBaseFunction getFunction() { @@ -22,7 +22,7 @@ public OceanBaseFunction getFunction() { } public OceanBaseExpression[] getArguments() { - return args; + return args.clone(); } public enum OceanBaseFunction { @@ -77,7 +77,7 @@ public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] o return castToMostGeneralType(result, new OceanBaseExpression[] { origArgs[1], origArgs[2] }); } }, - + IFNULL(2, "IFNULL") { @Override @@ -87,7 +87,7 @@ public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] o result = args[1]; } else { result = args[0]; - }//args[0] and args[1] both null, if type is varchar, return null of varchar + } // args[0] and args[1] both null, if type is varchar, return null of varchar return castToMostGeneralType(result, origArgs); } @@ -111,11 +111,12 @@ public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpre final int nrArgs; private final boolean variadic; - private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] typeExpressions, BinaryOperator op) { + private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, + OceanBaseExpression[] typeExpressions, BinaryOperator op) { boolean containsNull = Stream.of(evaluatedArgs).anyMatch(arg -> arg.isNull()); if (containsNull) { - //IFNULL(GREATEST('aa',NULL), 0) -> '0' - // case1:c1 is float，value is NULL;select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' + // IFNULL(GREATEST('aa',NULL), 0) -> '0' + // case1:c1 is float，value is NULL;select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' // select COALESCE(GREATEST(1, concat(t1.c1)), 1) from t1;->1 // select COALESCE(GREATEST('0', 1, concat(t1.c1)), 1) from t1;->1 // select COALESCE(GREATEST('0', concat(t1.c1)), 1) from t1;->'1' @@ -125,31 +126,35 @@ private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, Oc // select IFNULL(GREATEST("iffI|2&nBJLQQ", NULL, '0'), 1) from t0;->'1' OceanBaseDataType type; boolean allVarchar = true; - for (OceanBaseExpression expr : typeExpressions) { - if (expr instanceof OceanBaseColumnReference){ + for (OceanBaseExpression expr : typeExpressions) { + if (expr instanceof OceanBaseColumnReference) { type = ((OceanBaseColumnReference) expr).getColumn().getType(); - if(type == OceanBaseDataType.FLOAT) + if (type == OceanBaseDataType.FLOAT) { type = OceanBaseDataType.VARCHAR; - }else + } + } else { type = expr.getExpectedValue().getType(); - if(type != null && type.isNumeric()){ + } + if (type != null && type.isNumeric()) { allVarchar = false; break; } } - if (allVarchar) + if (allVarchar) { return OceanBaseConstant.createStringConstant("null"); - else + } else { return OceanBaseConstant.createNullConstant(); + } } OceanBaseConstant least = evaluatedArgs[1]; - /*select least(1,'H8*GPLuBjDj#Xem]W'); -> 0 - select least('1','H8*GPLuBjDj#Xem]W'); ->1 - select LEAST('000000000001', 'b', 1);->0*/ + /* + * select least(1,'H8*GPLuBjDj#Xem]W'); -> 0 select least('1','H8*GPLuBjDj#Xem]W'); ->1 select + * LEAST('000000000001', 'b', 1);->0 + */ OceanBaseDataType dataType = evaluatedArgs[0].getType(); boolean sameDataType = true; for (OceanBaseConstant arg : evaluatedArgs) { - if (arg.getType() != dataType){ + if (arg.getType() != dataType) { sameDataType = false; break; } @@ -157,19 +162,21 @@ private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, Oc for (OceanBaseConstant arg : evaluatedArgs) { OceanBaseConstant left; OceanBaseConstant right; - if(sameDataType){ + if (sameDataType) { left = least; right = arg; - }else{ - //select GREATEST('1.47529e18', -1188315266);->1.47529e18 - if(least.getType() == OceanBaseDataType.VARCHAR) + } else { + // select GREATEST('1.47529e18', -1188315266);->1.47529e18 + if (least.getType() == OceanBaseDataType.VARCHAR) { left = least.castAsDouble(); - else + } else { left = least; - if(arg.getType() == OceanBaseDataType.VARCHAR) + } + if (arg.getType() == OceanBaseDataType.VARCHAR) { right = arg.castAsDouble(); - else + } else { right = arg; + } } least = op.apply(right, left); } @@ -192,7 +199,7 @@ public int getNrArgs() { return nrArgs; } - public abstract OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args); + public abstract OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args); public static OceanBaseFunction getRandomFunction() { return Randomly.fromOptions(values()); @@ -221,27 +228,29 @@ public OceanBaseConstant getExpectedValue() { return func.apply(constants, args); } - public static OceanBaseConstant castToMostGeneralType(OceanBaseConstant cons, OceanBaseExpression... typeExpressions) { + public static OceanBaseConstant castToMostGeneralType(OceanBaseConstant cons, + OceanBaseExpression... typeExpressions) { OceanBaseDataType type = getMostGeneralType(typeExpressions); if (cons.isNull()) { - if (type == OceanBaseDataType.FLOAT || type == OceanBaseDataType.VARCHAR) + if (type == OceanBaseDataType.FLOAT || type == OceanBaseDataType.VARCHAR) { return OceanBaseConstant.createStringConstant("null"); - else + } else { return cons; - }else{ - switch (type) { - case INT: - if (cons.isInt()) { + } + } else { + switch (type) { + case INT: + if (cons.isInt()) { + return cons; + } else { + return OceanBaseConstant.createIntConstant(cons.castAs(CastType.SIGNED).getInt()); + } + case VARCHAR: + return OceanBaseConstant.createStringConstant(cons.castAsString()); + default: return cons; - } else { - return OceanBaseConstant.createIntConstant(cons.castAs(CastType.SIGNED).getInt()); } - case VARCHAR: - return OceanBaseConstant.createStringConstant(cons.castAsString()); - default: - return cons; } - } } public static OceanBaseDataType getMostGeneralType(OceanBaseExpression... expressions) { @@ -250,15 +259,17 @@ public static OceanBaseDataType getMostGeneralType(OceanBaseExpression... expres OceanBaseDataType exprType; if (expr instanceof OceanBaseColumnReference) { exprType = ((OceanBaseColumnReference) expr).getColumn().getType(); - if(((OceanBaseColumnReference) expr).getColumn().isZeroFill()) + if (((OceanBaseColumnReference) expr).getColumn().isZeroFill()) { exprType = OceanBaseDataType.VARCHAR; + } } else { exprType = expr.getExpectedValue().getType(); } if (type == null) { type = exprType; - if (exprType == OceanBaseDataType.FLOAT) + if (exprType == OceanBaseDataType.FLOAT) { type = OceanBaseDataType.VARCHAR; + } } else if (exprType == OceanBaseDataType.VARCHAR || exprType == OceanBaseDataType.FLOAT) { type = OceanBaseDataType.VARCHAR; } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java index d99447fb6..174defdb8 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -1,7 +1,7 @@ package sqlancer.oceanbase.ast; -import java.math.BigInteger; import java.math.BigDecimal; +import java.math.BigInteger; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -17,7 +17,7 @@ public boolean isInt() { public boolean isNull() { return false; } - + public boolean isDouble() { return false; } @@ -85,7 +85,7 @@ public OceanBaseDoubleConstant(double val) { public String getTextRepresentation() { return String.valueOf(val); } - + @Override public double getDouble() { return this.val; @@ -105,22 +105,24 @@ public boolean asBooleanNotNull() { public OceanBaseConstant castAs(CastType type) { if (type == CastType.SIGNED) { long value = new Double(val).longValue(); - if (val - value >=0.5) - value = value +1; + if (val - value >= 0.5) { + value = value + 1; + } return new OceanBaseIntConstant(value, true); } else if (type == CastType.UNSIGNED) { long value = new Double(val).longValue(); - if (val - value >=0.5) - value = value +1; + if (val - value >= 0.5) { + value = value + 1; + } return new OceanBaseIntConstant(value, false); } else { throw new AssertionError(); } - } + } @Override public String castAsString() { - return String.valueOf(new BigDecimal(val));//select IFNULL(1.713591018E9, '11') -> 1713591018 + return String.valueOf(new BigDecimal(val)); // select IFNULL(1.713591018E9, '11') -> 1713591018 } @Override @@ -132,31 +134,32 @@ public boolean isDouble() { protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { if (rightVal.isNull()) { return OceanBaseConstant.createNullConstant(); - } else if (rightVal instanceof OceanBaseIntConstant){ + } else if (rightVal instanceof OceanBaseIntConstant) { return OceanBaseConstant.createBoolean(val < rightVal.getInt()); - } else if (rightVal instanceof OceanBaseDoubleConstant){ + } else if (rightVal instanceof OceanBaseDoubleConstant) { return OceanBaseConstant.createBoolean(val < rightVal.getDouble()); - } else if (rightVal instanceof OceanBaseTextConstant){ + } else if (rightVal instanceof OceanBaseTextConstant) { return isLessThan(rightVal.castAsDouble()); - } else{ + } else { throw new AssertionError(rightVal); - } + } } @Override public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { if (rightVal.isNull()) { return OceanBaseConstant.createNullConstant(); - } else if (rightVal instanceof OceanBaseIntConstant){ + } else if (rightVal instanceof OceanBaseIntConstant) { return OceanBaseConstant.createBoolean(val == rightVal.getInt()); - } else if (rightVal instanceof OceanBaseDoubleConstant){ + } else if (rightVal instanceof OceanBaseDoubleConstant) { return OceanBaseConstant.createBoolean(val == rightVal.getDouble()); - } else if (rightVal instanceof OceanBaseTextConstant){ + } else if (rightVal instanceof OceanBaseTextConstant) { return isEquals(rightVal.castAsDouble()); } else { throw new AssertionError(rightVal); } } + @Override public OceanBaseDataType getType() { return OceanBaseDataType.DOUBLE; @@ -182,37 +185,32 @@ private void checkIfSmallFloatingPointText() { throw new IgnoreMeException(); } } + @Override public boolean isNull() { - if(value.equalsIgnoreCase("NULL")) - return true; - else - return false; + return value.equalsIgnoreCase("NULL"); } @Override - public boolean isEmpty(){ - //"" " " - if (value.length() ==0 ){ + public boolean isEmpty() { + // "" " " + if (value.length() == 0) { return true; - }else{ - for(int i =0;i< value.length(); i++){ - String sub =value.substring(i, i+1); - if(!sub.equals(" ")) - return false; + } else { + for (int i = 0; i < value.length(); i++) { + String sub = value.substring(i, i + 1); + if (!sub.equals(" ")) { + return false; + } } return true; - } + } } @Override public boolean asBooleanNotNull() { for (int i = value.length(); i >= 1; i--) { try { - char currentChar = value.charAt(i-1); - int currentVal= Integer.valueOf(currentChar); - if (currentVal < 48||currentVal > 57) - continue; String substring = value.substring(0, i); Double val = Double.valueOf(substring); return val != 0 && !Double.isNaN(val); @@ -247,10 +245,11 @@ public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { } else if (rightVal instanceof OceanBaseDoubleConstant) { return castAsDouble().isEquals(rightVal); } else if (rightVal.isString()) { - if (isEmpty() && rightVal.isEmpty()) + if (isEmpty() && rightVal.isEmpty()) { return OceanBaseConstant.createBoolean(true); - else + } else { return OceanBaseConstant.createBoolean(value.equalsIgnoreCase(rightVal.getString())); + } } else { throw new AssertionError(rightVal); } @@ -268,8 +267,9 @@ public boolean isString() { @Override public OceanBaseConstant castAs(CastType type) { - if(isNull()) + if (isNull()) { return OceanBaseConstant.createNullConstant(); + } if (type == CastType.SIGNED || type == CastType.UNSIGNED) { String value = this.value; while (value.startsWith(" ") || value.startsWith("\t") || value.startsWith("\n")) { @@ -280,11 +280,6 @@ public OceanBaseConstant castAs(CastType type) { } for (int i = value.length(); i >= 1; i--) { try { - //select CAST("꯵z)" AS SIGNED); ->0 - char currentChar = value.charAt(i-1); - int currentVal= Integer.valueOf(currentChar); - if (currentVal < 48||currentVal > 57) - throw new IgnoreMeException(); String substring = value.substring(0, i); long val = Long.parseLong(substring); return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); @@ -309,11 +304,6 @@ public OceanBaseConstant castAsDouble() { } for (int i = value.length(); i >= 1; i--) { try { - char currentChar = value.charAt(i-1); - int currentVal= Integer.valueOf(currentChar); - if (currentVal < 48||currentVal > 57) - //ignore special char - throw new IgnoreMeException(); String substring = value.substring(0, i); double val = Double.parseDouble(substring); return OceanBaseConstant.createDoubleConstant(val); @@ -414,7 +404,7 @@ public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { throw new IgnoreMeException(); } return isEquals(rightVal.castAs(CastType.SIGNED)); - } else if (rightVal instanceof OceanBaseDoubleConstant){ + } else if (rightVal instanceof OceanBaseDoubleConstant) { return OceanBaseConstant.createBoolean(value == rightVal.getDouble()); } else { throw new AssertionError(rightVal); @@ -440,7 +430,7 @@ public String castAsString() { return Long.toUnsignedString(value); } } - + @Override public OceanBaseConstant castAsDouble() { return this; @@ -481,7 +471,7 @@ protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { throw new IgnoreMeException(); } return isLessThan(rightVal.castAs(isSigned ? CastType.SIGNED : CastType.UNSIGNED)); - } else if (rightVal instanceof OceanBaseDoubleConstant){ + } else if (rightVal instanceof OceanBaseDoubleConstant) { return OceanBaseConstant.createBoolean(value < rightVal.getDouble()); } else { throw new AssertionError(rightVal); diff --git a/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java index e5c482a56..c4caec195 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java @@ -2,8 +2,6 @@ import java.util.List; -import sqlancer.IgnoreMeException; - public class OceanBaseInOperation implements OceanBaseExpression { private final OceanBaseExpression expr; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java index 642553173..a87ff5c27 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java @@ -1,7 +1,7 @@ package sqlancer.oceanbase.ast; -import java.util.Collections; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import sqlancer.common.ast.SelectBase; @@ -28,12 +28,15 @@ public SelectType getFromOptions() { public void setFromOptions(SelectType fromOptions) { this.fromOptions = fromOptions; } - public void setGroupByClause(List groupBys){ + + public void setGroupByClause(List groupBys) { this.groupBys = groupBys; } - public List getGroupByClause(){ + + public List getGroupByClause() { return this.groupBys; } + public void setModifiers(List modifiers) { this.modifiers = modifiers; } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseText.java b/src/sqlancer/oceanbase/ast/OceanBaseText.java index 566effde4..f64cba17c 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseText.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseText.java @@ -1,11 +1,10 @@ package sqlancer.oceanbase.ast; - public class OceanBaseText implements OceanBaseExpression { - private OceanBaseExpression expr; - private String text; - private boolean prefix; + private final OceanBaseExpression expr; + private final String text; + private final boolean prefix; public OceanBaseText(OceanBaseExpression expr, String text, boolean prefix) { this.expr = expr; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java index c598e3bd0..e659f75be 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java @@ -65,11 +65,12 @@ public OceanBaseUnaryPrefixOperation(OceanBaseExpression expr, OceanBaseUnaryPre @Override public OceanBaseConstant getExpectedValue() { OceanBaseConstant subExprVal = expr.getExpectedValue(); - if (op == OceanBaseUnaryPrefixOperator.PLUS){ - if (subExprVal.isNull() && subExprVal.getType() == null) + if (op == OceanBaseUnaryPrefixOperator.PLUS) { + if (subExprVal.isNull() && subExprVal.getType() == null) { return OceanBaseConstant.createNullConstant(); - else + } else { return subExprVal; + } } if (subExprVal.isNull()) { return OceanBaseConstant.createNullConstant(); @@ -82,9 +83,11 @@ public OceanBaseConstant getExpectedValue() { public OperatorKind getOperatorKind() { return OperatorKind.PREFIX; } + public OceanBaseExpression getExpr() { return expr; } + public OceanBaseUnaryPrefixOperator getOp() { return op; } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java index b3f7de7bb..e67256448 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; @@ -29,6 +28,7 @@ public static SQLQueryAdapter create(OceanBaseGlobalState globalState) { private enum Action { COMPRESSION; + private String[] potentialErrors; Action(String... couldCauseErrors) { @@ -59,6 +59,8 @@ private SQLQueryAdapter create() { sb.append(Randomly.fromOptions("ZLIB_1.0", "LZ4_1.0", "NONE")); sb.append("'"); break; + default: + break; } } for (Action a : selectedActions) { diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index de76917e8..f2a6b2731 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -15,6 +15,7 @@ public class OceanBaseDeleteGenerator { private final StringBuilder sb = new StringBuilder(); private final OceanBaseGlobalState globalState; private final Randomly r; + public OceanBaseDeleteGenerator(OceanBaseGlobalState globalState) { this.globalState = globalState; this.r = globalState.getRandomly(); @@ -26,11 +27,12 @@ public static SQLQueryAdapter delete(OceanBaseGlobalState globalState) { private SQLQueryAdapter generate() { OceanBaseTable randomTable = globalState.getSchema().getRandomTable(); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(randomTable.getColumns()); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState) + .setColumns(randomTable.getColumns()); ExpectedErrors errors = new ExpectedErrors(); sb.append("DELETE"); if (Randomly.getBoolean()) { - sb.append(" /*+parallel("+r.getLong(0, 10)+") enable_parallel_dml*/ "); + sb.append(" /*+parallel(" + r.getLong(0, 10) + ") enable_parallel_dml*/ "); } sb.append(" FROM "); sb.append(randomTable.getName()); @@ -39,12 +41,9 @@ private SQLQueryAdapter generate() { sb.append(OceanBaseVisitor.asString(gen.generateExpression())); OceanBaseErrors.addExpressionErrors(errors); } - errors.addAll(Arrays.asList("doesn't have this option", - "Truncated incorrect DOUBLE value", - "Truncated incorrect INTEGER value", - "Truncated incorrect DECIMAL value", - "Data truncated for functional index", - "Incorrect value", "Out of range value for column", + errors.addAll(Arrays.asList("doesn't have this option", "Truncated incorrect DOUBLE value", + "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", + "Data truncated for functional index", "Incorrect value", "Out of range value for column", "Data truncation: %s value is out of range in '%s'")); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java index 5e7c06fb2..58a17d3ac 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java @@ -1,7 +1,6 @@ package sqlancer.oceanbase.gen; import sqlancer.IgnoreMeException; -import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.oceanbase.OceanBaseGlobalState; @@ -22,9 +21,8 @@ public static SQLQueryAdapter generate(OceanBaseGlobalState globalState) { sb.append(table.getRandomIndex().getIndexName()); sb.append(" ON "); sb.append(table.getName()); - return new SQLQueryAdapter(sb.toString(), - ExpectedErrors.from("LOCK=NONE is not supported", "ALGORITHM=INPLACE is not supported", - "Data truncation", "Data truncated for functional index")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("LOCK=NONE is not supported", + "ALGORITHM=INPLACE is not supported", "Data truncation", "Data truncated for functional index")); } } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java index 279223d73..f4f573698 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java @@ -1,11 +1,9 @@ package sqlancer.oceanbase.gen; +import java.sql.Connection; import java.util.ArrayList; import java.util.List; -import java.sql.Connection; - -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.oceanbase.OceanBaseGlobalState; @@ -25,8 +23,6 @@ import sqlancer.oceanbase.ast.OceanBaseExists; import sqlancer.oceanbase.ast.OceanBaseExpression; import sqlancer.oceanbase.ast.OceanBaseInOperation; -import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; -import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; import sqlancer.oceanbase.ast.OceanBaseStringExpression; import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; @@ -36,27 +32,23 @@ public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator OceanBaseColumns = new ArrayList<>(); public OceanBaseExpressionGenerator(OceanBaseGlobalState state) { this.state = state; } + public OceanBaseExpressionGenerator setCon(Connection con) { return this; } + public OceanBaseExpressionGenerator setState(OceanBaseGlobalState state) { this.state = state; return this; } public OceanBaseExpressionGenerator setOceanBaseColumns(List columns) { - this.OceanBaseColumns = columns; return this; } - public OceanBaseExpressionGenerator(Randomly r) { - this.r = r; - } public OceanBaseExpressionGenerator setRowVal(OceanBaseRowValue rowVal) { this.rowVal = rowVal; @@ -64,7 +56,7 @@ public OceanBaseExpressionGenerator setRowVal(OceanBaseRowValue rowVal) { } private enum Actions { - COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, + COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, EXISTS; } @@ -95,7 +87,8 @@ public OceanBaseExpression generateExpression(int depth) { return new OceanBaseBinaryComparisonOperation(generateExpression(depth + 1), generateExpression(depth + 1), BinaryComparisonOperator.getRandom()); case CAST: - return new OceanBaseCastOperation(generateExpression(depth + 1), OceanBaseCastOperation.CastType.getRandom()); + return new OceanBaseCastOperation(generateExpression(depth + 1), + OceanBaseCastOperation.CastType.getRandom()); case IN_OPERATION: OceanBaseExpression expr = generateExpression(depth + 1); List rightList = new ArrayList<>(); @@ -114,7 +107,8 @@ private OceanBaseExpression getExists() { if (Randomly.getBoolean()) { return new OceanBaseExists(new OceanBaseStringExpression("SELECT 1", OceanBaseConstant.createTrue())); } else { - return new OceanBaseExists(new OceanBaseStringExpression("SELECT 1 from dual wHERE FALSE", OceanBaseConstant.createFalse())); + return new OceanBaseExists( + new OceanBaseStringExpression("SELECT 1 from dual wHERE FALSE", OceanBaseConstant.createFalse())); } } @@ -133,11 +127,12 @@ private OceanBaseExpression getComputableFunction(int depth) { private enum ConstantType { INT, NULL, STRING, DOUBLE; + public static ConstantType[] valuesPQS() { return new ConstantType[] { INT, NULL, STRING }; } } - + @Override public OceanBaseExpression generateConstant() { ConstantType[] values; @@ -146,6 +141,7 @@ public OceanBaseExpression generateConstant() { } else { values = ConstantType.values(); } + OceanBaseConstant constant; switch (Randomly.fromOptions(values)) { case INT: return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); @@ -153,11 +149,12 @@ public OceanBaseExpression generateConstant() { return OceanBaseConstant.createNullConstant(); case STRING: String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); - OceanBaseConstant createStringConstant = OceanBaseConstant.createStringConstant(string); - return createStringConstant; + constant = OceanBaseConstant.createStringConstant(string); + return constant; case DOUBLE: double val = state.getRandomly().getDouble(); - return new OceanBaseDoubleConstant(val); + constant = new OceanBaseDoubleConstant(val); + return constant; default: throw new AssertionError(); } @@ -176,6 +173,7 @@ public OceanBaseExpression generateColumn() { } public OceanBaseExpression generateConstant(OceanBaseColumn col) { + OceanBaseConstant constant; switch (col.getType().name()) { case "INT": return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); @@ -183,22 +181,24 @@ public OceanBaseExpression generateConstant(OceanBaseColumn col) { return OceanBaseConstant.createNullConstant(); case "VARCHAR": String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); - OceanBaseConstant createStringConstant = OceanBaseConstant.createStringConstant(string); - return createStringConstant; + constant = OceanBaseConstant.createStringConstant(string); + return constant; case "DOUBLE": double val = state.getRandomly().getDouble(); - return new OceanBaseDoubleConstant(val); + constant = new OceanBaseDoubleConstant(val); + return constant; case "FLOAT": - val = state.getRandomly().getDouble(); - return new OceanBaseDoubleConstant(val); + val = state.getRandomly().getDouble(); + constant = new OceanBaseDoubleConstant(val); + return constant; case "DECIMAL": - val = state.getRandomly().getDouble(); + val = state.getRandomly().getDouble(); return new OceanBaseDoubleConstant(val); default: throw new AssertionError(); } } - + @Override public OceanBaseExpression negatePredicate(OceanBaseExpression predicate) { return new OceanBaseUnaryPrefixOperation(predicate, OceanBaseUnaryPrefixOperator.NOT); @@ -206,7 +206,7 @@ public OceanBaseExpression negatePredicate(OceanBaseExpression predicate) { @Override public OceanBaseExpression isNull(OceanBaseExpression expr) { - return new OceanBaseUnaryPostfixOperation(expr, OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + return new OceanBaseUnaryPostfixOperation(expr, OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, + false); } } - diff --git a/src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java new file mode 100644 index 000000000..8c5b66841 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java @@ -0,0 +1,125 @@ +package sqlancer.oceanbase.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; + +public class OceanBaseHintGenerator { + private final OceanBaseSelect select; + private final List tables; + private final StringBuilder sb = new StringBuilder(); + private final Randomly r = new Randomly(); + + enum IndexHint { + PDML, NO_PRED_DEDUCE, MERGE_JOIN, HASH_JOIN, NL_JOIN, BNL_JOIN, NO_MERGE_JOIN, NO_HASH_JOIN, NO_NL_JOIN, + NO_BNL_JOIN, HASH_AGG, NL_MATERIALIZATION, LATE_MATERIALIZATION, USE_INDEX, TOPK, LEADING, ORDERED, NO_REWRITE; + } + + public OceanBaseHintGenerator(OceanBaseSelect select, List tables) { + this.select = select; + this.tables = tables; + } + + public static void generateHints(OceanBaseSelect select, List tables) { + new OceanBaseHintGenerator(select, tables).generate(); + + } + + private void generate() { + switch (Randomly.fromOptions(IndexHint.values())) { + case PDML: + sb.append(" parallel(" + r.getInteger(0, 10) + "),enable_parallel_dml "); + break; + case NO_PRED_DEDUCE: + sb.append("NO_PRED_DEDUCE"); + break; + case MERGE_JOIN: + tablesHint("USE_MERGE "); + break; + case HASH_JOIN: + tablesHint("USE_HASH "); + break; + case NL_JOIN: + tablesHint("USE_NL "); + break; + case BNL_JOIN: + tablesHint("USE_BNL "); + break; + case NO_MERGE_JOIN: + sb.append(" NO_USE_MERGE "); + break; + case NO_HASH_JOIN: + sb.append(" NO_USE_HASH "); + break; + case NO_NL_JOIN: + sb.append(" NO_USE_NL "); + break; + case NO_BNL_JOIN: + sb.append(" NO_USE_BNL "); + break; + case HASH_AGG: + sb.append("USE_HASH_AGGREGATION "); + break; + case NL_MATERIALIZATION: + sb.append("USE_NL_MATERIALIZATION "); + break; + case LATE_MATERIALIZATION: + sb.append("USE_LATE_MATERIALIZATION "); + break; + case USE_INDEX: + indexesHint("INDEX_HINT "); + break; + case TOPK: + sb.append("TOPK (50 50) "); + break; + case LEADING: + tablesHint(" LEADING "); + break; + case ORDERED: + sb.append("ORDERED "); + break; + case NO_REWRITE: + sb.append("NO_REWRITE "); + break; + default: + throw new AssertionError(); + } + + select.setHint(new OceanBaseStringExpression(sb.toString(), + new OceanBaseConstant.OceanBaseTextConstant(sb.toString()))); + } + + private void indexesHint(String string) { + sb.append(string); + sb.append("("); + OceanBaseTable table = Randomly.fromList(tables); + List allIndexes = table.getIndexes(); + if (allIndexes.isEmpty()) { + throw new IgnoreMeException(); + } + List indexSubset = Randomly.nonEmptySubset(allIndexes); + sb.append(table.getName()); + sb.append(", "); + sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); + sb.append(")"); + } + + private void tablesHint(String string) { + sb.append(string); + sb.append("("); + appendTables(); + sb.append(")"); + } + + private void appendTables() { + List tableSubset = Randomly.nonEmptySubset(tables); + sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + } +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java index aca0511ce..79d206f70 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java @@ -20,7 +20,7 @@ public class OceanBaseInsertGenerator { private final ExpectedErrors errors = new ExpectedErrors(); private final OceanBaseGlobalState globalState; private final Randomly r; - private int type = 0; + private int type; public OceanBaseInsertGenerator(OceanBaseGlobalState globalState) { this.globalState = globalState; @@ -46,7 +46,7 @@ private SQLQueryAdapter generateReplace() { private SQLQueryAdapter generateInsert() { sb.append("INSERT"); if (Randomly.getBoolean()) { - sb.append(" /*+parallel("+r.getLong(0, 10)+") enable_parallel_dml*/ "); + sb.append(" /*+parallel(" + r.getLong(0, 10) + ") enable_parallel_dml*/ "); } if (Randomly.getBoolean()) { sb.append(" "); @@ -62,7 +62,7 @@ private SQLQueryAdapter generateInto() { sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); sb.append(") "); sb.append("VALUES"); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); int nrRows; if (Randomly.getBoolean()) { nrRows = 1; @@ -84,12 +84,10 @@ private SQLQueryAdapter generateInto() { } sb.append(")"); } - if(Randomly.getBoolean() && type == 0){ - + if (Randomly.getBoolean() && type == 0) { List upcolumns = table.getRandomNonEmptyColumnSubset(); - if (upcolumns.size() > 0 ){ + if (!upcolumns.isEmpty()) { sb.append(" ON DUPLICATE KEY UPDATE "); - sb.append(upcolumns.get(0).getName()); sb.append("="); sb.append(gen.generateConstant(upcolumns.get(0))); @@ -106,5 +104,4 @@ private SQLQueryAdapter generateInto() { errors.add("Duplicated primary key"); return new SQLQueryAdapter(sb.toString(), errors); } - } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java index bf4f679a9..2a193c1ae 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java @@ -6,7 +6,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -23,8 +22,6 @@ public class OceanBaseTableGenerator { private final String tableName; private final Randomly r; private int columnId; - private boolean tableHasNullableColumn; - private int keysSpecified; private final List columns = new ArrayList<>(); private final OceanBaseSchema schema; private final OceanBaseGlobalState globalState; @@ -109,8 +106,8 @@ private void appendPartitionOptions() { } private enum TableOptions { - BS,BLOOM,AUTO_INCREMENT; - + BS, BLOOM, AUTO_INCREMENT; + public static List getRandomTableOptions() { List options; if (Randomly.getBooleanWithSmallProbability()) { @@ -134,25 +131,27 @@ private void appendTableOptions() { sb.append(", "); } switch (o) { - case AUTO_INCREMENT: - sb.append("AUTO_INCREMENT = " + r.getPositiveInteger()); - break; - case BLOOM: - sb.append("USE_BLOOM_FILTER = "); - if(Randomly.getBoolean()) - sb.append(" FALSE "); - else - sb.append(" true "); - break; - case BS: - sb.append(" BLOCK_SIZE = "); - if(Randomly.getBoolean()) - sb.append(" 16384 "); - else - sb.append(" 32768 "); - break; - default: - throw new AssertionError(o); + case AUTO_INCREMENT: + sb.append("AUTO_INCREMENT = " + r.getPositiveInteger()); + break; + case BLOOM: + sb.append("USE_BLOOM_FILTER = "); + if (Randomly.getBoolean()) { + sb.append(" FALSE "); + } else { + sb.append(" true "); + } + break; + case BS: + sb.append(" BLOCK_SIZE = "); + if (Randomly.getBoolean()) { + sb.append(" 16384 "); + } else { + sb.append(" 32768 "); + } + break; + default: + throw new AssertionError(o); } } } @@ -171,7 +170,7 @@ private enum ColumnOptions { private void appendColumnDefinition() { sb.append(" "); - + OceanBaseDataType randomType = OceanBaseDataType.getRandom(globalState); boolean isTextType = randomType == OceanBaseDataType.VARCHAR; appendTypeString(randomType); @@ -180,9 +179,6 @@ private void appendColumnDefinition() { boolean columnHasPrimaryKey = false; List columnOptions = Randomly.subset(ColumnOptions.values()); - if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { - tableHasNullableColumn = true; - } if (isTextType) { columnOptions.remove(ColumnOptions.PRIMARY_KEY); columnOptions.remove(ColumnOptions.UNIQUE); @@ -196,7 +192,6 @@ private void appendColumnDefinition() { if (Randomly.getBoolean()) { sb.append("NULL"); } - tableHasNullableColumn = true; isNull = true; } else { sb.append("NOT NULL"); @@ -204,13 +199,11 @@ private void appendColumnDefinition() { break; case UNIQUE: sb.append("UNIQUE"); - keysSpecified++; if (Randomly.getBoolean()) { sb.append(" KEY"); } break; case COMMENT: - // TODO: generate randomly sb.append(String.format("COMMENT '%s' ", "asdf")); break; case PRIMARY_KEY: @@ -238,7 +231,7 @@ private void appendTypeString(OceanBaseDataType randomType) { sb.append(Randomly.fromOptions("TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT")); if (Randomly.getBoolean()) { sb.append("("); - sb.append(Randomly.getNotCachedInteger(0, 255)); + sb.append(Randomly.getNotCachedInteger(0, 255)); sb.append(")"); } break; diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index ff20328d5..f89434498 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -1,28 +1,37 @@ package sqlancer.oceanbase.gen; +import java.util.List; + +import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.Randomly; -import sqlancer.oceanbase.*; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseVisitor; +public class OceanBaseUpdateGenerator { -import java.sql.SQLException; -import java.util.HashSet; -import java.util.List; -import java.util.Set; + private final StringBuilder sb = new StringBuilder(); + private final OceanBaseGlobalState globalState; + private final Randomly r; -public class OceanBaseUpdateGenerator { - private OceanBaseUpdateGenerator() { + public OceanBaseUpdateGenerator(OceanBaseGlobalState globalState) { + this.globalState = globalState; + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter update(OceanBaseGlobalState globalState) { + return new OceanBaseUpdateGenerator(globalState).generate(); } - public static SQLQueryAdapter getQuery(OceanBaseGlobalState globalState) throws SQLException { - Randomly r = globalState.getRandomly(); + private SQLQueryAdapter generate() { ExpectedErrors errors = new ExpectedErrors(); OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); - StringBuilder sb = new StringBuilder("UPDATE "); + sb.append("UPDATE "); if (Randomly.getBoolean()) { - sb.append(" /*+parallel("+r.getInteger(0, 10)+") enable_parallel_dml*/ "); + sb.append(" /*+parallel(" + r.getInteger(0, 10) + ") enable_parallel_dml*/ "); } sb.append(table.getName()); sb.append(" SET "); @@ -44,7 +53,7 @@ public static SQLQueryAdapter getQuery(OceanBaseGlobalState globalState) throws sb.append(" WHERE "); OceanBaseErrors.addExpressionErrors(errors); sb.append(OceanBaseVisitor.asString(gen.generateExpression())); - errors.add("Data Too Long"); + errors.add("Data Too Long"); } errors.add("Duplicated primary key"); OceanBaseErrors.addInsertErrors(errors); diff --git a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java index 09daea34a..b7812a7ff 100644 --- a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java +++ b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java @@ -1,9 +1,7 @@ package sqlancer.oceanbase.gen.datadef; -import java.util.List; -import java.util.Set; -import java.util.HashSet; import java.util.ArrayList; +import java.util.List; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; @@ -14,18 +12,13 @@ import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.OceanBaseVisitor; -import sqlancer.oceanbase.ast.OceanBaseExpression; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; public class OceanBaseIndexGenerator { private final Randomly r; private StringBuilder sb = new StringBuilder(); - private boolean columnIsPrimaryKey; - private boolean containsInPlace; private OceanBaseSchema schema; - private List columns = new ArrayList<>(); + private final List columns = new ArrayList<>(); private final OceanBaseGlobalState globalState; public OceanBaseIndexGenerator(OceanBaseSchema schema, Randomly r, OceanBaseGlobalState globalState) { @@ -47,17 +40,13 @@ public SQLQueryAdapter create() { indexType(); sb.append(" ON "); OceanBaseTable table = schema.getRandomTable(); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(table.getName()); sb.append("("); List randomColumn = table.getRandomNonEmptyColumnSubset(); int i = 0; for (OceanBaseColumn c : randomColumn) { if (i++ != 0) { - sb.append(", "); - } - if (c.isPrimaryKey()) { - columnIsPrimaryKey = true; + sb.append(", "); } c.isPartioned = true; columns.add(c); @@ -96,6 +85,7 @@ public SQLQueryAdapter create() { private enum PartitionOptions { HASH } + private void appendPartitionOptions() { if (Randomly.getBoolean()) { return; @@ -103,11 +93,11 @@ private void appendPartitionOptions() { OceanBaseColumn colIndex = Randomly.fromList(columns); - if(colIndex.isPartioned == false){ + if (!colIndex.isPartioned) { return; } - if(colIndex.getType() == OceanBaseDataType.VARCHAR){ + if (colIndex.getType() == OceanBaseDataType.VARCHAR) { sb.append(" PARTITION BY"); sb.append(" KEY"); sb.append(" ("); @@ -116,8 +106,7 @@ private void appendPartitionOptions() { sb.append(")"); sb.append(" partitions "); sb.append(r.getInteger(1, 20)); - } - else if(OceanBaseDataType.INT == colIndex.getType()){ + } else if (OceanBaseDataType.INT == colIndex.getType()) { sb.append(" PARTITION BY"); sb.append(" HASH("); String name = colIndex.getName(); @@ -125,8 +114,7 @@ else if(OceanBaseDataType.INT == colIndex.getType()){ sb.append(") "); sb.append(" partitions "); sb.append(r.getInteger(1, 20)); - } - else{ + } else { return; } } diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java index 85748870e..900ac8caa 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java @@ -1,5 +1,11 @@ package sqlancer.oceanbase.oracle; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; @@ -8,37 +14,34 @@ import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; import sqlancer.oceanbase.OceanBaseVisitor; -import sqlancer.oceanbase.ast.*; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseComputableFunction; import sqlancer.oceanbase.ast.OceanBaseComputableFunction.OceanBaseFunction; import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; -import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; -import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.Random; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { +public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { // SELECT COUNT(*) FROM t0 WHERE ; // SELECT SUM(count) FROM (SELECT IS TRUE as count FROM t0); // SELECT (SELECT COUNT(*) FROM t0 WHERE c0 IS NOT 0) = (SELECT COUNT(*) FROM // (SELECT c0 is NOT 0 FROM t0)); - private OceanBaseSchema s; - private Randomly r; + private final OceanBaseSchema s; private String firstQueryString; - private String secondQueryString; private static final int NOT_FOUND = -1; + private enum Option { + TRUE, FALSE_NULL, NOT_NOT_TRUE, NOT_FALSE_NOT_NULL, IF, IFNULL, COALESCE + }; + public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { super(globalState); this.s = globalState.getSchema(); @@ -56,14 +59,15 @@ public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { errors.add("digit expected after"); } + @Override public void check() throws SQLException { OceanBaseSchema.OceanBaseTable randomTable = s.getRandomTable(); List columns = randomTable.getColumns(); OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state).setColumns(columns); OceanBaseExpression randomWhereCondition = gen.generateExpression(); List groupBys = Collections.emptyList(); // getRandomExpressions(columns); - List tableList = (Arrays.asList(randomTable)).stream().map(t -> new OceanBaseTableReference(t)) - .collect(Collectors.toList()); + List tableList = Arrays.asList(randomTable).stream() + .map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList()); int firstCount = getFirstQueryCount(tableList, randomWhereCondition, groupBys); int secondCount = getSecondQuery(tableList, randomWhereCondition, groupBys); if (firstCount != secondCount && firstCount != NOT_FOUND && secondCount != NOT_FOUND) { @@ -79,7 +83,7 @@ public void check() throws SQLException { } private int getSecondQuery(List tableList, OceanBaseExpression randomWhereCondition, - List groupBys) throws SQLException { + List groupBys) throws SQLException { OceanBaseSelect select = new OceanBaseSelect(); select.setGroupByClause(groupBys); OceanBaseExpression expr = getTrueExpr(randomWhereCondition); @@ -117,8 +121,8 @@ private int getFirstQueryCount(List tableList, OceanBaseExp select.setGroupByClause(groupBys); // SELECT COUNT(t1.c3) FROM t1 WHERE (- (t1.c2)); // SELECT SUM(count) FROM (SELECT ((- (t1.c2)) IS TRUE) as count FROM t1);; - OceanBaseAggregate aggr = new OceanBaseAggregate( - new OceanBaseColumnName(new OceanBaseSchema.OceanBaseColumn("*", OceanBaseSchema.OceanBaseDataType.INT, false, 0, false)), + OceanBaseAggregate aggr = new OceanBaseAggregate(new OceanBaseColumnName( + new OceanBaseSchema.OceanBaseColumn("*", OceanBaseSchema.OceanBaseDataType.INT, false, 0, false)), OceanBaseAggregate.OceanBaseAggregateFunction.COUNT); select.setFetchColumns(Arrays.asList(aggr)); select.setFromList(tableList); @@ -145,60 +149,76 @@ private int getFirstQueryCount(List tableList, OceanBaseExp rs.close(); return firstCount; } - - private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition){ - int i = new Random().nextInt(8); - OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE,false); - - OceanBaseUnaryPostfixOperation isNotTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE,true); - - OceanBaseUnaryPostfixOperation isFalse= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE,false); - - OceanBaseUnaryPostfixOperation isNotFalse= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE,true); - - OceanBaseUnaryPostfixOperation isNULL= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL,false); - - OceanBaseUnaryPostfixOperation isNotNULL= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL,true); - - OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); - switch(i){ - case 0: - expr = isTrue; - break; - case 1: - expr = new OceanBaseUnaryPrefixOperation(new OceanBaseBinaryLogicalOperation(isFalse, isNULL, OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case 2: - expr = new OceanBaseUnaryPrefixOperation(new OceanBaseUnaryPrefixOperation(isTrue, OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case 3: - expr = new OceanBaseUnaryPrefixOperation(isNotTrue, OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case 4: - expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); - break; - case 5: - OceanBaseExpression[] args = new OceanBaseExpression[3]; - args[0] = OceanBaseConstant.createIntConstant(1); - args[1] = isTrue; - args[2] = OceanBaseConstant.createIntConstant(0); - expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); - break; - case 6: - OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; - ifArgs[0] = OceanBaseConstant.createNullConstant(); - ifArgs[1] = isTrue; - expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); - break; - case 7: - OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; - coalesceArgs[0] = OceanBaseConstant.createNullConstant(); - coalesceArgs[1] = isTrue; - expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); - break; + + private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition) { + // we can treat "is true" as combinations of "is flase" and "not","is not true" and "not",etc. + OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE, false); + + OceanBaseUnaryPostfixOperation isFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, false); + + OceanBaseUnaryPostfixOperation isNotFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, true); + + OceanBaseUnaryPostfixOperation isNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + + OceanBaseUnaryPostfixOperation isNotNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, true); + + OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); + Option a = Randomly.fromOptions(Option.values()); + switch (a) { + case TRUE: + expr = isTrue; + break; + case FALSE_NULL: + // not((is false) or (is null)) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseBinaryLogicalOperation(isFalse, isNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_NOT_TRUE: + // not(not(is true))) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseUnaryPrefixOperation(isTrue, + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_FALSE_NOT_NULL: + // (is not false) and (is not null) + expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); + break; + case IF: + // if(1, xx is true, 0) + OceanBaseExpression[] args = new OceanBaseExpression[3]; + args[0] = OceanBaseConstant.createIntConstant(1); + args[1] = isTrue; + args[2] = OceanBaseConstant.createIntConstant(0); + expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); + break; + case IFNULL: + // ifnull(null, xx is true) + OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; + ifArgs[0] = OceanBaseConstant.createNullConstant(); + ifArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); + break; + case COALESCE: + // coalesce(null, xx is true) + OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; + coalesceArgs[0] = OceanBaseConstant.createNullConstant(); + coalesceArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); + break; + default: + expr = isTrue; + break; } return expr; } - } diff --git a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java index 0b9c04040..48fe1002e 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java @@ -1,9 +1,9 @@ package sqlancer.oceanbase.oracle; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.ArrayList; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -21,14 +21,14 @@ import sqlancer.oceanbase.ast.OceanBaseColumnReference; import sqlancer.oceanbase.ast.OceanBaseConstant; import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseTableReference; import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation.UnaryPostfixOperator; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; -import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; -import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; public class OceanBasePivotedQuerySynthesisOracle @@ -40,7 +40,7 @@ public class OceanBasePivotedQuerySynthesisOracle public OceanBasePivotedQuerySynthesisOracle(OceanBaseGlobalState globalState) throws SQLException { super(globalState); OceanBaseErrors.addExpressionErrors(errors); - errors.add("in 'order clause'"); + errors.add("in 'order clause'"); errors.add("value is out of range"); } @@ -54,9 +54,11 @@ public Query getRectifiedQuery() throws SQLException { columns = randomFromTables.getColumns(); pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); - selectStatement.setFromList(tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList())); + selectStatement + .setFromList(tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList())); - fetchColumns = columns.stream().map(c -> new OceanBaseColumnReference(c, null)).map(d -> d.setRef(true)).collect(Collectors.toList()); + fetchColumns = columns.stream().map(c -> new OceanBaseColumnReference(c, null)).map(d -> d.setRef(true)) + .collect(Collectors.toList()); selectStatement.setFetchColumns(fetchColumns); OceanBaseExpression whereClause = generateRectifiedExpression(columns, pivotRow); selectStatement.setWhereClause(whereClause); @@ -135,11 +137,13 @@ protected Query getContainmentCheckQuery(Query query) throws S if (i++ != 0) { sb.append(" AND "); } - if(pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) - sb.append("concat("); + if (pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) { + sb.append("concat("); + } sb.append("result." + c.getTable().getName() + c.getName()); - if(pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) - sb.append(",'')"); + if (pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) { + sb.append(",'')"); + } if (pivotRow.getValues().get(c).isNull()) { sb.append(" IS NULL"); } else { diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java index 3e05a305b..51f06a3fc 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java @@ -2,26 +2,26 @@ import java.sql.SQLException; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTables; -import sqlancer.oceanbase.ast.*; -import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseTableReference; import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; import sqlancer.oceanbase.gen.OceanBaseHintGenerator; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -public abstract class OceanBaseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { +public abstract class OceanBaseTLPBase + extends TernaryLogicPartitioningOracleBase implements TestOracle { OceanBaseSchema s; OceanBaseTables targetTables; @@ -53,7 +53,7 @@ public void check() throws SQLException { List generateFetchColumns() { return Arrays.asList(OceanBaseColumnReference.create(targetTables.getColumns().get(0), null)); } - + @Override protected ExpressionGenerator getGen() { return gen; diff --git a/test/sqlancer/dbms/TestOceanBaseNoREC.java b/test/sqlancer/dbms/TestOceanBaseNoREC.java index ddf6bcda0..9c650265c 100644 --- a/test/sqlancer/dbms/TestOceanBaseNoREC.java +++ b/test/sqlancer/dbms/TestOceanBaseNoREC.java @@ -17,12 +17,12 @@ public void testNoREC() { assumeTrue(oceanBaseIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--database-prefix", - "pqsdb", "--num-queries", - TestConfig.NUM_QUERIES, - "--username","sqlancer@test", "--password", "sqlancer", - //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "--num-threads", "4", "--database-prefix", "norecdb", "--num-queries", TestConfig.NUM_QUERIES, + "--username", "sqlancer@test", "--password", "sqlancer", + // after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then + // create user sqlancer: + // mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by + // 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "NoREC" })); } diff --git a/test/sqlancer/dbms/TestOceanBasePQS.java b/test/sqlancer/dbms/TestOceanBasePQS.java index 4123e76c0..adb2007d5 100644 --- a/test/sqlancer/dbms/TestOceanBasePQS.java +++ b/test/sqlancer/dbms/TestOceanBasePQS.java @@ -17,12 +17,13 @@ public void testPQS() { assumeTrue(oceanBaseIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", - "pqsdb", "--num-queries", - TestConfig.NUM_QUERIES, - "--username","sqlancer@test", "--password", "sqlancer", - //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", + "--database-prefix", "pqsdb", "--num-queries", TestConfig.NUM_QUERIES, "--username", + "sqlancer@test", "--password", "sqlancer", + // after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then + // create user sqlancer: + // mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by + // 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "PQS" })); } diff --git a/test/sqlancer/dbms/TestOceanBaseTLP.java b/test/sqlancer/dbms/TestOceanBaseTLP.java index 6eef3fc91..fe84118e6 100644 --- a/test/sqlancer/dbms/TestOceanBaseTLP.java +++ b/test/sqlancer/dbms/TestOceanBaseTLP.java @@ -17,12 +17,12 @@ public void testTLP() { assumeTrue(oceanBaseIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--database-prefix", - "pqsdb", "--num-queries", - TestConfig.NUM_QUERIES, - "--username","sqlancer@test", "--password", "sqlancer", - //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "--num-threads", "4", "--database-prefix", "tlpdb", "--num-queries", TestConfig.NUM_QUERIES, + "--username", "sqlancer@test", "--password", "sqlancer", + // after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then + // create user sqlancer: + // mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by + // 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "TLP_WHERE" })); } From 473aec4b10e3030c186d0d49736997f74ee41566 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 18 Dec 2021 11:49:55 +0100 Subject: [PATCH 0470/1284] Update H2 --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac460d4be..2cf0a2d07 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -152,8 +152,8 @@ jobs: git clone https://github.com/h2database/h2database cd h2database/h2 ./build.sh jar - cp bin/h2-2.0.201.jar ../../target/lib/h2-1.4.200.jar - cp bin/h2-2.0.201.jar ~/.m2/repository/com/h2database/h2/1.4.200/h2-1.4.200.jar + cp bin/h2-2.0.203-SNAPSHOT.jar ../../target/lib/h2-1.4.200.jar + cp bin/h2-2.0.203-SNAPSHOT.jar ~/.m2/repository/com/h2database/h2/1.4.200/h2-1.4.200.jar ls -lh ../../target/lib - name: Run Tests run: H2_AVAILABLE=true mvn -Dtest=TestH2 test From f06df58ca2b7ca600634b1f00b0ccb25c3905112 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 18:49:27 +0000 Subject: [PATCH 0471/1284] Bump ecj from 3.27.0 to 3.28.0 Bumps ecj from 3.27.0 to 3.28.0. --- updated-dependencies: - dependency-name: org.eclipse.jdt:ecj dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9a998533e..5f8c44003 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ org.eclipse.jdt ecj - 3.27.0 + 3.28.0 From f2940d54af4d32b623f3568d171d177efadf5f28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 19:02:25 +0000 Subject: [PATCH 0472/1284] Bump formatter-maven-plugin from 2.17.0 to 2.17.1 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.17.0 to 2.17.1. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.17.0...formatter-maven-plugin-2.17.1) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f8c44003..4b68b6ab2 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.17.0 + 2.17.1 eclipseformat From 098209f6dc10de7d5b9d6c05c34d0c7d3aee6df3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 19:28:34 +0000 Subject: [PATCH 0473/1284] Bump junit-jupiter-engine from 5.8.1 to 5.8.2 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.8.1 to 5.8.2. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.8.1...r5.8.2) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4b68b6ab2..10700e923 100644 --- a/pom.xml +++ b/pom.xml @@ -248,7 +248,7 @@ org.junit.jupiter junit-jupiter-engine - 5.8.1 + 5.8.2 test From f12363af9d2fdf57c6a2bd9bc9e0dfd4cbb0ebcb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 19:59:04 +0000 Subject: [PATCH 0474/1284] Bump h2 from 1.4.200 to 2.0.202 Bumps [h2](https://github.com/h2database/h2database) from 1.4.200 to 2.0.202. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-1.4.200...version-2.0.202) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 10700e923..31a3552c4 100644 --- a/pom.xml +++ b/pom.xml @@ -264,7 +264,7 @@ com.h2database h2 - 1.4.200 + 2.0.202 org.mongodb From b469d181105cd2d2ce7c3c7cbd831c41b7259f1a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 18 Dec 2021 21:33:14 +0100 Subject: [PATCH 0475/1284] No longer update H2 to the latest version Previously, H2 had not been updated in a long time. Currently, updating to the latest version is no longer necessary. --- .github/workflows/main.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cf0a2d07..e11a8d053 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -147,14 +147,6 @@ jobs: java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up H2 - run: | - git clone https://github.com/h2database/h2database - cd h2database/h2 - ./build.sh jar - cp bin/h2-2.0.203-SNAPSHOT.jar ../../target/lib/h2-1.4.200.jar - cp bin/h2-2.0.203-SNAPSHOT.jar ~/.m2/repository/com/h2database/h2/1.4.200/h2-1.4.200.jar - ls -lh ../../target/lib - name: Run Tests run: H2_AVAILABLE=true mvn -Dtest=TestH2 test From 6a6ab923a2e699c0e3a61cdd169c6e45e0342faa Mon Sep 17 00:00:00 2001 From: bashaojing Date: Wed, 22 Dec 2021 16:40:21 +0800 Subject: [PATCH 0476/1284] fix some build warnings --- .../oceanbase/OceanBaseUserCheckException.java | 2 ++ .../oceanbase/ast/OceanBaseComputableFunction.java | 12 ++++++------ src/sqlancer/oceanbase/ast/OceanBaseConstant.java | 8 +++----- .../gen/datadef/OceanBaseIndexGenerator.java | 4 ---- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/oceanbase/OceanBaseUserCheckException.java b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java index 3aedc6f48..38136fc8a 100644 --- a/src/sqlancer/oceanbase/OceanBaseUserCheckException.java +++ b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java @@ -2,6 +2,8 @@ public class OceanBaseUserCheckException extends RuntimeException { + private static final long serialVersionUID = 1L; + public OceanBaseUserCheckException(String s) { super(s); } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java index e9dd03ebe..0ac995d32 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java @@ -30,7 +30,7 @@ public enum OceanBaseFunction { BIT_COUNT(1, "BIT_COUNT") { @Override - public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args) { OceanBaseConstant arg = evaluatedArgs[0]; if (arg.isNull()) { return OceanBaseConstant.createNullConstant(); @@ -44,7 +44,7 @@ public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpre COALESCE(2, "COALESCE") { @Override - public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression... origArgs) { OceanBaseConstant result = OceanBaseConstant.createNullConstant(); for (OceanBaseConstant arg : args) { if (!arg.isNull()) { @@ -64,7 +64,7 @@ public boolean isVariadic() { IF(3, "IF") { @Override - public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression... origArgs) { OceanBaseConstant cond = args[0]; OceanBaseConstant left = args[1]; OceanBaseConstant right = args[2]; @@ -81,7 +81,7 @@ public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] o IFNULL(2, "IFNULL") { @Override - public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression... origArgs) { OceanBaseConstant result; if (args[0].isNull()) { result = args[1]; @@ -95,14 +95,14 @@ public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] o LEAST(2, "LEAST", true) { @Override - public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args) { return aggregate(evaluatedArgs, args, (min, cur) -> cur.isLessThan(min).asBooleanNotNull() ? cur : min); } }, GREATEST(2, "GREATEST", true) { @Override - public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args) { return aggregate(evaluatedArgs, args, (max, cur) -> cur.isLessThan(max).asBooleanNotNull() ? max : cur); } }; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java index 174defdb8..779d725b1 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -93,7 +93,7 @@ public double getDouble() { @Override public long getInt() { - return new Double(val).longValue(); + return (long) val; } @Override @@ -104,13 +104,13 @@ public boolean asBooleanNotNull() { @Override public OceanBaseConstant castAs(CastType type) { if (type == CastType.SIGNED) { - long value = new Double(val).longValue(); + long value = (long) val; if (val - value >= 0.5) { value = value + 1; } return new OceanBaseIntConstant(value, true); } else if (type == CastType.UNSIGNED) { - long value = new Double(val).longValue(); + long value = (long) val; if (val - value >= 0.5) { value = value + 1; } @@ -284,7 +284,6 @@ public OceanBaseConstant castAs(CastType type) { long val = Long.parseLong(substring); return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); } catch (NumberFormatException e) { - throw new IgnoreMeException(); } } return OceanBaseConstant.createIntConstant(0, type == CastType.SIGNED ? true : false); @@ -308,7 +307,6 @@ public OceanBaseConstant castAsDouble() { double val = Double.parseDouble(substring); return OceanBaseConstant.createDoubleConstant(val); } catch (NumberFormatException e) { - throw new IgnoreMeException(); } } return OceanBaseConstant.createIntConstant(0); diff --git a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java index b7812a7ff..726d9223d 100644 --- a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java +++ b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java @@ -82,10 +82,6 @@ public SQLQueryAdapter create() { return new SQLQueryAdapter(string, errors, true); } - private enum PartitionOptions { - HASH - } - private void appendPartitionOptions() { if (Randomly.getBoolean()) { return; From 85297ba89c7f58ebfbf2613027cac558695a0fbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Dec 2021 16:01:15 +0000 Subject: [PATCH 0477/1284] Bump spotbugs-maven-plugin from 4.5.0.0 to 4.5.2.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.5.0.0 to 4.5.2.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.5.0.0...spotbugs-maven-plugin-4.5.2.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 31a3552c4..e723a791e 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.5.0.0 + 4.5.2.0 spotbugs From cedfd697c7933771f8fef63976877c7bdcb97519 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Dec 2021 09:12:10 +0000 Subject: [PATCH 0478/1284] Bump h2 from 2.0.202 to 2.0.204 Bumps [h2](https://github.com/h2database/h2database) from 2.0.202 to 2.0.204. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.0.202...version-2.0.204) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e723a791e..95871af6a 100644 --- a/pom.xml +++ b/pom.xml @@ -264,7 +264,7 @@ com.h2database h2 - 2.0.202 + 2.0.204 org.mongodb From c842f2ffd9010423f3698f6f8a377a956fd9dba3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Dec 2021 12:07:03 +0100 Subject: [PATCH 0479/1284] Remove the obsolete H2 UNDO_LOG option --- src/sqlancer/h2/H2SetGenerator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sqlancer/h2/H2SetGenerator.java b/src/sqlancer/h2/H2SetGenerator.java index 88833e1e9..12e3b39eb 100644 --- a/src/sqlancer/h2/H2SetGenerator.java +++ b/src/sqlancer/h2/H2SetGenerator.java @@ -33,8 +33,7 @@ private enum Option { OPTIMIZE_REUSE_RESULTS((r) -> Randomly.fromOptions(0, 1)), // QUERY_STATISTICS((r) -> Randomly.fromOptions("TRUE", "FALSE")), // QUERY_STATISTICS_MAX_ENTRIES((r) -> Randomly.getNotCachedInteger(0, 100000)), // - REFERENTIAL_INTEGRITY((r) -> Randomly.fromOptions("TRUE", "FALSE")), // - UNDO_LOG((r) -> Randomly.fromOptions("TRUE", "FALSE")); + REFERENTIAL_INTEGRITY((r) -> Randomly.fromOptions("TRUE", "FALSE")); // private Function prod; From 8ef43e3456caec7b5469c0062618684e88ce54a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Dec 2021 09:11:31 +0000 Subject: [PATCH 0480/1284] Bump clickhouse-jdbc from 0.3.1-patch to 0.3.2 Bumps [clickhouse-jdbc](https://github.com/ClickHouse/clickhouse-jdbc) from 0.3.1-patch to 0.3.2. - [Release notes](https://github.com/ClickHouse/clickhouse-jdbc/releases) - [Changelog](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/ClickHouse/clickhouse-jdbc/compare/v0.3.1-patch...v0.3.2) --- updated-dependencies: - dependency-name: ru.yandex.clickhouse:clickhouse-jdbc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 95871af6a..97a948316 100644 --- a/pom.xml +++ b/pom.xml @@ -259,7 +259,7 @@ ru.yandex.clickhouse clickhouse-jdbc - 0.3.1-patch + 0.3.2 com.h2database From 1f0aabc7e57c2ce6ce0dd5d5220637008fa5b047 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 09:10:16 +0000 Subject: [PATCH 0481/1284] Bump h2 from 2.0.204 to 2.0.206 Bumps [h2](https://github.com/h2database/h2database) from 2.0.204 to 2.0.206. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.0.204...version-2.0.206) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97a948316..a40f09afc 100644 --- a/pom.xml +++ b/pom.xml @@ -264,7 +264,7 @@ com.h2database h2 - 2.0.204 + 2.0.206 org.mongodb From b198033cdea7f22de04fd733ac2d715fcf9fb483 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 09:15:41 +0000 Subject: [PATCH 0482/1284] Bump maven-jar-plugin from 3.2.0 to 3.2.1 Bumps [maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 3.2.0 to 3.2.1. - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.0...maven-jar-plugin-3.2.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a40f09afc..b1e66e1c1 100644 --- a/pom.xml +++ b/pom.xml @@ -127,7 +127,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.0 + 3.2.1 From ddf45799dc84e45781d6ccc8b4ecd81567380dfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 12:37:08 +0000 Subject: [PATCH 0483/1284] Bump spotbugs-maven-plugin from 4.5.2.0 to 4.5.3.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.5.2.0 to 4.5.3.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.5.2.0...spotbugs-maven-plugin-4.5.3.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b1e66e1c1..f1c20c736 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.5.2.0 + 4.5.3.0 spotbugs From 55f718c6a1cd593b4e4440ab90dc0c5845e24245 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jan 2022 09:13:39 +0000 Subject: [PATCH 0484/1284] Bump jcommander from 1.81 to 1.82 Bumps [jcommander](https://github.com/cbeust/jcommander) from 1.81 to 1.82. - [Release notes](https://github.com/cbeust/jcommander/releases) - [Changelog](https://github.com/cbeust/jcommander/blob/master/CHANGELOG.md) - [Commits](https://github.com/cbeust/jcommander/compare/1.81...1.82) --- updated-dependencies: - dependency-name: com.beust:jcommander dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f1c20c736..c48c7ae1b 100644 --- a/pom.xml +++ b/pom.xml @@ -218,7 +218,7 @@ com.beust jcommander - 1.81 + 1.82 org.postgresql From 9b48d6da9a829faea0aecc2e31ee415ccfe1116c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Oct 2021 09:10:15 +0000 Subject: [PATCH 0485/1284] Bump plexus-compiler-eclipse from 2.8.8 to 2.9.0 Bumps plexus-compiler-eclipse from 2.8.8 to 2.9.0. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c48c7ae1b..146ab3ff2 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.8.8 + 2.9.0 org.eclipse.jdt From 21e4ac3e1640b03fd16b115faa51297da2ee2efc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 11 Jan 2022 23:51:21 +0100 Subject: [PATCH 0486/1284] Add a plexus-compiler-related dependency to avoid an error See https://github.com/codehaus-plexus/plexus-compiler/issues/162. --- pom.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 146ab3ff2..eb2a81eeb 100644 --- a/pom.xml +++ b/pom.xml @@ -69,8 +69,8 @@ maven-compiler-plugin 3.8.1 - 8 - 8 + 1.8 + 1.8 ${project.basedir}/.settings/org.eclipse.jdt.core.prefs @@ -89,6 +89,11 @@ ecj 3.28.0 + + org.codehaus.plexus + plexus-compiler-api + 2.9.0 + From ec5af7f17902e56374f4ddc9304252bd0635ea65 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Jan 2022 21:20:50 +0100 Subject: [PATCH 0487/1284] Update the version to 2.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eb2a81eeb..b57ed0106 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.sqlancer sqlancer - 1.1.0 + 2.0.0 SQLancer http://www.sqlancer.com/ SQLancer finds logic bugs in Database Management Systems through automatic testing From 8774dc4b8d659271ae218830983c6c34b2593225 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Jan 2022 22:00:15 +0100 Subject: [PATCH 0488/1284] Enable manually publishing releases --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7777bdb6c..0ce243919 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,6 +2,7 @@ name: Publish package to the Maven Central Repository and Docker Hub on: release: types: [created] + workflow_dispatch: jobs: publish: runs-on: ubuntu-latest From 9571c574b8cd70bd5df745441adc3ccbc37e624f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Jan 2022 22:22:46 +0100 Subject: [PATCH 0489/1284] Update to Java 11 This is expected to fix an error when executing the release GitHub Action: Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.8.1:compile (default-compile) on project sqlancer: Execution default-compile of goal org.apache.maven.plugins:maven-compiler-plugin:3.8.1:compile failed: An API incompatibility was encountered while executing org.apache.maven.plugins:maven-compiler-plugin:3.8.1:compile: java.lang.UnsupportedClassVersionError: org/eclipse/jdt/core/compiler/CompilationProgress has been compiled by a more recent version of the Java Runtime (class file version 55.0), this version of the Java Runtime only recognizes class file versions up to 52.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b57ed0106..4c84a6b07 100644 --- a/pom.xml +++ b/pom.xml @@ -69,8 +69,8 @@ maven-compiler-plugin 3.8.1 - 1.8 - 1.8 + 11 + 11 ${project.basedir}/.settings/org.eclipse.jdt.core.prefs From 0e549f6085c0df8bbb2a8368287d8cd16508f6b5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Jan 2022 22:49:05 +0100 Subject: [PATCH 0490/1284] Also update the release GitHub Actions Java version to 11 --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0ce243919..342a358f3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: - name: Set up Maven Central Repository uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD @@ -28,10 +28,10 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Push to Docker Hub From 14f02a11bfc30d11ac6885bbf7b34b53eb712b31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 09:10:14 +0000 Subject: [PATCH 0491/1284] Bump maven-compiler-plugin from 3.8.1 to 3.9.0 Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.8.1 to 3.9.0. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.8.1...maven-compiler-plugin-3.9.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4c84a6b07..fe50b6321 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ maven-compiler-plugin - 3.8.1 + 3.9.0 11 11 From d343ac7f5e3bb4e31e1ad51108d55f442b94909b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 09:48:09 +0000 Subject: [PATCH 0492/1284] Bump maven-jar-plugin from 3.2.1 to 3.2.2 Bumps [maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 3.2.1 to 3.2.2. - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.1...maven-jar-plugin-3.2.2) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fe50b6321..d7e63786c 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.1 + 3.2.2 From 02544435e90a9b60296f2dc368fdbb8d1bd7f992 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 13 Jan 2022 00:07:36 +0100 Subject: [PATCH 0493/1284] Directly include the JCommander classes in the SQLancer JAR for better usability See https://github.com/sqlancer/sqlancer/issues/299. Do not include the JDBC drivers so that they can be easily replaced. --- pom.xml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pom.xml b/pom.xml index d7e63786c..4c115d93d 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,26 @@ src test + + org.apache.maven.plugins + maven-shade-plugin + 1.6 + + + package + + shade + + + + + com.beust:jcommander + + + + + + org.apache.maven.plugins maven-surefire-plugin @@ -125,6 +145,7 @@ false false true + jcommander From c46a1e81e108215a343fe38bcb1cc2411490ff2d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 14 Jan 2022 00:14:07 +0100 Subject: [PATCH 0494/1284] Force re-creating the JAR on every build This prevents warnings such as the following when the JAR already exists: [WARNING] We have a duplicate com/beust/ah/A.class in /home/manuel/.m2/repository/com/beust/jcommander/1.82/jcommander-1.82.jar --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 4c115d93d..7100f0dc7 100644 --- a/pom.xml +++ b/pom.xml @@ -155,6 +155,7 @@ maven-jar-plugin 3.2.2 + true true From bb013028ebd0a0200c4e1d130397736f52d0954b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 14 Jan 2022 00:28:46 +0100 Subject: [PATCH 0495/1284] Delete the backed-up original-sqlancer-*.jar file created by the shade plugin --- pom.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pom.xml b/pom.xml index 7100f0dc7..097828323 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,26 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + package + + run + + + + + + + + + + + org.apache.maven.plugins maven-surefire-plugin From 9de71b1d21c5089567cee3b116f588ccf78f521b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 09:14:35 +0000 Subject: [PATCH 0496/1284] Bump maven-antrun-plugin from 1.8 to 3.0.0 Bumps [maven-antrun-plugin](https://github.com/apache/maven-antrun-plugin) from 1.8 to 3.0.0. - [Release notes](https://github.com/apache/maven-antrun-plugin/releases) - [Commits](https://github.com/apache/maven-antrun-plugin/compare/maven-antrun-plugin-1.8...maven-antrun-plugin-3.0.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-antrun-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 097828323..f4b4150b2 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ org.apache.maven.plugins maven-antrun-plugin - 1.8 + 3.0.0 package From c51fb20fa02b9343d000c2434c88557dde76d90a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 09:28:18 +0000 Subject: [PATCH 0497/1284] Bump maven-shade-plugin from 1.6 to 3.2.4 Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 1.6 to 3.2.4. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-1.6...maven-shade-plugin-3.2.4) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-shade-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f4b4150b2..0a1930e9f 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ org.apache.maven.plugins maven-shade-plugin - 1.6 + 3.2.4 package From 7d69a1d42be348150c407638f296f20bc0817572 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 09:44:03 +0000 Subject: [PATCH 0498/1284] Bump slf4j-simple from 1.7.32 to 1.7.33 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.32 to 1.7.33. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.32...v_1.7.33) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a1930e9f..e35b7b3d2 100644 --- a/pom.xml +++ b/pom.xml @@ -301,7 +301,7 @@ org.slf4j slf4j-simple - 1.7.32 + 1.7.33 ru.yandex.clickhouse From 79681e323ba18dc8943d74eb13f6bf8734087154 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 17 Jan 2022 23:10:41 +0100 Subject: [PATCH 0499/1284] Add missing right parentheses in option descriptions --- src/sqlancer/clickhouse/ClickHouseOptions.java | 2 +- src/sqlancer/cockroachdb/CockroachDBOptions.java | 2 +- src/sqlancer/mariadb/MariaDBOptions.java | 2 +- src/sqlancer/mysql/MySQLOptions.java | 2 +- src/sqlancer/oceanbase/OceanBaseOptions.java | 2 +- src/sqlancer/postgres/PostgresOptions.java | 2 +- src/sqlancer/tidb/TiDBOptions.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index c25905648..c58697c98 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -19,7 +19,7 @@ import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "ClickHouse (default port: " + ClickHouseOptions.DEFAULT_PORT - + ", default host: " + ClickHouseOptions.DEFAULT_HOST) + + ", default host: " + ClickHouseOptions.DEFAULT_HOST + ")") public class ClickHouseOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 8123; diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 7673c4c0a..f8ab8e07f 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -23,7 +23,7 @@ import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "CockroachDB (default port: " + CockroachDBOptions.DEFAULT_PORT - + " default host: " + CockroachDBOptions.DEFAULT_HOST) + + " default host: " + CockroachDBOptions.DEFAULT_HOST + ")") public class CockroachDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 26257; diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 63e142aec..02e6eab64 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -15,7 +15,7 @@ import sqlancer.mariadb.oracle.MariaDBNoRECOracle; @Parameters(separators = "=", commandDescription = "MariaDB (default port: " + MariaDBOptions.DEFAULT_PORT - + ", default host: " + MariaDBOptions.DEFAULT_HOST) + + ", default host: " + MariaDBOptions.DEFAULT_HOST + ")") public class MariaDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 5540f00a8..2a3f16968 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -15,7 +15,7 @@ import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @Parameters(separators = "=", commandDescription = "MySQL (default port: " + MySQLOptions.DEFAULT_PORT - + ", default host: " + MySQLOptions.DEFAULT_HOST) + + ", default host: " + MySQLOptions.DEFAULT_HOST + ")") public class MySQLOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index eb7200437..b19de3983 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -16,7 +16,7 @@ import sqlancer.oceanbase.oracle.OceanBaseTLPWhereOracle; @Parameters(separators = "=", commandDescription = "OceanBase (default port: " + OceanBaseOptions.DEFAULT_PORT - + ", default host: " + OceanBaseOptions.DEFAULT_HOST) + + ", default host: " + OceanBaseOptions.DEFAULT_HOST + ")") public class OceanBaseOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 2881; diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 6c0a97847..ed2856451 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -20,7 +20,7 @@ import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; @Parameters(separators = "=", commandDescription = "PostgreSQL (default port: " + PostgresOptions.DEFAULT_PORT - + ", default host: " + PostgresOptions.DEFAULT_HOST) + + ", default host: " + PostgresOptions.DEFAULT_HOST + ")") public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index e5da34fe0..f9137f6f7 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -18,7 +18,7 @@ import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @Parameters(separators = "=", commandDescription = "TiDB (default port: " + TiDBOptions.DEFAULT_PORT - + ", default host: " + TiDBOptions.DEFAULT_HOST) + + ", default host: " + TiDBOptions.DEFAULT_HOST + ")") public class TiDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 4000; From 298bab1d418bb00407bfafe53fa48e917caa7d48 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 17 Jan 2022 23:20:37 +0100 Subject: [PATCH 0500/1284] Add a description to database implementations without any --- src/sqlancer/arangodb/ArangoDBOptions.java | 2 ++ src/sqlancer/duckdb/DuckDBOptions.java | 2 +- src/sqlancer/h2/H2Options.java | 3 +++ src/sqlancer/mongodb/MongoDBOptions.java | 2 ++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index 6e7a607f0..b821fe8c3 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -7,6 +7,7 @@ import java.util.List; import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; @@ -15,6 +16,7 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; +@Parameters(commandDescription = "ArangoDB (experimental)") public class ArangoDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index c163b5d0b..4e1c49cd1 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -21,7 +21,7 @@ import sqlancer.duckdb.test.DuckDBQueryPartitioningHavingTester; import sqlancer.duckdb.test.DuckDBQueryPartitioningWhereTester; -@Parameters +@Parameters(commandDescription = "DuckDB") public class DuckDBOptions implements DBMSSpecificOptions { @Parameter(names = "--test-collate", arity = 1) diff --git a/src/sqlancer/h2/H2Options.java b/src/sqlancer/h2/H2Options.java index 3ae576e1c..45b47f203 100644 --- a/src/sqlancer/h2/H2Options.java +++ b/src/sqlancer/h2/H2Options.java @@ -4,12 +4,15 @@ import java.util.Arrays; import java.util.List; +import com.beust.jcommander.Parameters; + import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; import sqlancer.h2.H2Options.H2OracleFactory; import sqlancer.h2.H2Provider.H2GlobalState; +@Parameters(commandDescription = "H2") public class H2Options implements DBMSSpecificOptions { public enum H2OracleFactory implements OracleFactory { diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 5c618f5f7..9e02de67b 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -8,6 +8,7 @@ import java.util.List; import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; @@ -17,6 +18,7 @@ import sqlancer.mongodb.test.MongoDBDocumentRemovalTester; import sqlancer.mongodb.test.MongoDBQueryPartitioningWhereTester; +@Parameters(commandDescription = "MongoDB (experimental)") public class MongoDBOptions implements DBMSSpecificOptions { @Parameter(names = "--test-validation", description = "Enable/Disable validation of schema with Schema Validation", arity = 1) From a4756aedac3377925c72842827ba5ca28d51a68e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 17 Jan 2022 23:28:52 +0100 Subject: [PATCH 0501/1284] Check the usage naming conventions --- src/sqlancer/Main.java | 3 -- test/sqlancer/TestUsageNamingConvention.java | 34 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 test/sqlancer/TestUsageNamingConvention.java diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a3ff789e4..c499ef9ec 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -404,9 +404,6 @@ public static int executeMain(String... args) throws AssertionError { Builder commandBuilder = JCommander.newBuilder().addObject(options); for (DatabaseProvider provider : providers) { String name = provider.getDBMSName(); - if (!name.toLowerCase().equals(name)) { - throw new AssertionError(name + " should be in lowercase!"); - } DBMSExecutorFactory executorFactory = new DBMSExecutorFactory<>(provider, options); commandBuilder = commandBuilder.addCommand(name, executorFactory.getCommand()); nameToProvider.put(name, executorFactory); diff --git a/test/sqlancer/TestUsageNamingConvention.java b/test/sqlancer/TestUsageNamingConvention.java new file mode 100644 index 000000000..94f5daba0 --- /dev/null +++ b/test/sqlancer/TestUsageNamingConvention.java @@ -0,0 +1,34 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.JCommander.Builder; +import com.beust.jcommander.Parameters; + +public class TestUsageNamingConvention { + + @Test + void testNonEmptyDescription() { + List> providers = Main.getDBMSProviders(); + MainOptions options = new MainOptions(); + Builder commandBuilder = JCommander.newBuilder().addObject(options); + for (DatabaseProvider provider : providers) { + String name = provider.getDBMSName(); + if (!name.toLowerCase().equals(name)) { + throw new AssertionError(name + " should be in lowercase!"); + } + commandBuilder.addCommand(provider.getDBMSName(), provider.getOptionClass()); + Parameters param = provider.getOptionClass().getAnnotation(Parameters.class); + assertNotEquals(null, param, provider.getOptionClass().toString()); + String databaseDescription = param.commandDescription(); + assertNotEquals(null, databaseDescription, "description cannot be empty " + provider.getOptionClass()); + assertNotEquals("", databaseDescription, "description cannot be empty " + provider.getOptionClass()); + } + } + +} From d00361ee457dc67c003471bbef4beccd06b7ceea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 09:12:23 +0000 Subject: [PATCH 0502/1284] Bump mysql-connector-java from 8.0.27 to 8.0.28 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.27 to 8.0.28. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/commits) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e35b7b3d2..9d17c95c0 100644 --- a/pom.xml +++ b/pom.xml @@ -280,7 +280,7 @@ mysql mysql-connector-java - 8.0.27 + 8.0.28 org.mariadb.jdbc From aee9b692c2862c9c6d7c779b329352777b2d3b81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 12:19:50 +0000 Subject: [PATCH 0503/1284] Bump h2 from 2.0.206 to 2.1.210 Bumps [h2](https://github.com/h2database/h2database) from 2.0.206 to 2.1.210. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.0.206...version-2.1.210) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9d17c95c0..dc709bda2 100644 --- a/pom.xml +++ b/pom.xml @@ -311,7 +311,7 @@ com.h2database h2 - 2.0.206 + 2.1.210 org.mongodb From 6924ce5d78b732df2f8b2e7cafdca97a8f358d1f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 18 Jan 2022 22:16:16 +0100 Subject: [PATCH 0504/1284] Remove the Travis configuration and other obsolete files --- .github/workflows~HEAD | 21 ------ .github/workflows~HEAD_0 | 21 ------ .github/workflows~HEAD_1 | 21 ------ .travis.yml | 146 --------------------------------------- codecov.yml | 16 ----- 5 files changed, 225 deletions(-) delete mode 100644 .github/workflows~HEAD delete mode 100644 .github/workflows~HEAD_0 delete mode 100644 .github/workflows~HEAD_1 delete mode 100644 .travis.yml delete mode 100644 codecov.yml diff --git a/.github/workflows~HEAD b/.github/workflows~HEAD deleted file mode 100644 index 0e057d405..000000000 --- a/.github/workflows~HEAD +++ /dev/null @@ -1,21 +0,0 @@ -name: CI tests - -on: [push, pull_request] - -jobs: - misc: - name: Misc tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build - run: mvn package -DskipTests=true - - name: Verify - run: mvn verify -DskipTests=true - - name: Misc Tests - run: mvn '-Dtest=!sqlancer.dbms.**' test diff --git a/.github/workflows~HEAD_0 b/.github/workflows~HEAD_0 deleted file mode 100644 index 0e057d405..000000000 --- a/.github/workflows~HEAD_0 +++ /dev/null @@ -1,21 +0,0 @@ -name: CI tests - -on: [push, pull_request] - -jobs: - misc: - name: Misc tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build - run: mvn package -DskipTests=true - - name: Verify - run: mvn verify -DskipTests=true - - name: Misc Tests - run: mvn '-Dtest=!sqlancer.dbms.**' test diff --git a/.github/workflows~HEAD_1 b/.github/workflows~HEAD_1 deleted file mode 100644 index 0e057d405..000000000 --- a/.github/workflows~HEAD_1 +++ /dev/null @@ -1,21 +0,0 @@ -name: CI tests - -on: [push, pull_request] - -jobs: - misc: - name: Misc tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build - run: mvn package -DskipTests=true - - name: Verify - run: mvn verify -DskipTests=true - - name: Misc Tests - run: mvn '-Dtest=!sqlancer.dbms.**' test diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 318ea6fa7..000000000 --- a/.travis.yml +++ /dev/null @@ -1,146 +0,0 @@ -dist: bionic -language: java - -script: -- cd src && python check_names.py && cd .. - -cache: - directories: - - target/lib -after_success: - - bash <(curl -s https://codecov.io/bash) -after_failure: - - cat target/pmd.xml -branches: - only: - - master - -matrix: - include: - - name: SQLite3 - jdk : openjdk8 - script: - - mvn -Dtest=TestSQLitePQS test - - mvn -Dtest=TestSQLite3 test - - name: PostgreSQL - jdk : openjdk8 - before_install: - - sudo apt-get update - - sudo apt-get --yes remove postgresql\* - - sudo apt-get install -y postgresql-12 postgresql-client-12 - - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/12/main/postgresql.conf - - sudo cp /etc/postgresql/{10,12}/main/pg_hba.conf - - sudo service postgresql restart 12 - addons: - postgresql: "12.3" - env: - global: - - PGPORT=5432 - services: - - postgresql - before_script: - - sudo apt-get install locales - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -U postgres - - createdb test -U postgres - script: - - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test - - name: MariaDB - jdk : openjdk8 - before_install: - - sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 - - sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main' - - sudo apt update - - sudo apt install mariadb-server - - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - - MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test - - name : MySQL - jdk : openjdk8 - script: - - sudo apt-get update && sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 - - sudo apt-get remove mysql-* && wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && yes | sudo dpkg -i *.deb - - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS - - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP - - name: CockroachDB - jdk : openjdk8 - before_install: - - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.2.linux-amd64.tgz | tar xvz - - cd cockroach-v20.1.2.linux-amd64/ && ./cockroach start-single-node --insecure & - - sleep 15 - - cd cockroach-v20.1.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - script: - - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - - name: TiDB - jdk : openjdk8 - services: - - docker - before_install: - - docker pull pingcap/tidb:latest - - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest - - sleep 15 - - sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - script: - - TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test - - name: DuckDB - jdk : openjdk8 - script: - - mvn -Dtest=TestDuckDB test - - name: DuckDB (Java 8) - jdk : openjdk8 - script: - - mvn -Dtest=TestDuckDB test - - name: DuckDB (java 13) - jdk : openjdk8 - script: - - mvn -Dtest=TestDuckDB test - - name: Misc - jdk : openjdk8 - script: - - mvn '-Dtest=!sqlancer.dbms.**' test - - name: Citus (PostgreSQL extension) - jdk : openjdk8 - before_install: - - sudo apt-get update - - sudo apt-get install -y postgresql-server-dev-12 postgresql-12 locales - - git clone https://github.com/citusdata/citus.git - - cd citus - - ./configure - - make clean -sj9 - - sudo make install -sj9 - - cd .. - before_script: - - sudo chown -R $USER:$USER /var/run/postgresql - - export PATH=/usr/lib/postgresql/12/bin:$PATH - - mkdir -p citus/coordinator citus/worker1 citus/worker2 - - initdb -D citus/coordinator - - initdb -D citus/worker1 - - initdb -D citus/worker2 - - echo "shared_preload_libraries = 'citus'" >> citus/coordinator/postgresql.conf - - echo "shared_preload_libraries = 'citus'" >> citus/worker1/postgresql.conf - - echo "shared_preload_libraries = 'citus'" >> citus/worker2/postgresql.conf - - pg_ctl -D citus/coordinator -o "-p 9700" -l coordinator_logfile start - - pg_ctl -D citus/worker1 -o "-p 9701" -l worker1_logfile start - - pg_ctl -D citus/worker2 -o "-p 9702" -l worker2_logfile start - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9700 -d postgres -U $USER - - createdb test -p 9700 -U $USER - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9701 -d postgres -U $USER - - createdb test -p 9701 -U $USER - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9702 -d postgres -U $USER - - createdb test -p 9702 -U $USER - - psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test - - psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test - - psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test - - psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test - - psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test - script: - - CITUS_AVAILABLE=true mvn -Dtest=TestCitus test - - name: ClickHouse - jdk : openjdk8 - services: - - docker - before_install: - - docker pull yandex/clickhouse-server:latest - - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest - - sleep 5 - script: - - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index ed891a44d..000000000 --- a/codecov.yml +++ /dev/null @@ -1,16 +0,0 @@ -codecov: - require_ci_to_pass: no - notify: - after_n_builds: 11 - -coverage: - range: "50...100" - -coverage: - status: - project: - default: - threshold: 2% - patch: - default: - threshold: 2% From f278315dbf040ebb69ca141f73b4584614c66925 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:22:05 +0100 Subject: [PATCH 0505/1284] Use a ServiceLoader to pick up database testing implementations This change allows SQLancer to pick up database providers in other JARs than the SQLancer JAR, if they are on the classpath. --- pom.xml | 5 +++ src/sqlancer/Main.java | 42 +++++++------------ src/sqlancer/arangodb/ArangoDBProvider.java | 3 ++ src/sqlancer/citus/CitusProvider.java | 4 ++ .../clickhouse/ClickHouseProvider.java | 4 ++ .../cockroachdb/CockroachDBProvider.java | 4 ++ src/sqlancer/cosmos/CosmosProvider.java | 3 ++ src/sqlancer/duckdb/DuckDBProvider.java | 4 ++ src/sqlancer/h2/H2Provider.java | 4 ++ src/sqlancer/mariadb/MariaDBProvider.java | 4 ++ src/sqlancer/mongodb/MongoDBProvider.java | 3 ++ src/sqlancer/mysql/MySQLProvider.java | 4 ++ src/sqlancer/oceanbase/OceanBaseProvider.java | 4 ++ src/sqlancer/postgres/PostgresProvider.java | 4 ++ src/sqlancer/sqlite3/SQLite3Provider.java | 4 ++ src/sqlancer/tidb/TiDBProvider.java | 4 ++ 16 files changed, 72 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index dc709bda2..bf7576302 100644 --- a/pom.xml +++ b/pom.xml @@ -262,6 +262,11 @@ + + com.google.auto.service + auto-service + 1.0.1 + com.beust jcommander diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index c499ef9ec..77ecdcabe 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -12,6 +12,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.ServiceLoader; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -21,23 +22,9 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.JCommander.Builder; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.citus.CitusProvider; -import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.cockroachdb.CockroachDBProvider; import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.cosmos.CosmosProvider; -import sqlancer.duckdb.DuckDBProvider; -import sqlancer.h2.H2Provider; -import sqlancer.mariadb.MariaDBProvider; -import sqlancer.mongodb.MongoDBProvider; -import sqlancer.mysql.MySQLProvider; -import sqlancer.oceanbase.OceanBaseProvider; -import sqlancer.postgres.PostgresProvider; -import sqlancer.sqlite3.SQLite3Provider; -import sqlancer.tidb.TiDBProvider; public final class Main { @@ -543,22 +530,21 @@ private boolean run(MainOptions options, ExecutorService execService, return threadsShutdown == 0 ? 0 : options.getErrorExitCode(); } + /** + * To register a new provider, it is necessary to implement the DatabaseProvider interface and add an additional + * configuration file, see https://docs.oracle.com/javase/9/docs/api/java/util/ServiceLoader.html. Currently, we use + * an @AutoService annotation to create the configuration file automatically. This allows SQLancer to pick up + * providers in other JARs on the classpath. + * + * @return The list of service providers on the classpath + */ static List> getDBMSProviders() { List> providers = new ArrayList<>(); - providers.add(new SQLite3Provider()); - providers.add(new CockroachDBProvider()); - providers.add(new MySQLProvider()); - providers.add(new MariaDBProvider()); - providers.add(new TiDBProvider()); - providers.add(new PostgresProvider()); - providers.add(new CitusProvider()); - providers.add(new ClickHouseProvider()); - providers.add(new DuckDBProvider()); - providers.add(new H2Provider()); - providers.add(new MongoDBProvider()); - providers.add(new CosmosProvider()); - providers.add(new ArangoDBProvider()); - providers.add(new OceanBaseProvider()); + @SuppressWarnings("rawtypes") + ServiceLoader loader = ServiceLoader.load(DatabaseProvider.class); + for (DatabaseProvider provider : loader) { + providers.add(provider); + } return providers; } diff --git a/src/sqlancer/arangodb/ArangoDBProvider.java b/src/sqlancer/arangodb/ArangoDBProvider.java index a44c4e422..7bdc2fa01 100644 --- a/src/sqlancer/arangodb/ArangoDBProvider.java +++ b/src/sqlancer/arangodb/ArangoDBProvider.java @@ -5,8 +5,10 @@ import com.arangodb.ArangoDB; import com.arangodb.ArangoDatabase; +import com.google.auto.service.AutoService; import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.ExecutionTimer; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; @@ -19,6 +21,7 @@ import sqlancer.common.log.LoggableFactory; import sqlancer.common.query.Query; +@AutoService(DatabaseProvider.class) public class ArangoDBProvider extends ProviderAdapter { diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 172149a52..aaebd6709 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -10,7 +10,10 @@ import java.util.List; import java.util.stream.Collectors; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -51,6 +54,7 @@ import sqlancer.postgres.gen.PostgresTruncateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; +@AutoService(DatabaseProvider.class) public class CitusProvider extends PostgresProvider { @SuppressWarnings("unchecked") diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index d936238d4..3585aa9df 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -6,7 +6,10 @@ import java.sql.Statement; import java.util.stream.Collectors; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -21,6 +24,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +@AutoService(DatabaseProvider.class) public class ClickHouseProvider extends SQLProviderAdapter { public ClickHouseProvider() { diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 2fcdf1930..abc7e6e20 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -8,6 +8,9 @@ import java.util.List; import java.util.stream.Collectors; +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; import sqlancer.MainOptions; @@ -34,6 +37,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +@AutoService(DatabaseProvider.class) public class CockroachDBProvider extends SQLProviderAdapter { public CockroachDBProvider() { diff --git a/src/sqlancer/cosmos/CosmosProvider.java b/src/sqlancer/cosmos/CosmosProvider.java index a8681d3af..424625b4f 100644 --- a/src/sqlancer/cosmos/CosmosProvider.java +++ b/src/sqlancer/cosmos/CosmosProvider.java @@ -1,11 +1,13 @@ package sqlancer.cosmos; +import com.google.auto.service.AutoService; import com.mongodb.ConnectionString; import com.mongodb.MongoClientSettings; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Randomly; @@ -17,6 +19,7 @@ import sqlancer.mongodb.MongoDBQueryAdapter; import sqlancer.mongodb.gen.MongoDBTableGenerator; +@AutoService(DatabaseProvider.class) public class CosmosProvider extends ProviderAdapter { diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 73adb3cb0..8453bb6f7 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -3,7 +3,10 @@ import java.sql.DriverManager; import java.sql.SQLException; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -22,6 +25,7 @@ import sqlancer.duckdb.gen.DuckDBUpdateGenerator; import sqlancer.duckdb.gen.DuckDBViewGenerator; +@AutoService(DatabaseProvider.class) public class DuckDBProvider extends SQLProviderAdapter { public DuckDBProvider() { diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 916849199..7f01e6196 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -4,7 +4,10 @@ import java.sql.DriverManager; import java.sql.SQLException; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -15,6 +18,7 @@ import sqlancer.common.query.SQLQueryProvider; import sqlancer.h2.H2Provider.H2GlobalState; +@AutoService(DatabaseProvider.class) public class H2Provider extends SQLProviderAdapter { public H2Provider() { diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index dc0420fca..c84691203 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -7,6 +7,9 @@ import java.util.ArrayList; import java.util.List; +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -24,6 +27,7 @@ import sqlancer.mariadb.gen.MariaDBTruncateGenerator; import sqlancer.mariadb.gen.MariaDBUpdateGenerator; +@AutoService(DatabaseProvider.class) public class MariaDBProvider extends SQLProviderAdapter { public static final int MAX_EXPRESSION_DEPTH = 3; diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java index 3b090dade..09662a863 100644 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ b/src/sqlancer/mongodb/MongoDBProvider.java @@ -3,11 +3,13 @@ import java.util.ArrayList; import java.util.List; +import com.google.auto.service.AutoService; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.ExecutionTimer; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; @@ -21,6 +23,7 @@ import sqlancer.mongodb.gen.MongoDBInsertGenerator; import sqlancer.mongodb.gen.MongoDBTableGenerator; +@AutoService(DatabaseProvider.class) public class MongoDBProvider extends ProviderAdapter { diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 447325951..0f1704bf3 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -5,7 +5,10 @@ import java.sql.SQLException; import java.sql.Statement; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -31,6 +34,7 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLOptimize; import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; +@AutoService(DatabaseProvider.class) public class MySQLProvider extends SQLProviderAdapter { public MySQLProvider() { diff --git a/src/sqlancer/oceanbase/OceanBaseProvider.java b/src/sqlancer/oceanbase/OceanBaseProvider.java index d7308ab63..b7c5794f1 100644 --- a/src/sqlancer/oceanbase/OceanBaseProvider.java +++ b/src/sqlancer/oceanbase/OceanBaseProvider.java @@ -5,7 +5,10 @@ import java.sql.SQLException; import java.sql.Statement; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -24,6 +27,7 @@ import sqlancer.oceanbase.gen.OceanBaseUpdateGenerator; import sqlancer.oceanbase.gen.datadef.OceanBaseIndexGenerator; +@AutoService(DatabaseProvider.class) public class OceanBaseProvider extends SQLProviderAdapter { public OceanBaseProvider() { diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index e0503f2c7..0aeb29328 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -8,7 +8,10 @@ import java.sql.Statement; import java.util.Arrays; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -43,6 +46,7 @@ // EXISTS // IN +@AutoService(DatabaseProvider.class) public class PostgresProvider extends SQLProviderAdapter { /** diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 54f1a6db4..eb88a0e0a 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -7,7 +7,10 @@ import java.util.Arrays; import java.util.List; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -40,6 +43,7 @@ import sqlancer.sqlite3.gen.dml.SQLite3UpdateGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; +@AutoService(DatabaseProvider.class) public class SQLite3Provider extends SQLProviderAdapter { public static boolean allowFloatingPointFp = true; diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index c744de3f2..f0d8f261b 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -5,7 +5,10 @@ import java.sql.SQLException; import java.sql.Statement; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -28,6 +31,7 @@ import sqlancer.tidb.gen.TiDBUpdateGenerator; import sqlancer.tidb.gen.TiDBViewGenerator; +@AutoService(DatabaseProvider.class) public class TiDBProvider extends SQLProviderAdapter { public TiDBProvider() { From a1aa0ff994e98a64ab6cb746561fceda36168c30 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:38:51 +0100 Subject: [PATCH 0506/1284] Update the Checkstyle version --- pom.xml | 7 +++++++ src/sqlancer/mysql/ast/MySQLConstant.java | 4 ++-- src/sqlancer/oceanbase/ast/OceanBaseConstant.java | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index bf7576302..97e053869 100644 --- a/pom.xml +++ b/pom.xml @@ -203,6 +203,13 @@ org.apache.maven.plugins maven-checkstyle-plugin 3.1.2 + + + com.puppycrawl.tools + checkstyle + 9.2.1 + + configs/checkstyle.xml diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index 11a807841..2e4922f8e 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -164,12 +164,12 @@ public MySQLConstant castAs(CastType type) { try { String substring = value.substring(0, i); long val = Long.parseLong(substring); - return MySQLConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); + return MySQLConstant.createIntConstant(val, type == CastType.SIGNED); } catch (NumberFormatException e) { // ignore } } - return MySQLConstant.createIntConstant(0, type == CastType.SIGNED ? true : false); + return MySQLConstant.createIntConstant(0, type == CastType.SIGNED); } else { throw new AssertionError(); } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java index 779d725b1..91a481fc6 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -282,11 +282,11 @@ public OceanBaseConstant castAs(CastType type) { try { String substring = value.substring(0, i); long val = Long.parseLong(substring); - return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); + return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED); } catch (NumberFormatException e) { } } - return OceanBaseConstant.createIntConstant(0, type == CastType.SIGNED ? true : false); + return OceanBaseConstant.createIntConstant(0, type == CastType.SIGNED); } else { throw new AssertionError(); } From f5e2f2ccbcf2fd76599cfe84dc263ef974523a82 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:54:53 +0100 Subject: [PATCH 0507/1284] Remove the obsolete Codecov badge --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7008c1d92..dd2e28ee9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ [![Build Status](https://github.com/sqlancer/sqlancer/workflows/ci/badge.svg)](https://github.com/sqlancer/sqlancer/actions) -[![codecov](https://codecov.io/gh/sqlancer/sqlancer/branch/master/graph/badge.svg)](https://codecov.io/gh/sqlancer/sqlancer) [![Twitter](https://img.shields.io/twitter/follow/sqlancer_dbms?style=social)](https://twitter.com/sqlancer_dbms) # SQLancer From 8ab8f994727afc844dfc90ff80a18ae41842a908 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:56:42 +0100 Subject: [PATCH 0508/1284] Update the requirements --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dd2e28ee9..9ec390fd4 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ SQLancer operates in the following two phases: # Getting Started Requirements: -* Java 8 or above +* Java 11 or above * [Maven](https://maven.apache.org/) (`sudo apt install maven` on Ubuntu) -* The DBMS that you want to test (SQLite is an embedded DBMS and is included) +* The DBMS that you want to test (embedded DBMSs such as DuckDB, H2, and SQLite do not require a setup) The following commands clone SQLancer, create a JAR, and start SQLancer to test SQLite using Non-optimizing Reference Engine Construction (NoREC): From 0dda6dfc023d41ed3e763f02c24b7f197864c6ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:57:13 +0100 Subject: [PATCH 0509/1284] Update the note about being a research prototype --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 9ec390fd4..2209e7589 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,6 @@ If the execution prints progress information every five seconds, then the tool w If you launch SQLancer without parameters, available options and commands are displayed. Note that general options that are supported by all DBMS-testing implementations (e.g., `--num-threads`) need to precede the name of DBMS to be tested (e.g., `sqlite3`). Options that are supported only for specific DBMS (e.g., `--test-rtree` for SQLite3), or options for which each testing implementation provides different values (e.g. `--oracle NoREC`) need to go after the DBMS name. -# Research Prototype - -This project should at this stage still be seen as a research prototype. We believe that the tool is not ready to be used. However, we have received many requests by companies, organizations, and individual developers, which is why we decided to prematurely release the tool. Expect errors, incompatibilities, lack of documentation, and insufficient code quality. That being said, we are working hard to address these issues and enhance SQLancer to become a production-quality piece of software. We welcome any issue reports, extension requests, and code contributions. - # Testing Approaches | Approach | Description | From cf24a9f3dd1584d15a5a3f526d73489425a6ba8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Jan 2022 09:09:54 +0000 Subject: [PATCH 0510/1284] Bump mariadb-java-client from 2.7.4 to 2.7.5 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.7.4 to 2.7.5. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/2.7.5/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/2.7.4...2.7.5) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97e053869..f6db87db6 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 2.7.4 + 2.7.5 org.duckdb From 9ffe1d05baed8355650bd3f958b5815f420bcfde Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Jan 2022 12:15:38 +0100 Subject: [PATCH 0511/1284] Ignore expected Citus error "columnar_parallelscan_estimate not implemented" Closes https://github.com/sqlancer/sqlancer/issues/402. --- src/sqlancer/citus/gen/CitusTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java index c340400f3..86e5d40cf 100644 --- a/src/sqlancer/citus/gen/CitusTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -11,6 +11,8 @@ public CitusTableGenerator(String tableName, PostgresSchema newSchema, boolean g PostgresGlobalState globalState) { super(tableName, newSchema, generateOnlyKnown, globalState); CitusCommon.addCitusErrors(errors); + errors.add("columnar_parallelscan_estimate not implemented"); // see + // https://github.com/sqlancer/sqlancer/issues/402 } public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, From ecdb34a2548238ce2b46551650c11fee76f52071 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jan 2022 09:10:58 +0000 Subject: [PATCH 0512/1284] Bump slf4j-simple from 1.7.33 to 1.7.34 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.33 to 1.7.34. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/commits) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f6db87db6..5be2ef0c8 100644 --- a/pom.xml +++ b/pom.xml @@ -313,7 +313,7 @@ org.slf4j slf4j-simple - 1.7.33 + 1.7.34 ru.yandex.clickhouse From 2b08717ed3703b977cd1245cd850126ede02c800 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 09:09:23 +0000 Subject: [PATCH 0513/1284] Bump mariadb-java-client from 2.7.5 to 3.0.3 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.7.5 to 3.0.3. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/2.7.5...3.0.3) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5be2ef0c8..ae96a248b 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 2.7.5 + 3.0.3 org.duckdb From 129d5cc5239a9d2ffd4ed201f12ceeefc6cb8d27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 10:20:35 +0000 Subject: [PATCH 0514/1284] Bump slf4j-simple from 1.7.34 to 1.7.35 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.34 to 1.7.35. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.34...v_1.7.35) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ae96a248b..4e9888c2a 100644 --- a/pom.xml +++ b/pom.xml @@ -313,7 +313,7 @@ org.slf4j slf4j-simple - 1.7.34 + 1.7.35 ru.yandex.clickhouse From 140f990af3608b9fca59ea72964c63b84e890c66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 09:12:26 +0000 Subject: [PATCH 0515/1284] Bump checkstyle from 9.2.1 to 9.3 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 9.2.1 to 9.3. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-9.2.1...checkstyle-9.3) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4e9888c2a..67227fd21 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 9.2.1 + 9.3 From 0474249582677b7492d6f5f7fbeceed0c61fab0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 09:13:09 +0000 Subject: [PATCH 0516/1284] Bump plexus-compiler-eclipse from 2.9.0 to 2.10.0 Bumps plexus-compiler-eclipse from 2.9.0 to 2.10.0. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 67227fd21..e2a1b332f 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.9.0 + 2.10.0 org.eclipse.jdt From 73a8dc0d8bb37d0acb40817b4d9ef59ad4cff052 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 10:45:26 +0000 Subject: [PATCH 0517/1284] Bump plexus-compiler-api from 2.9.0 to 2.10.0 Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.9.0 to 2.10.0. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.9.0...plexus-compiler-2.10.0) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e2a1b332f..88a3cb2a5 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.codehaus.plexus plexus-compiler-api - 2.9.0 + 2.10.0 From 12e8c0655c968c38966f059e085c5d4a0c4551d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 10:59:12 +0000 Subject: [PATCH 0518/1284] Bump slf4j-simple from 1.7.35 to 1.7.36 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.35 to 1.7.36. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.35...v_1.7.36) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 88a3cb2a5..538e543b1 100644 --- a/pom.xml +++ b/pom.xml @@ -313,7 +313,7 @@ org.slf4j slf4j-simple - 1.7.35 + 1.7.36 ru.yandex.clickhouse From 5b27bb91dfcc184bcc66fbed0510941684cdc329 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 11:24:26 +0000 Subject: [PATCH 0519/1284] Bump postgresql from 42.3.1 to 42.3.2 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.1 to 42.3.2. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.1...REL42.3.2) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 538e543b1..d90ed9edd 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.1 + 42.3.2 org.xerial From efb390487cc9a8c6a9dfbc3f0e00c08d88339397 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 09:12:07 +0000 Subject: [PATCH 0520/1284] Bump maven-javadoc-plugin from 3.3.1 to 3.3.2 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.3.1 to 3.3.2. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.3.1...maven-javadoc-plugin-3.3.2) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d90ed9edd..6c24bbe87 100644 --- a/pom.xml +++ b/pom.xml @@ -397,7 +397,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.3.1 + 3.3.2 8 From 8f1a9ba0a29228a817d35cd92fa571fd76dbfa33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 10:50:32 +0000 Subject: [PATCH 0521/1284] Bump maven-compiler-plugin from 3.9.0 to 3.10.0 Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.9.0...maven-compiler-plugin-3.10.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6c24bbe87..ea5e9b09d 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ maven-compiler-plugin - 3.9.0 + 3.10.0 11 11 From 6390f6fc0f371a233a918702f07afb9e024afeb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Feb 2022 01:20:38 +0000 Subject: [PATCH 0522/1284] Bump postgresql from 42.3.2 to 42.3.3 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.2 to 42.3.3. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.2...REL42.3.3) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea5e9b09d..ff08a32cc 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.2 + 42.3.3 org.xerial From de46775c6d134dd18b800c06f6979bf5b77e1e62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 09:13:30 +0000 Subject: [PATCH 0523/1284] Bump checkstyle from 9.3 to 10.0 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 9.3 to 10.0. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-9.3...checkstyle-10.0) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ff08a32cc..bf5c9131b 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 9.3 + 10.0 From 961a62f081a9ca4f71a1e56486a11459b3bb7ee1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Mar 2022 09:13:02 +0000 Subject: [PATCH 0524/1284] Bump formatter-maven-plugin from 2.17.1 to 2.18.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.17.1 to 2.18.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.17.1...formatter-maven-plugin-2.18.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bf5c9131b..0da27bc94 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.17.1 + 2.18.0 eclipseformat From eaab8836e59517decda30a14d0748827c41b9387 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 09:12:42 +0000 Subject: [PATCH 0525/1284] Bump plexus-compiler-eclipse from 2.10.0 to 2.11.1 Bumps plexus-compiler-eclipse from 2.10.0 to 2.11.1. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0da27bc94..70edee8c6 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.10.0 + 2.11.1 org.eclipse.jdt From a24fc38fa3b224c64a363a43b0c9011c5017c96d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 19:21:13 +0000 Subject: [PATCH 0526/1284] Bump plexus-compiler-api from 2.10.0 to 2.11.1 Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.10.0 to 2.11.1. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.10.0...plexus-compiler-2.11.1) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 70edee8c6..edfe8edf8 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.codehaus.plexus plexus-compiler-api - 2.10.0 + 2.11.1 From 2b65eff97eed1a6ac21e18ab74093fc46f5ab6de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Mar 2022 09:11:08 +0000 Subject: [PATCH 0527/1284] Bump maven-compiler-plugin from 3.10.0 to 3.10.1 Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.10.0 to 3.10.1. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.10.0...maven-compiler-plugin-3.10.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index edfe8edf8..73cfcf926 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ maven-compiler-plugin - 3.10.0 + 3.10.1 11 11 From 60d563cb745e200fce876821765ea4712b2231a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 09:11:43 +0000 Subject: [PATCH 0528/1284] Bump maven-jxr-plugin from 3.1.1 to 3.2.0 Bumps [maven-jxr-plugin](https://github.com/apache/maven-jxr) from 3.1.1 to 3.2.0. - [Release notes](https://github.com/apache/maven-jxr/releases) - [Commits](https://github.com/apache/maven-jxr/compare/jxr-3.1.1...jxr-3.2.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jxr-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 73cfcf926..207975280 100644 --- a/pom.xml +++ b/pom.xml @@ -341,7 +341,7 @@ org.apache.maven.plugins maven-jxr-plugin - 3.1.1 + 3.2.0 From b6522767fe8bcfa77333cdbd73a36db76f1986f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Mar 2022 09:14:54 +0000 Subject: [PATCH 0529/1284] Bump spotbugs-maven-plugin from 4.5.3.0 to 4.6.0.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.5.3.0 to 4.6.0.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.5.3.0...spotbugs-maven-plugin-4.6.0.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 207975280..a43bb22c2 100644 --- a/pom.xml +++ b/pom.xml @@ -250,7 +250,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.5.3.0 + 4.6.0.0 spotbugs From 0098bccb5baaac9854e3dc96763743ed555d2ce3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Mar 2022 09:17:57 +0000 Subject: [PATCH 0530/1284] Bump checkstyle from 10.0 to 10.1 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.0 to 10.1. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.0...checkstyle-10.1) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a43bb22c2..aad66fd4a 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.0 + 10.1 From dcabc89e2fa1dd9619e07a2650cd3d2489268add Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Mon, 28 Mar 2022 12:16:29 +0800 Subject: [PATCH 0531/1284] Fix incorrect exit code and possible data race of shutdownThreads * if --max-generated-database is set to positive number and run without error, the exit code should be 0 instead of the one set in MainOptions * shutdownThreads should be atomic due to multi-thread access to it --- src/sqlancer/Main.java | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 77ecdcabe..a4404a6fa 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -17,6 +17,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import com.beust.jcommander.JCommander; @@ -33,7 +34,7 @@ public final class Main { public static volatile AtomicLong nrDatabases = new AtomicLong(); public static volatile AtomicLong nrSuccessfulActions = new AtomicLong(); public static volatile AtomicLong nrUnsuccessfulActions = new AtomicLong(); - static int threadsShutdown; + public static volatile AtomicLong threadsShutdown = new AtomicLong(); static boolean progressMonitorStarted; static { @@ -446,6 +447,7 @@ private String formatInteger(long intValue) { return options.getErrorExitCode(); } } + final AtomicBoolean someOneFails = new AtomicBoolean(false); for (int i = 0; i < options.getTotalNumberTries(); i++) { final String databaseName = options.getDatabasePrefix() + i; @@ -466,23 +468,18 @@ public void run() { private void runThread(final String databaseName) { Randomly r = new Randomly(seed); try { - if (options.getMaxGeneratedDatabases() == -1) { - // run without a limit - boolean continueRunning = true; - while (continueRunning) { - continueRunning = run(options, execService, executorFactory, r, databaseName); - } - } else { - for (int i = 0; i < options.getMaxGeneratedDatabases(); i++) { - boolean continueRunning = run(options, execService, executorFactory, r, databaseName); - if (!continueRunning) { - break; - } + int maxNrDbs = options.getMaxGeneratedDatabases(); + // run without a limit if maxNrDbs == -1 + for (int i = 0; i < maxNrDbs || maxNrDbs == -1; i++) { + Boolean continueRunning = run(options, execService, executorFactory, r, databaseName); + if (!continueRunning) { + someOneFails.set(true); + break; } } } finally { - threadsShutdown++; - if (threadsShutdown == options.getTotalNumberTries()) { + threadsShutdown.addAndGet(1); + if (threadsShutdown.get() == options.getTotalNumberTries()) { execService.shutdown(); } } @@ -527,7 +524,7 @@ private boolean run(MainOptions options, ExecutorService execService, e.printStackTrace(); } - return threadsShutdown == 0 ? 0 : options.getErrorExitCode(); + return someOneFails.get() ? options.getErrorExitCode() : 0; } /** @@ -585,7 +582,7 @@ public void run() { System.out.println(String.format( "[%s] Executed %d queries (%d queries/s; %.2f/s dbs, successful statements: %2d%%). Threads shut down: %d.", dateFormat.format(date), currentNrQueries, (int) throughput, throughputDbs, - successfulStatementsRatio, threadsShutdown)); + successfulStatementsRatio, threadsShutdown.get())); timeMillis = System.currentTimeMillis(); lastNrQueries = currentNrQueries; lastNrDbs = currentNrDbs; From ae302371f6b60c1f016b26276044c5b95f8cd44a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Apr 2022 09:14:53 +0000 Subject: [PATCH 0532/1284] Bump jacoco-maven-plugin from 0.8.7 to 0.8.8 Bumps [jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.7 to 0.8.8. - [Release notes](https://github.com/jacoco/jacoco/releases) - [Commits](https://github.com/jacoco/jacoco/compare/v0.8.7...v0.8.8) --- updated-dependencies: - dependency-name: org.jacoco:jacoco-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aad66fd4a..e03babb32 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ org.jacoco jacoco-maven-plugin - 0.8.7 + 0.8.8 From 199903c72f6f47ff1f60236964a39df421822c42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 09:19:58 +0000 Subject: [PATCH 0533/1284] Bump maven-javadoc-plugin from 3.3.2 to 3.4.0 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.3.2 to 3.4.0. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.3.2...maven-javadoc-plugin-3.4.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e03babb32..fd3cf7fa6 100644 --- a/pom.xml +++ b/pom.xml @@ -397,7 +397,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.3.2 + 3.4.0 8 From 9dccd866a0a12ce300e9c1fc7cf19bd751172a47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:02:12 +0000 Subject: [PATCH 0534/1284] Bump postgresql from 42.3.3 to 42.3.4 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.3 to 42.3.4. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.3...REL42.3.4) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd3cf7fa6..d7d42e2d9 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.3 + 42.3.4 org.xerial From 6f3be054db571d4b98b67bbf86ae8b5b6837b528 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:16:17 +0000 Subject: [PATCH 0535/1284] Bump h2 from 2.1.210 to 2.1.212 Bumps [h2](https://github.com/h2database/h2database) from 2.1.210 to 2.1.212. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.210...version-2.1.212) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d7d42e2d9..37c6fdfa5 100644 --- a/pom.xml +++ b/pom.xml @@ -323,7 +323,7 @@ com.h2database h2 - 2.1.210 + 2.1.212 org.mongodb From 9b22e488b7075da6d7e4e71c48aa80e2870f4e76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 11:55:19 +0000 Subject: [PATCH 0536/1284] Bump mariadb-java-client from 3.0.3 to 3.0.4 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.3 to 3.0.4. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.3...3.0.4) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 37c6fdfa5..e572c3f32 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.3 + 3.0.4 org.duckdb From 25eea828a412361436df7abd2db47645d9d1c47b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 12:18:32 +0000 Subject: [PATCH 0537/1284] Bump maven-shade-plugin from 3.2.4 to 3.3.0 Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.2.4 to 3.3.0. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.2.4...maven-shade-plugin-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-shade-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e572c3f32..1ae3092eb 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.4 + 3.3.0 package From 6b1e251f0d9cb0c6319a1e51809c90b44c0b1f23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Apr 2022 09:12:55 +0000 Subject: [PATCH 0538/1284] Bump maven-antrun-plugin from 3.0.0 to 3.1.0 Bumps [maven-antrun-plugin](https://github.com/apache/maven-antrun-plugin) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/apache/maven-antrun-plugin/releases) - [Commits](https://github.com/apache/maven-antrun-plugin/compare/maven-antrun-plugin-3.0.0...maven-antrun-plugin-3.1.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-antrun-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ae3092eb..df9da8077 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ org.apache.maven.plugins maven-antrun-plugin - 3.0.0 + 3.1.0 package From 36705035fcbe35e41a2bf3b4bb5eebca66120a9e Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Fri, 22 Apr 2022 21:27:08 +0800 Subject: [PATCH 0539/1284] Add overloaded assumeResultSetsAreEqual which accepts canonicalization function --- src/sqlancer/ComparatorHelper.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index f05fe5ee3..145342ac4 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -6,6 +6,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; import sqlancer.common.query.ExpectedErrors; @@ -114,6 +115,18 @@ public static void assumeResultSetsAreEqual(List resultSet, List } } + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, + String originalQueryString, List combinedString, SQLGlobalState state, + UnaryOperator canonicalizationRule) { + // Overloaded version of assumeResultSetsAreEqual that takes a canonicalization function which is applied to + // both result sets before their comparison. + List canonicalizedResultSet = resultSet.stream().map(canonicalizationRule).collect(Collectors.toList()); + List canonicalizedSecondResultSet = secondResultSet.stream().map(canonicalizationRule) + .collect(Collectors.toList()); + assumeResultSetsAreEqual(canonicalizedResultSet, canonicalizedSecondResultSet, originalQueryString, + combinedString, state); + } + public static List getCombinedResultSet(String firstQueryString, String secondQueryString, String thirdQueryString, List combinedString, boolean asUnion, SQLGlobalState state, ExpectedErrors errors) throws SQLException { From d03e1bde90741de0b522cdfeb6cc4400cd3e2a76 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Fri, 22 Apr 2022 22:32:40 +0800 Subject: [PATCH 0540/1284] Add tests for assumeResultSetsAreEqual --- test/sqlancer/TestComparatorHelper.java | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 test/sqlancer/TestComparatorHelper.java diff --git a/test/sqlancer/TestComparatorHelper.java b/test/sqlancer/TestComparatorHelper.java new file mode 100644 index 000000000..6815b9258 --- /dev/null +++ b/test/sqlancer/TestComparatorHelper.java @@ -0,0 +1,54 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; + +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.Test; + +public class TestComparatorHelper { + // TODO: Implement tests for the other ComparatorHelper methods + + @Test + public void testAssumeResultSetsAreEqualWithEqualSets() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "c"); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + + } + + @Test + public void testAssumeResultSetsAreEqualWithUnequalLengthSets() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "c", "d", "g"); + // NullPointerException is raised instead of AssertionError because state is null and the state.getState()... + // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions + // is raised + assertThrowsExactly(NullPointerException.class, () -> { + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + }); + } + + @Test + public void testAssumeResultSetsAreEqualWithUnequalValueSets() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "d"); + // NullPointerException is raised instead of AssertionError because state is null and the state.getState()... + // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions + // is raised + assertThrowsExactly(NullPointerException.class, () -> { + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + }); + } + + @Test + public void testAssumeResultSetsAreEqualWithCanonicalizationRule() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "d"); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null, (String s) -> { + return s.equals("d") ? "c" : s; + }); + } + +} From 5c5555af385b08004f033df2eedfb5504e736705 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 18 Apr 2022 15:41:17 +0800 Subject: [PATCH 0541/1284] Update duckdb implementation to that of hannes/sqlancer@persistent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hannes Mühleisen Co-authored-by: Mark Raasveldt Co-authored-by: Richard Wesley --- src/sqlancer/duckdb/DuckDBErrors.java | 28 +++++++++++----- src/sqlancer/duckdb/DuckDBProvider.java | 32 +++++++++++++++++-- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- .../duckdb/gen/DuckDBExpressionGenerator.java | 5 ++- .../duckdb/gen/DuckDBTableGenerator.java | 1 - 5 files changed, 52 insertions(+), 16 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 0cd9f52e3..cb45bb96c 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -23,7 +23,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("invalid UTF-8"); // TODO errors.add("String value is not valid UTF8"); - errors.add("Conversion: Invalid TypeId "); // TODO + errors.add("Invalid TypeId "); // TODO errors.add("GROUP BY clause cannot contain aggregates!"); // investigate @@ -33,7 +33,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Overflow in multiplication"); errors.add("Out of Range"); - errors.add("Conversion: Date out of range"); + errors.add("Date out of range"); // collate errors.add("Cannot combine types with different collation!"); @@ -42,6 +42,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { // // https://github.com/cwida/duckdb/issues/532 errors.add("Not implemented type: DATE"); errors.add("Not implemented type: TIMESTAMP"); + errors.add("Like pattern must not end with escape character!"); // LIKE errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a // table supports rowids @@ -50,7 +51,6 @@ public static void addExpressionErrors(ExpectedErrors errors) { // removed column errors.add("Contents of view were altered: types don't match!"); errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); - errors.add("Could not convert string"); // Could not convert string '1.7976931348623157E308' to DOUBLE } private static void addRegexErrors(ExpectedErrors errors) { @@ -66,32 +66,44 @@ private static void addRegexErrors(ExpectedErrors errors) { } private static void addFunctionErrors(ExpectedErrors errors) { - errors.add("SUBSTRING cannot handle negative offsets"); + errors.add("SUBSTRING cannot handle negative lengths"); errors.add("is undefined outside [-1,1]"); // ACOS etc errors.add("invalid type specifier"); // PRINTF errors.add("argument index out of range"); // PRINTF errors.add("invalid format string"); // PRINTF errors.add("number is too big"); // PRINTF + errors.add("Like pattern must not end with escape character!"); // LIKE errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part errors.add("extract specifier"); // date_part + errors.add("not recognized"); // date_part + errors.add("not supported"); // date_part + errors.add("Failed to cast"); + errors.add("Conversion Error"); + errors.add("Could not cast value"); errors.add("Insufficient padding in RPAD"); // RPAD errors.add("Could not choose a best candidate function for the function call"); // monthname + errors.add("expected a numeric precision field"); // ROUND + errors.add("with non-constant precision is not supported"); // ROUND } public static void addInsertErrors(ExpectedErrors errors) { + addRegexErrors(errors); + addFunctionErrors(errors); + errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); - errors.add("duplicate key value violates primary key or unique constraint"); + errors.add("duplicate key"); errors.add("can't be cast because the value is out of range for the destination type"); errors.add("Could not convert string"); - errors.add("timestamp field value out of range"); errors.add("Unimplemented type for cast"); - errors.add("date/time field value out of range"); + errors.add("field value out of range"); errors.add("CHECK constraint failed"); errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if // the table has a primary key - errors.add("Out of Range: Could not cast value"); + errors.add("Could not cast value"); + errors.add("create unique index, table contains duplicate data"); + errors.add("Failed to cast"); } public static void addGroupByErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 8453bb6f7..5be773885 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -2,6 +2,9 @@ import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Connection; +import java.sql.Statement; +import java.io.File; import com.google.auto.service.AutoService; @@ -118,11 +121,34 @@ public void generateDatabase(DuckDBGlobalState globalState) throws Exception { se.executeStatements(); } + public void tryDeleteFile(String fname) { + try { + File f = new File(fname); + f.delete(); + } catch(Exception e) { + } + } + + public void tryDeleteDatabase(String dbpath) { + if (dbpath.equals("") || dbpath.equals(":memory:")) { + return; + } + tryDeleteFile(dbpath); + tryDeleteFile(dbpath + ".wal"); + } + @Override public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLException { - String url = "jdbc:duckdb:"; - return new SQLConnection(DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword())); + String database_file = System.getProperty("duckdb.database.file", ""); + String url = "jdbc:duckdb:" + database_file; + tryDeleteDatabase(database_file); + + Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), + globalState.getOptions().getPassword()); + Statement stmt = conn.createStatement(); + stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); + stmt.close(); + return new SQLConnection(conn); } @Override diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 56e89686c..60395976c 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -237,7 +237,7 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master()")) { + try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master WHERE type='table' or type='view'")) { while (rs.next()) { tableNames.add(rs.getString("name")); } diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index f907e62f2..fc9ae1b2b 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -243,6 +243,7 @@ public enum DBFunction { DEGREES(1), // RADIANS(1), // MOD(2), // + XOR(2), // // string functions LENGTH(1), // LOWER(1), // @@ -379,8 +380,7 @@ public static Operator getRandom() { } public enum DuckDBBinaryArithmeticOperator implements Operator { - CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), XOR("#"), LSHIFT("<<"), - RSHIFT(">>"); + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); private String textRepr; @@ -400,7 +400,6 @@ public String getTextRepresentation() { } public enum DuckDBBinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index bb8fe2304..4d2365f07 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -58,7 +58,6 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); - DuckDBErrors.addExpressionErrors(errors); sb.append(")"); } } From 6dd5d531c40ab5c57ab76160866b5a019b5da5a6 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 18 Apr 2022 20:38:46 +0800 Subject: [PATCH 0542/1284] Bump duckdb_jdbc from 0.2.5 to 0.3.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index df9da8077..aa39f5f2b 100644 --- a/pom.xml +++ b/pom.xml @@ -302,7 +302,7 @@ org.duckdb duckdb_jdbc - 0.2.5 + 0.3.3 org.junit.jupiter From e6a2918c153cdf530832d6d4825a276fd62b0523 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 18 Apr 2022 20:39:52 +0800 Subject: [PATCH 0543/1284] Format updated implementation --- src/sqlancer/duckdb/DuckDBProvider.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 5be773885..3164b7d6a 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -1,10 +1,10 @@ package sqlancer.duckdb; +import java.io.File; +import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.sql.Connection; import java.sql.Statement; -import java.io.File; import com.google.auto.service.AutoService; @@ -125,7 +125,7 @@ public void tryDeleteFile(String fname) { try { File f = new File(fname); f.delete(); - } catch(Exception e) { + } catch (Exception e) { } } @@ -139,9 +139,9 @@ public void tryDeleteDatabase(String dbpath) { @Override public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLException { - String database_file = System.getProperty("duckdb.database.file", ""); - String url = "jdbc:duckdb:" + database_file; - tryDeleteDatabase(database_file); + String databaseFile = System.getProperty("duckdb.database.file", ""); + String url = "jdbc:duckdb:" + databaseFile; + tryDeleteDatabase(databaseFile); Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); From f43a0bebc8ae30b2da38d95d9e88f028dae3d606 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 18 Apr 2022 20:42:54 +0800 Subject: [PATCH 0544/1284] Re-enable QUERY_PARTITIONING oracle in CI for DuckDB --- test/sqlancer/dbms/TestDuckDB.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDB.java index a2f3e89be..d36d8610f 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDB.java @@ -11,8 +11,10 @@ public class TestDuckDB { @Test public void testDuckDB() { // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 - assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", "0", "duckdb", "--oracle", "QUERY_PARTITIONING" })); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "duckdb", "--oracle", + "QUERY_PARTITIONING" })); } } From 4a5724e8eb0d162987f32683f05e16c8feb3ce2c Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Sat, 23 Apr 2022 07:36:04 +0800 Subject: [PATCH 0545/1284] Add canonicalization rule for DuckDB results --- .../duckdb/test/DuckDBQueryPartitioningBase.java | 10 ++++++++++ .../test/DuckDBQueryPartitioningDistinctTester.java | 2 +- .../test/DuckDBQueryPartitioningGroupByTester.java | 2 +- .../test/DuckDBQueryPartitioningHavingTester.java | 2 +- .../test/DuckDBQueryPartitioningWhereTester.java | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index a8663b314..08c862973 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -36,6 +37,15 @@ public DuckDBQueryPartitioningBase(DuckDBGlobalState state) { DuckDBErrors.addExpressionErrors(errors); } + public static String canonicalizeResultValue(String value) { + // Rule: -0.0 should be canonicalized to 0.0 + if (Objects.equals(value, "-0.0")) { + return "0.0"; + } + + return value; + } + @Override public void check() throws SQLException { s = state.getSchema(); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java index 89b7ea602..d85d2a2d5 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java @@ -38,7 +38,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java index 7ac650f44..fab480fed 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java @@ -41,7 +41,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index e11dcefb3..fc8843729 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -46,7 +46,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java index 132c02a62..412ab6ffd 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java @@ -39,7 +39,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); } } From 6af2b84bb809b35bd64116302682a09c664f307c Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 25 Apr 2022 18:00:21 +0800 Subject: [PATCH 0546/1284] Add support for NULL data type in DuckDB --- src/sqlancer/duckdb/DuckDBSchema.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 60395976c..8577fff26 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -23,7 +23,7 @@ public class DuckDBSchema extends AbstractSchema public enum DuckDBDataType { - INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP; + INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP, NULL; public static DuckDBDataType getRandom() { return Randomly.fromOptions(values()); @@ -109,6 +109,8 @@ public String toString() { return Randomly.fromOptions("TIMESTAMP", "DATETIME"); case DATE: return Randomly.fromOptions("DATE"); + case NULL: + return Randomly.fromOptions("NULL"); default: throw new AssertionError(getPrimitiveDataType()); } @@ -197,6 +199,9 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { case "TIMESTAMP": primitiveType = DuckDBDataType.TIMESTAMP; break; + case "NULL": + primitiveType = DuckDBDataType.NULL; + break; case "INTERVAL": throw new IgnoreMeException(); // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP From 09dff0bc13a1d2b06a240497709af46e187a8c44 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 25 Apr 2022 18:02:32 +0800 Subject: [PATCH 0547/1284] Exclude NULL from DuckDB data type randomization, rename for clarity --- src/sqlancer/duckdb/DuckDBSchema.java | 12 ++++++++---- .../duckdb/gen/DuckDBAlterTableGenerator.java | 4 ++-- .../duckdb/gen/DuckDBExpressionGenerator.java | 5 +++-- src/sqlancer/duckdb/gen/DuckDBTableGenerator.java | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 8577fff26..6b94ce137 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -25,8 +25,12 @@ public enum DuckDBDataType { INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP, NULL; - public static DuckDBDataType getRandom() { - return Randomly.fromOptions(values()); + public static DuckDBDataType getRandomWithoutNull() { + DuckDBDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DuckDBDataType.NULL); + return dt; } } @@ -53,8 +57,8 @@ public int getSize() { return size; } - public static DuckDBCompositeDataType getRandom() { - DuckDBDataType type = DuckDBDataType.getRandom(); + public static DuckDBCompositeDataType getRandomWithoutNull() { + DuckDBDataType type = DuckDBDataType.getRandomWithoutNull(); int size = -1; switch (type) { case INT: diff --git a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java index 577ad566b..710fca1e2 100644 --- a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java @@ -34,13 +34,13 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { String columnName = table.getFreeColumnName(); sb.append(columnName); sb.append(" "); - sb.append(DuckDBCompositeDataType.getRandom().toString()); + sb.append(DuckDBCompositeDataType.getRandomWithoutNull().toString()); break; case ALTER_COLUMN: sb.append("ALTER COLUMN "); sb.append(table.getRandomColumn().getName()); sb.append(" SET DATA TYPE "); - sb.append(DuckDBCompositeDataType.getRandom().toString()); + sb.append(DuckDBCompositeDataType.getRandomWithoutNull().toString()); if (Randomly.getBoolean()) { sb.append(" USING "); DuckDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index fc9ae1b2b..e0270e816 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -98,7 +98,8 @@ protected Node generateExpression(int depth) { return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), DuckDBBinaryArithmeticOperator.getRandom()); case CAST: - return new DuckDBCastOperation(generateExpression(depth + 1), DuckDBCompositeDataType.getRandom()); + return new DuckDBCastOperation(generateExpression(depth + 1), + DuckDBCompositeDataType.getRandomWithoutNull()); case FUNC: DBFunction func = DBFunction.getRandom(); return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); @@ -132,7 +133,7 @@ public Node generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return DuckDBConstant.createNullConstant(); } - DuckDBDataType type = DuckDBDataType.getRandom(); + DuckDBDataType type = DuckDBDataType.getRandomWithoutNull(); switch (type) { case INT: if (!globalState.getDbmsSpecificOptions().testIntConstants) { diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index 4d2365f07..d3cd27fde 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -80,7 +80,7 @@ private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { String columnName = String.format("c%d", i); - DuckDBCompositeDataType columnType = DuckDBCompositeDataType.getRandom(); + DuckDBCompositeDataType columnType = DuckDBCompositeDataType.getRandomWithoutNull(); columns.add(new DuckDBColumn(columnName, columnType, false, false)); } return columns; From 27d178806918ffaa216d1a0498c7e7489476b243 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 10:16:25 +0000 Subject: [PATCH 0548/1284] Bump checkstyle from 10.1 to 10.2 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.1 to 10.2. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.1...checkstyle-10.2) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa39f5f2b..ceaa5cfb4 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.1 + 10.2 From abe47f30af704dad7d8f8f4bb679d10497a09bea Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Fri, 29 Apr 2022 13:07:20 +0800 Subject: [PATCH 0549/1284] fix bug in the copy constructor of SQLite3Select --- src/sqlancer/sqlite3/ast/SQLite3Select.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 340c708fb..bff7fb775 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -30,7 +30,7 @@ public SQLite3Select(SQLite3Select other) { limitClause = other.limitClause; orderByClause = new ArrayList<>(other.orderByClause); offsetClause = other.offsetClause; - fetchColumns = new ArrayList<>(fetchColumns); + fetchColumns = new ArrayList<>(other.fetchColumns); joinStatements = new ArrayList<>(); for (Join j : other.joinStatements) { joinStatements.add(new Join(j)); From 443ba9224952da4592043ed9f5b649c77d856948 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 09:12:19 +0000 Subject: [PATCH 0550/1284] Bump plexus-compiler-eclipse from 2.11.1 to 2.12.0 Bumps plexus-compiler-eclipse from 2.11.1 to 2.12.0. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ceaa5cfb4..4de9b0ad1 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.11.1 + 2.12.0 org.eclipse.jdt From ed5d897ff244358faa7be049f6a2e481d0533cc0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 11:54:01 +0000 Subject: [PATCH 0551/1284] Bump plexus-compiler-api from 2.11.1 to 2.12.0 Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.11.1 to 2.12.0. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.11.1...plexus-compiler-2.12.0) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4de9b0ad1..75247e015 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.codehaus.plexus plexus-compiler-api - 2.11.1 + 2.12.0 From fa889dedba13caac97bd76fceb133de13bb9aaca Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 10 May 2022 10:28:28 +0800 Subject: [PATCH 0552/1284] fix invalid trigger generation --- src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java index 610385501..1e5254948 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java @@ -100,11 +100,11 @@ public static SQLQueryAdapter create(SQLite3GlobalState globalState) throws SQLE sb.append(", 'asdf'"); } sb.append(")"); + sb.append(";"); break; default: throw new AssertionError(); } - sb.append(";"); } sb.append("END"); From f1800b099a5d366ae26314cbfdef7e002e3c2ee2 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: Tue, 10 May 2022 21:14:40 +0500 Subject: [PATCH 0553/1284] Add '--extensions' option. Allow to perform a sqlancer test with an relocatable extension installed. --- src/sqlancer/postgres/PostgresOptions.java | 3 +++ src/sqlancer/postgres/PostgresProvider.java | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index ed2856451..b6c79d752 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -38,6 +38,9 @@ public class PostgresOptions implements DBMSSpecificOptions { NOREC { @Override diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 0aeb29328..e9396a628 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -63,6 +63,7 @@ public class PostgresProvider extends SQLProviderAdapter Date: Tue, 17 May 2022 13:02:50 +0000 Subject: [PATCH 0554/1284] Deterministic Hex and Dec for SQLite3 --- .../sqlite3/SQLite3ToStringVisitor.java | 6 +++--- src/sqlancer/sqlite3/ast/SQLite3Constant.java | 20 +++++++++++++++++++ .../gen/SQLite3ExpressionGenerator.java | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 4d0820b25..f9f96a1d3 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -177,11 +177,11 @@ public void visit(SQLite3Constant c) { // sb.append(c.asInt() == 1 ? "TRUE" : "FALSE"); // } else { // - 0X8000000000000000 results in an error message otherwise - if (Randomly.getBoolean() || c.asInt() == Long.MIN_VALUE) { - sb.append(c.asInt()); - } else { + if (c.isHex()) { long intVal = c.asInt(); asHexString(intVal); + } else { + sb.append(c.asInt()); } // } break; diff --git a/src/sqlancer/sqlite3/ast/SQLite3Constant.java b/src/sqlancer/sqlite3/ast/SQLite3Constant.java index abe3cd033..a6edd7bf9 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Constant.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Constant.java @@ -63,9 +63,21 @@ public SQLite3Constant applyLess(SQLite3Constant right, SQLite3CollateSequence c public static class SQLite3IntConstant extends SQLite3Constant { private final long value; + private final boolean isHex; + + public SQLite3IntConstant(long value, boolean isHex) { + this.value = value; + this.isHex = isHex; + } public SQLite3IntConstant(long value) { this.value = value; + this.isHex = false; + } + + @Override + public boolean isHex() { + return isHex; } @Override @@ -511,6 +523,10 @@ public SQLite3Constant applyLess(SQLite3Constant right, SQLite3CollateSequence c public abstract Object getValue(); + public boolean isHex() { + throw new UnsupportedOperationException(this.getDataType().toString()); + } + public long asInt() { throw new UnsupportedOperationException(this.getDataType().toString()); } @@ -533,6 +549,10 @@ public static SQLite3Constant createIntConstant(long val) { return new SQLite3IntConstant(val); } + public static SQLite3Constant createIntConstant(long val, boolean isHex) { + return new SQLite3IntConstant(val, isHex); + } + public static SQLite3Constant createBinaryConstant(byte[] val) { return new SQLite3BinaryConstant(val); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index e7480dc19..d21f00af3 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -185,7 +185,7 @@ private SQLite3Expression getRandomLiteralValueInternal(Randomly r) { switch (randomLiteral) { case INTEGER: if (Randomly.getBoolean()) { - return SQLite3Constant.createIntConstant(r.getInteger()); + return SQLite3Constant.createIntConstant(r.getInteger(), Randomly.getBoolean()); } else { return SQLite3Constant.createTextConstant(String.valueOf(r.getInteger())); } From 1da51fb1542c0fba292d9ba39b50cb3bd2010ab1 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 17 May 2022 13:25:44 +0000 Subject: [PATCH 0555/1284] fix too large hex bug in SQLite3 --- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index f9f96a1d3..f81259732 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -177,11 +177,11 @@ public void visit(SQLite3Constant c) { // sb.append(c.asInt() == 1 ? "TRUE" : "FALSE"); // } else { // - 0X8000000000000000 results in an error message otherwise - if (c.isHex()) { + if (!c.isHex() || c.asInt() == Long.MIN_VALUE) { + sb.append(c.asInt()); + } else { long intVal = c.asInt(); asHexString(intVal); - } else { - sb.append(c.asInt()); } // } break; From 57724bc433be2b13a7b0f1647487ef2fc11bbcb1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 May 2022 09:13:35 +0000 Subject: [PATCH 0556/1284] Bump spotbugs-maven-plugin from 4.6.0.0 to 4.7.0.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.6.0.0 to 4.7.0.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.6.0.0...spotbugs-maven-plugin-4.7.0.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 75247e015..3f251a9a1 100644 --- a/pom.xml +++ b/pom.xml @@ -250,7 +250,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.6.0.0 + 4.7.0.0 spotbugs From 6d7490afb828a12c6fd2aef170bf8fe4eb38fb2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 May 2022 09:31:53 +0000 Subject: [PATCH 0557/1284] Bump postgresql from 42.3.4 to 42.3.5 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.4 to 42.3.5. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.4...REL42.3.5) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3f251a9a1..d6f229e24 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.4 + 42.3.5 org.xerial From ea4d2a0767233453012a2a5e49839c5d7e5fec83 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: Sat, 28 May 2022 22:17:55 +0300 Subject: [PATCH 0558/1284] Add an expected error on usage of the chr() routine. According to PG15 commit e9e63b7, chr() in next release will return an error on negative input. Be prepared to that change. --- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 771fc8651..a3e04d046 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -63,6 +63,7 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("is of type boolean but expression is of type text"); errors.add("a negative number raised to a non-integer power yields a complex result"); errors.add("could not determine polymorphic type because input has type unknown"); + errors.add("character number must be positive"); addToCharFunctionErrors(errors); addBitStringOperationErrors(errors); addFunctionErrors(errors); From b2c504675d39164dc28a604bba9380b0a70f909d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 29 May 2022 09:26:56 +0000 Subject: [PATCH 0559/1284] Bump mariadb-java-client from 3.0.4 to 3.0.5 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.4 to 3.0.5. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.4...3.0.5) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6f229e24..d1b9e3660 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.4 + 3.0.5 org.duckdb From 3386cb2ad002e0664d72abf22b8e8b64bf0ceb38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 29 May 2022 09:39:39 +0000 Subject: [PATCH 0560/1284] Bump postgresql from 42.3.5 to 42.3.6 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.5 to 42.3.6. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.5...REL42.3.6) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d1b9e3660..096779d86 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.5 + 42.3.6 org.xerial From a88dcad000ac626e03bb1f6f7f3a0cb56c31d8a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 May 2022 09:15:23 +0000 Subject: [PATCH 0561/1284] Bump checkstyle from 10.2 to 10.3 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.2 to 10.3. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.2...checkstyle-10.3) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 096779d86..01b84368a 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.2 + 10.3 From 48548200dc90d6c02824f54b4836b2b919c58473 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 May 2022 09:30:43 +0000 Subject: [PATCH 0562/1284] Bump mysql-connector-java from 8.0.28 to 8.0.29 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.28 to 8.0.29. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.28...8.0.29) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 01b84368a..409521b23 100644 --- a/pom.xml +++ b/pom.xml @@ -292,7 +292,7 @@ mysql mysql-connector-java - 8.0.28 + 8.0.29 org.mariadb.jdbc From e9472486476cf9e62f09b557d12327cdb0608762 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jun 2022 09:12:19 +0000 Subject: [PATCH 0563/1284] Bump formatter-maven-plugin from 2.18.0 to 2.19.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.18.0 to 2.19.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.18.0...formatter-maven-plugin-2.19.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 409521b23..aa42765d6 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.18.0 + 2.19.0 eclipseformat From 9ad4cf4381ec294e79849c3349e56eb2040cc97d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jun 2022 09:13:48 +0000 Subject: [PATCH 0564/1284] Bump postgresql from 42.3.6 to 42.4.0 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.6 to 42.4.0. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.6...REL42.4.0) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa42765d6..2273d6762 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.6 + 42.4.0 org.xerial From d41212d665fa4398829937d0ce59fffd03c74614 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jun 2022 09:16:58 +0000 Subject: [PATCH 0565/1284] Bump h2 from 2.1.212 to 2.1.214 Bumps [h2](https://github.com/h2database/h2database) from 2.1.212 to 2.1.214. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.212...version-2.1.214) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2273d6762..f43896a43 100644 --- a/pom.xml +++ b/pom.xml @@ -323,7 +323,7 @@ com.h2database h2 - 2.1.212 + 2.1.214 org.mongodb From fbc3fce132bfe0c567b5d3e55a4cb1c474206a06 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Thu, 16 Jun 2022 11:50:49 +0800 Subject: [PATCH 0566/1284] Swap parameters of Untyped generateExpressions to (nr, depth) --- .../arangodb/gen/ArangoDBComputedExpressionGenerator.java | 2 +- src/sqlancer/common/gen/UntypedExpressionGenerator.java | 2 +- src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java | 6 +++--- src/sqlancer/h2/H2ExpressionGenerator.java | 4 ++-- .../mongodb/gen/MongoDBComputedExpressionGenerator.java | 2 +- src/sqlancer/tidb/TiDBExpressionGenerator.java | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java index 8a3b98871..01e2e557a 100644 --- a/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java +++ b/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java @@ -65,7 +65,7 @@ protected Node generateExpression(int depth) { return generateLeafNode(); } ComputedFunction function = ComputedFunction.getRandom(); - return new NewFunctionNode<>(generateExpressions(depth + 1, function.getNrArgs()), function); + return new NewFunctionNode<>(generateExpressions(function.getNrArgs(), depth + 1), function); } @Override diff --git a/src/sqlancer/common/gen/UntypedExpressionGenerator.java b/src/sqlancer/common/gen/UntypedExpressionGenerator.java index b52989624..5a244b53b 100644 --- a/src/sqlancer/common/gen/UntypedExpressionGenerator.java +++ b/src/sqlancer/common/gen/UntypedExpressionGenerator.java @@ -42,7 +42,7 @@ public List generateExpressions(int nr) { return expressions; } - public List generateExpressions(int depth, int nr) { + public List generateExpressions(int nr, int depth) { List expressions = new ArrayList<>(); for (int i = 0; i < nr; i++) { expressions.add(generateExpression(depth)); diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index e0270e816..eeba94027 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -48,7 +48,7 @@ protected Node generateExpression(int depth) { if (allowAggregates && Randomly.getBoolean()) { DuckDBAggregateFunction aggregate = DuckDBAggregateFunction.getRandom(); allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); + return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); if (!globalState.getDbmsSpecificOptions().testCollate) { @@ -108,11 +108,11 @@ protected Node generateExpression(int depth) { generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); case IN: return new NewInOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, nr), generateExpressions(depth + 1, nr), + generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), generateExpression(depth + 1)); case LIKE_ESCAPE: return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index b60529ad7..d04251409 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -53,14 +53,14 @@ protected Node generateExpression(int depth) { H2UnaryPrefixOperator.getRandom()); case IN: return new NewInOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case BETWEEN: return new NewBetweenOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, nr), generateExpressions(depth + 1, nr), + generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), generateExpression(depth + 1)); case BINARY_ARITHMETIC: return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), diff --git a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java index fd5959ce6..347d5bc97 100644 --- a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java @@ -33,7 +33,7 @@ protected Node generateExpression(int depth) { return generateLeafNode(); } ComputedFunction func = ComputedFunction.getRandom(); - return new NewFunctionNode<>(generateExpressions(depth + 1, func.getNrArgs()), func); + return new NewFunctionNode<>(generateExpressions(func.getNrArgs(), depth + 1), func); } public MongoDBComputedExpressionGenerator(MongoDBGlobalState globalState) { diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index dc378cc42..a960363fe 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -91,7 +91,7 @@ protected TiDBExpression generateExpression(int depth) { Randomly.fromOptions("utf8mb4_bin", "latin1_bin", "binary", "ascii_bin", "utf8_bin")); case FUNCTION: TiDBFunction func = TiDBFunction.getRandom(); - return new TiDBFunctionCall(func, generateExpressions(depth, func.getNrArgs())); + return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); case BINARY_BIT: return new TiDBBinaryBitOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryBitOperator.getRandom()); @@ -117,8 +117,8 @@ protected TiDBExpression generateExpression(int depth) { throw new IgnoreMeException(); } int nr = Randomly.fromOptions(1, 2); - return new TiDBCase(generateExpression(depth + 1), generateExpressions(depth + 1, nr), - generateExpressions(depth + 1, nr), generateExpression(depth + 1)); + return new TiDBCase(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); default: throw new AssertionError(); } From fc900a6ae389ff57f9a74c67e5a7381447d91956 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 09:11:20 +0000 Subject: [PATCH 0567/1284] Bump duckdb_jdbc from 0.3.3 to 0.4.0 Bumps [duckdb_jdbc](https://github.com/duckdb/duckdb) from 0.3.3 to 0.4.0. - [Release notes](https://github.com/duckdb/duckdb/releases) - [Commits](https://github.com/duckdb/duckdb/compare/v0.3.3...v0.4.0) --- updated-dependencies: - dependency-name: org.duckdb:duckdb_jdbc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f43896a43..e09cc0954 100644 --- a/pom.xml +++ b/pom.xml @@ -302,7 +302,7 @@ org.duckdb duckdb_jdbc - 0.3.3 + 0.4.0 org.junit.jupiter From 718c93307441c2e08b55e26226de59ab03c2b48d Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 21 Jun 2022 22:08:52 +0800 Subject: [PATCH 0568/1284] Fix TiDB connection bug --- src/sqlancer/tidb/TiDBProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index f0d8f261b..af41fb143 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -162,8 +162,8 @@ public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLExcep s.execute(createDatabaseCommand); } con.close(); - con = DriverManager.getConnection("jdbc:mysql://127.0.0.1:4000/" + databaseName, - globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); + con = DriverManager.getConnection(url + databaseName, globalState.getOptions().getUserName(), + globalState.getOptions().getPassword()); return new SQLConnection(con); } From 5636a1e560227362af5d674459d23ab58437e407 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Tue, 28 Jun 2022 11:37:11 +0800 Subject: [PATCH 0569/1284] from duckdb copy and change to databend --- pom.xml | 2 + src/sqlancer/databend/DatabendErrors.java | 114 +++++ src/sqlancer/databend/DatabendOptions.java | 151 ++++++ src/sqlancer/databend/DatabendProvider.java | 154 +++++++ src/sqlancer/databend/DatabendSchema.java | 277 +++++++++++ .../databend/DatabendToStringVisitor.java | 91 ++++ .../databend/ast/DatabendConstant.java | 192 ++++++++ .../databend/ast/DatabendExpression.java | 5 + src/sqlancer/databend/ast/DatabendJoin.java | 124 +++++ src/sqlancer/databend/ast/DatabendSelect.java | 18 + .../gen/DatabendAlterTableGenerator.java | 70 +++ .../databend/gen/DatabendDeleteGenerator.java | 30 ++ .../gen/DatabendExpressionGenerator.java | 436 ++++++++++++++++++ .../databend/gen/DatabendIndexGenerator.java | 59 +++ .../databend/gen/DatabendInsertGenerator.java | 53 +++ .../gen/DatabendRandomQuerySynthesizer.java | 70 +++ .../databend/gen/DatabendTableGenerator.java | 89 ++++ .../databend/gen/DatabendUpdateGenerator.java | 48 ++ .../databend/gen/DatabendViewGenerator.java | 37 ++ .../databend/test/DatabendNoRECOracle.java | 133 ++++++ ...abendQueryPartitioningAggregateTester.java | 188 ++++++++ .../test/DatabendQueryPartitioningBase.java | 82 ++++ ...tabendQueryPartitioningDistinctTester.java | 44 ++ ...atabendQueryPartitioningGroupByTester.java | 53 +++ ...DatabendQueryPartitioningHavingTester.java | 62 +++ .../DatabendQueryPartitioningWhereTester.java | 45 ++ test/sqlancer/dbms/TestDatabend.java | 18 + 27 files changed, 2645 insertions(+) create mode 100644 src/sqlancer/databend/DatabendErrors.java create mode 100644 src/sqlancer/databend/DatabendOptions.java create mode 100644 src/sqlancer/databend/DatabendProvider.java create mode 100644 src/sqlancer/databend/DatabendSchema.java create mode 100644 src/sqlancer/databend/DatabendToStringVisitor.java create mode 100644 src/sqlancer/databend/ast/DatabendConstant.java create mode 100644 src/sqlancer/databend/ast/DatabendExpression.java create mode 100644 src/sqlancer/databend/ast/DatabendJoin.java create mode 100644 src/sqlancer/databend/ast/DatabendSelect.java create mode 100644 src/sqlancer/databend/gen/DatabendAlterTableGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendDeleteGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendExpressionGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendIndexGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendInsertGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java create mode 100644 src/sqlancer/databend/gen/DatabendTableGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendUpdateGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendViewGenerator.java create mode 100644 src/sqlancer/databend/test/DatabendNoRECOracle.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningBase.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java create mode 100644 test/sqlancer/dbms/TestDatabend.java diff --git a/pom.xml b/pom.xml index e09cc0954..0cdbf30b4 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,7 @@ + org.apache.maven.plugins maven-compiler-plugin 3.10.1 @@ -153,6 +154,7 @@ org.apache.maven.plugins maven-dependency-plugin + 3.2.0 copy-dependencies diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java new file mode 100644 index 000000000..92c7b17f7 --- /dev/null +++ b/src/sqlancer/databend/DatabendErrors.java @@ -0,0 +1,114 @@ +package sqlancer.databend; + +import sqlancer.common.query.ExpectedErrors; + +public final class DatabendErrors { + + private DatabendErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.add("with non-constant precision is not supported"); + errors.add("Like pattern must not end with escape character"); + errors.add("Could not convert string"); + errors.add("ORDER term out of range - should be between "); + errors.add("You might need to add explicit type casts."); + errors.add("can't be cast because the value is out of range for the destination type"); + errors.add("field value out of range"); + errors.add("Unimplemented type for cast"); + + errors.add("Escape string must be empty or one character."); + errors.add("Type mismatch when combining rows"); // BETWEEN + + errors.add("invalid UTF-8"); // TODO + errors.add("String value is not valid UTF8"); + + errors.add("Invalid TypeId "); // TODO + + errors.add("GROUP BY clause cannot contain aggregates!"); // investigate + + addRegexErrors(errors); + + addFunctionErrors(errors); + + errors.add("Overflow in multiplication"); + errors.add("Out of Range"); + errors.add("Date out of range"); + + // collate + errors.add("Cannot combine types with different collation!"); + errors.add("collations are only supported for type varchar"); + + // // https://github.com/cwida/Databend/issues/532 + errors.add("Not implemented type: DATE"); + errors.add("Not implemented type: TIMESTAMP"); + errors.add("Like pattern must not end with escape character!"); // LIKE + + errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a + // table supports rowids + + errors.add("does not have a column named"); // TODO: this only happens for views whose underlying table has a + // removed column + errors.add("Contents of view were altered: types don't match!"); + errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + } + + private static void addRegexErrors(ExpectedErrors errors) { + errors.add("missing ]"); + errors.add("missing )"); + errors.add("invalid escape sequence"); + errors.add("no argument for repetition operator: "); + errors.add("bad repetition operator"); + errors.add("trailing \\"); + errors.add("invalid perl operator"); + errors.add("invalid character class range"); + errors.add("width is not integer"); + } + + private static void addFunctionErrors(ExpectedErrors errors) { + errors.add("SUBSTRING cannot handle negative lengths"); + errors.add("is undefined outside [-1,1]"); // ACOS etc + errors.add("invalid type specifier"); // PRINTF + errors.add("argument index out of range"); // PRINTF + errors.add("invalid format string"); // PRINTF + errors.add("number is too big"); // PRINTF + errors.add("Like pattern must not end with escape character!"); // LIKE + errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part + errors.add("extract specifier"); // date_part + errors.add("not recognized"); // date_part + errors.add("not supported"); // date_part + errors.add("Failed to cast"); + errors.add("Conversion Error"); + errors.add("Could not cast value"); + errors.add("Insufficient padding in RPAD"); // RPAD + errors.add("Could not choose a best candidate function for the function call"); // monthname + errors.add("expected a numeric precision field"); // ROUND + errors.add("with non-constant precision is not supported"); // ROUND + } + + public static void addInsertErrors(ExpectedErrors errors) { + addRegexErrors(errors); + addFunctionErrors(errors); + + errors.add("NOT NULL constraint failed"); + errors.add("PRIMARY KEY or UNIQUE constraint violated"); + errors.add("duplicate key"); + errors.add("can't be cast because the value is out of range for the destination type"); + errors.add("Could not convert string"); + errors.add("Unimplemented type for cast"); + errors.add("field value out of range"); + errors.add("CHECK constraint failed"); + errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid + errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if + // the table has a primary key + errors.add("Could not cast value"); + errors.add("create unique index, table contains duplicate data"); + errors.add("Failed to cast"); + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY term out of range"); + } + +} diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java new file mode 100644 index 000000000..1e70bcb66 --- /dev/null +++ b/src/sqlancer/databend/DatabendOptions.java @@ -0,0 +1,151 @@ +package sqlancer.databend; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.databend.test.*; +import sqlancer.databend.DatabendOptions.DatabendOracleFactory; +import sqlancer.databend.test.DatabendQueryPartitioningAggregateTester; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@Parameters(commandDescription = "Databend") +public class DatabendOptions implements DBMSSpecificOptions { + + @Parameter(names = "--test-collate", arity = 1) + public boolean testCollate = true; + + @Parameter(names = "--test-check", description = "Allow generating CHECK constraints in tables", arity = 1) + public boolean testCheckConstraints = true; + + @Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1) + public boolean testDefaultValues = true; + + @Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1) + public boolean testNotNullConstraints = true; + + @Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1) + public boolean testFunctions = true; + + @Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1) + public boolean testCasts = true; + + @Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions", arity = 1) + public boolean testBetween = true; + + @Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1) + public boolean testIn = true; + + @Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1) + public boolean testCase = true; + + @Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1) + public boolean testBinaryLogicals = true; + + @Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1) + public boolean testIntConstants = true; + + @Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1) + public boolean testStringConstants = true; + + @Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testDateConstants = true; + + @Parameter(names = "--test-timestamp-constants", description = "Allow generating TIMESTAMP constants", arity = 1) + public boolean testTimestampConstants = true; + + @Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1) + public boolean testFloatConstants = true; + + @Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1) + public boolean testBooleanConstants = true; + + @Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1) + public boolean testBinaryComparisons = true; + + @Parameter(names = "--test-indexes", description = "Allow explicit (i.e. CREATE INDEX) and implicit (i.e., UNIQUE and PRIMARY KEY) indexes", arity = 1) + public boolean testIndexes = true; + + @Parameter(names = "--test-rowid", description = "Test tables' rowid columns", arity = 1) + public boolean testRowid = true; + + @Parameter(names = "--max-num-views", description = "The maximum number of views that can be generated for a database", arity = 1) + public int maxNumViews = 1; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates = 5; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(DatabendOracleFactory.QUERY_PARTITIONING); + + public enum DatabendOracleFactory implements OracleFactory { + NOREC { + + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendNoRECOracle(globalState); + } + + }, + HAVING { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + List oracles = new ArrayList<>(); + oracles.add(new DatabendQueryPartitioningWhereTester(globalState)); + oracles.add(new DatabendQueryPartitioningHavingTester(globalState)); + oracles.add(new DatabendQueryPartitioningAggregateTester(globalState)); + oracles.add(new DatabendQueryPartitioningDistinctTester(globalState)); + oracles.add(new DatabendQueryPartitioningGroupByTester(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + + } + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java new file mode 100644 index 000000000..fd1f2d88f --- /dev/null +++ b/src/sqlancer/databend/DatabendProvider.java @@ -0,0 +1,154 @@ +package sqlancer.databend; + +import com.google.auto.service.AutoService; +import sqlancer.*; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.databend.gen.*; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.mysql.MySQLOptions; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +@AutoService(DatabaseProvider.class) +public class DatabendProvider extends SQLProviderAdapter { + + public DatabendProvider() { + super(DatabendGlobalState.class, DatabendOptions.class); + } + + public enum Action implements AbstractAction { + + INSERT(DatabendInsertGenerator::getQuery), // + CREATE_INDEX(DatabendIndexGenerator::getQuery), // + VACUUM((g) -> new SQLQueryAdapter("VACUUM;")), // + ANALYZE((g) -> new SQLQueryAdapter("ANALYZE;")), // + DELETE(DatabendDeleteGenerator::generate), // + UPDATE(DatabendUpdateGenerator::getQuery), // + CREATE_VIEW(DatabendViewGenerator::generate), // + EXPLAIN((g) -> { + ExpectedErrors errors = new ExpectedErrors(); + DatabendErrors.addExpressionErrors(errors); + DatabendErrors.addGroupByErrors(errors); + return new SQLQueryAdapter( + "EXPLAIN " + DatabendToStringVisitor + .asString(DatabendRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), + errors); + }); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(DatabendGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(DatabendGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case CREATE_INDEX: + if (!globalState.getDbmsSpecificOptions().testIndexes) { + return 0; + } + // fall through + case UPDATE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + case VACUUM: // seems to be ignored + case ANALYZE: // seems to be ignored + case EXPLAIN: + return r.getInteger(0, 2); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + case CREATE_VIEW: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + default: + throw new AssertionError(a); + } + } + + public static class DatabendGlobalState extends SQLGlobalState { + + @Override + protected DatabendSchema readSchema() throws SQLException { + return DatabendSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(DatabendGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new DatabendTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + DatabendProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + public void tryDeleteFile(String fname) { + try { + File f = new File(fname); + f.delete(); + } catch (Exception e) { + } + } + + public void tryDeleteDatabase(String dbpath) { + if (dbpath.equals("") || dbpath.equals(":memory:")) { + return; + } + tryDeleteFile(dbpath); + tryDeleteFile(dbpath + ".wal"); + } + + @Override + public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = MySQLOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = MySQLOptions.DEFAULT_PORT; + } + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), + globalState.getOptions().getPassword()); + Statement stmt = conn.createStatement(); + stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); + stmt.close(); + return new SQLConnection(conn); + } + + @Override + public String getDBMSName() { + return "Databend"; + } + +} diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java new file mode 100644 index 000000000..d0da77efd --- /dev/null +++ b/src/sqlancer/databend/DatabendSchema.java @@ -0,0 +1,277 @@ +package sqlancer.databend; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.*; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class DatabendSchema extends AbstractSchema { + + public enum DatabendDataType { + + INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP, NULL; + + public static DatabendDataType getRandomWithoutNull() { + DatabendDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DatabendDataType.NULL); + return dt; + } + + } + + public static class DatabendCompositeDataType { + + private final DatabendDataType dataType; + + private final int size; + + public DatabendCompositeDataType(DatabendDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public DatabendDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static DatabendCompositeDataType getRandomWithoutNull() { + DatabendDataType type = DatabendDataType.getRandomWithoutNull(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case BOOLEAN: + case VARCHAR: + case DATE: + case TIMESTAMP: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new DatabendCompositeDataType(type, size); + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 8: + return Randomly.fromOptions("BIGINT", "INT8"); + case 4: + return Randomly.fromOptions("INTEGER", "INT", "INT4", "SIGNED"); + case 2: + return Randomly.fromOptions("SMALLINT", "INT2"); + case 1: + return Randomly.fromOptions("TINYINT", "INT1"); + default: + throw new AssertionError(size); + } + case VARCHAR: + return "VARCHAR"; + case FLOAT: + switch (size) { + case 8: + return Randomly.fromOptions("DOUBLE"); + case 4: + return Randomly.fromOptions("REAL", "FLOAT4"); + default: + throw new AssertionError(size); + } + case BOOLEAN: + return Randomly.fromOptions("BOOLEAN", "BOOL"); + case TIMESTAMP: + return Randomly.fromOptions("TIMESTAMP", "DATETIME"); + case DATE: + return Randomly.fromOptions("DATE"); + case NULL: + return Randomly.fromOptions("NULL"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + } + + public static class DatabendColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + + public DatabendColumn(String name, DatabendCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class DatabendTables extends AbstractTables { + + public DatabendTables(List tables) { + super(tables); + } + + } + + public DatabendSchema(List databaseTables) { + super(databaseTables); + } + + public DatabendTables getRandomTableNonEmptyTables() { + return new DatabendTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + private static DatabendCompositeDataType getColumnType(String typeString) { + DatabendDataType primitiveType; + int size = -1; + if (typeString.startsWith("DECIMAL")) { // Ugly hack + return new DatabendCompositeDataType(DatabendDataType.FLOAT, 8); + } + switch (typeString) { + case "INTEGER": + primitiveType = DatabendDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = DatabendDataType.INT; + size = 2; + break; + case "BIGINT": + case "HUGEINT": // TODO: 16-bit int + primitiveType = DatabendDataType.INT; + size = 8; + break; + case "TINYINT": + primitiveType = DatabendDataType.INT; + size = 1; + break; + case "VARCHAR": + primitiveType = DatabendDataType.VARCHAR; + break; + case "FLOAT": + primitiveType = DatabendDataType.FLOAT; + size = 4; + break; + case "DOUBLE": + primitiveType = DatabendDataType.FLOAT; + size = 8; + break; + case "BOOLEAN": + primitiveType = DatabendDataType.BOOLEAN; + break; + case "DATE": + primitiveType = DatabendDataType.DATE; + break; + case "TIMESTAMP": + primitiveType = DatabendDataType.TIMESTAMP; + break; + case "NULL": + primitiveType = DatabendDataType.NULL; + break; + case "INTERVAL": + throw new IgnoreMeException(); + // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP + // '1969-12-29 06:50:27')) + default: + throw new AssertionError(typeString); + } + return new DatabendCompositeDataType(primitiveType, size); + } + + public static class DatabendTable extends AbstractRelationalTable { + + public DatabendTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static DatabendSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); + for (DatabendColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new DatabendSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master WHERE type='table' or type='view'")) { + while (rs.next()) { + tableNames.add(rs.getString("name")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("SELECT * FROM pragma_table_info('%s');", tableName))) { + while (rs.next()) { + String columnName = rs.getString("name"); + String dataType = rs.getString("type"); + boolean isNullable = rs.getString("notnull").contentEquals("false"); + boolean isPrimaryKey = rs.getString("pk").contains("true"); + DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); + columns.add(c); + } + } + } + if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { + // https://github.com/cwida/Databend/issues/589 + // https://github.com/cwida/Databend/issues/588 + // TODO: implement an option to enable/disable rowids + columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, false)); + } + return columns; + } + +} diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java new file mode 100644 index 000000000..50a37aeec --- /dev/null +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -0,0 +1,91 @@ +package sqlancer.databend; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; + +public class DatabendToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof DatabendConstant) { + visit((DatabendConstant) expr); + } else if (expr instanceof DatabendSelect) { + visit((DatabendSelect) expr); + } else if (expr instanceof DatabendJoin) { + visit((DatabendJoin) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(DatabendJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getOuterType() != null) { + sb.append(join.getOuterType()); + } + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(DatabendConstant constant) { + sb.append(constant.toString()); + } + + private void visit(DatabendSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(Node expr) { + DatabendToStringVisitor visitor = new DatabendToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java new file mode 100644 index 000000000..5e8d959c6 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -0,0 +1,192 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.Node; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public class DatabendConstant implements Node { + + private DatabendConstant() { + } + + public static class DatabendNullConstant extends DatabendConstant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class DatabendIntConstant extends DatabendConstant { + + private final long value; + + public DatabendIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class DatabendDoubleConstant extends DatabendConstant { + + private final double value; + + public DatabendDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + } + + public static class DatabendTextConstant extends DatabendConstant { + + private final String value; + + public DatabendTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static class DatabendBitConstant extends DatabendConstant { + + private final String value; + + public DatabendBitConstant(long value) { + this.value = Long.toBinaryString(value); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "B'" + value + "'"; + } + + } + + public static class DatabendDateConstant extends DatabendConstant { + + public String textRepr; + + public DatabendDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + } + + public static class DatabendTimestampConstant extends DatabendConstant { + + public String textRepr; + + public DatabendTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + } + + public static class DatabendBooleanConstant extends DatabendConstant { + + private final boolean value; + + public DatabendBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static Node createStringConstant(String text) { + return new DatabendTextConstant(text); + } + + public static Node createFloatConstant(double val) { + return new DatabendDoubleConstant(val); + } + + public static Node createIntConstant(long val) { + return new DatabendIntConstant(val); + } + + public static Node createNullConstant() { + return new DatabendNullConstant(); + } + + public static Node createBooleanConstant(boolean val) { + return new DatabendBooleanConstant(val); + } + + public static Node createDateConstant(long integer) { + return new DatabendDateConstant(integer); + } + + public static Node createTimestampConstant(long integer) { + return new DatabendTimestampConstant(integer); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendExpression.java b/src/sqlancer/databend/ast/DatabendExpression.java new file mode 100644 index 000000000..31b9bf859 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendExpression.java @@ -0,0 +1,5 @@ +package sqlancer.databend.ast; + +public interface DatabendExpression { + +} diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java new file mode 100644 index 000000000..ddc72ed53 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -0,0 +1,124 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.util.ArrayList; +import java.util.List; + +public class DatabendJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + private OuterType outerType; + + public enum JoinType { + INNER, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum OuterType { + FULL, LEFT, RIGHT; + + public static OuterType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public DatabendJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + private void setOuterType(OuterType outerType) { + this.outerType = outerType; + } + + public OuterType getOuterType() { + return outerType; + } + + public static List> getJoins( + List> tableList, DatabendGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); + switch (JoinType.getRandom()) { + case INNER: + joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + joinExpressions.add(DatabendJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); + break; + case LEFT: + joinExpressions + .add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions + .add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static DatabendJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DatabendJoin(left, right, JoinType.RIGHT, predicate); + } + + public static DatabendJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DatabendJoin(left, right, JoinType.LEFT, predicate); + } + + public static DatabendJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DatabendJoin(left, right, JoinType.INNER, predicate); + } + + public static Node createNaturalJoin(TableReferenceNode left, + TableReferenceNode right, OuterType naturalJoinType) { + DatabendJoin join = new DatabendJoin(left, right, JoinType.NATURAL, null); + join.setOuterType(naturalJoinType); + return join; + } + +} diff --git a/src/sqlancer/databend/ast/DatabendSelect.java b/src/sqlancer/databend/ast/DatabendSelect.java new file mode 100644 index 000000000..b3ccc08f5 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendSelect.java @@ -0,0 +1,18 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class DatabendSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + +} diff --git a/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java b/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java new file mode 100644 index 000000000..90920c819 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java @@ -0,0 +1,70 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendTable; + +public final class DatabendAlterTableGenerator { + + private DatabendAlterTableGenerator() { + } + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN + } + + public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add(" does not have a column with name \"rowid\""); + errors.add("Table does not contain column rowid referenced in alter statement"); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(DatabendCompositeDataType.getRandomWithoutNull().toString()); + break; + case ALTER_COLUMN: + sb.append("ALTER COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" SET DATA TYPE "); + sb.append(DatabendCompositeDataType.getRandomWithoutNull().toString()); + if (Randomly.getBoolean()) { + sb.append(" USING "); + DatabendErrors.addExpressionErrors(errors); + sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); + } + errors.add("Cannot change the type of this column: an index depends on it!"); + errors.add("Cannot change the type of a column that has a UNIQUE or PRIMARY KEY constraint specified"); + errors.add("Unimplemented type for cast"); + errors.add("Conversion:"); + errors.add("Cannot change the type of a column that has a CHECK constraint specified"); + break; + case DROP_COLUMN: + sb.append("DROP COLUMN "); + sb.append(table.getRandomColumn().getName()); + errors.add("named in key does not exist"); // TODO + errors.add("Cannot drop this column:"); + errors.add("Cannot drop column: table only has one column remaining!"); + errors.add("because there is a CHECK constraint that depends on it"); + errors.add("because there is a UNIQUE constraint that depends on it"); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java new file mode 100644 index 000000000..25dd83f45 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -0,0 +1,30 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendTable; + +public final class DatabendDeleteGenerator { + + private DatabendDeleteGenerator() { + } + + public static SQLQueryAdapter generate(DatabendGlobalState globalState) { + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(DatabendToStringVisitor.asString( + new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); + } + DatabendErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java new file mode 100644 index 000000000..0e7722175 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java @@ -0,0 +1,436 @@ +package sqlancer.databend.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.*; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public final class DatabendExpressionGenerator extends UntypedExpressionGenerator, DatabendColumn> { + + private final DatabendGlobalState globalState; + + public DatabendExpressionGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, CAST, FUNC, BETWEEN, CASE, + IN, COLLATE, LIKE_ESCAPE + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBoolean()) { + DatabendAggregateFunction aggregate = DatabendAggregateFunction.getRandom(); + allowAggregates = false; + return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + } + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + if (!globalState.getDbmsSpecificOptions().testCollate) { + possibleOptions.remove(Expression.COLLATE); + } + if (!globalState.getDbmsSpecificOptions().testFunctions) { + possibleOptions.remove(Expression.FUNC); + } + if (!globalState.getDbmsSpecificOptions().testCasts) { + possibleOptions.remove(Expression.CAST); + } + if (!globalState.getDbmsSpecificOptions().testBetween) { + possibleOptions.remove(Expression.BETWEEN); + } + if (!globalState.getDbmsSpecificOptions().testIn) { + possibleOptions.remove(Expression.IN); + } + if (!globalState.getDbmsSpecificOptions().testCase) { + possibleOptions.remove(Expression.CASE); + } + if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { + possibleOptions.remove(Expression.BINARY_COMPARISON); + } + if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { + possibleOptions.remove(Expression.BINARY_LOGICAL); + } + Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case COLLATE: + return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + DatabendCollate.getRandom()); + case UNARY_PREFIX: + return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), + DatabendUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + DatabendUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = DatabendBinaryComparisonOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = DatabendBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), DatabendBinaryArithmeticOperator.getRandom()); + case CAST: + return new DatabendCastOperation(generateExpression(depth + 1), + DatabendCompositeDataType.getRandomWithoutNull()); + case FUNC: + DBFunction func = DBFunction.getRandom(); + return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new NewBetweenOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new NewInOperatorNode(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new NewCaseOperatorNode(generateExpression(depth + 1), + generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), + generateExpression(depth + 1)); + case LIKE_ESCAPE: + return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), "LIKE", "ESCAPE"); + default: + throw new AssertionError(); + } + } + + @Override + protected Node generateColumn() { + DatabendColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode(column); + } + + @Override + public Node generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return DatabendConstant.createNullConstant(); + } + DatabendDataType type = DatabendDataType.getRandomWithoutNull(); + switch (type) { + case INT: + if (!globalState.getDbmsSpecificOptions().testIntConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DATE: + if (!globalState.getDbmsSpecificOptions().testDateConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createDateConstant(globalState.getRandomly().getInteger()); + case TIMESTAMP: + if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case VARCHAR: + if (!globalState.getDbmsSpecificOptions().testStringConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createStringConstant(globalState.getRandomly().getString()); + case BOOLEAN: + if (!globalState.getDbmsSpecificOptions().testBooleanConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + if (!globalState.getDbmsSpecificOptions().testFloatConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createFloatConstant(globalState.getRandomly().getDouble()); + default: + throw new AssertionError(); + } + } + + @Override + public List> generateOrderBys() { + List> expr = super.generateOrderBys(); + List> newExpr = new ArrayList<>(expr.size()); + for (Node curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + }; + + public static class DatabendCastOperation extends NewUnaryPostfixOperatorNode { + + public DatabendCastOperation(Node expr, DatabendCompositeDataType type) { + super(expr, new Operator() { + + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + } + + } + + public enum DatabendAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1), STRING_AGG(1), FIRST(1), SUM(1), STDDEV_SAMP(1), STDDEV_POP(1), VAR_POP(1), + VAR_SAMP(1), COVAR_POP(1), COVAR_SAMP(1); + + private int nrArgs; + + DatabendAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static DatabendAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } + + public enum DBFunction { + // trigonometric functions + ACOS(1), // + ASIN(1), // + ATAN(1), // + COS(1), // + SIN(1), // + TAN(1), // + COT(1), // + ATAN2(1), // + // math functions + ABS(1), // + CEIL(1), // + CEILING(1), // + FLOOR(1), // + LOG(1), // + LOG10(1), LOG2(1), // + LN(1), // + PI(0), // + SQRT(1), // + POWER(1), // + CBRT(1), // + ROUND(2), // + SIGN(1), // + DEGREES(1), // + RADIANS(1), // + MOD(2), // + XOR(2), // + // string functions + LENGTH(1), // + LOWER(1), // + UPPER(1), // + SUBSTRING(3), // + REVERSE(1), // + CONCAT(1, true), // + CONCAT_WS(1, true), CONTAINS(2), // + PREFIX(2), // + SUFFIX(2), // + INSTR(2), // + PRINTF(1, true), // + REGEXP_MATCHES(2), // + REGEXP_REPLACE(3), // + STRIP_ACCENTS(1), // + + // date functions + DATE_PART(2), AGE(2), + + COALESCE(3), NULLIF(2), + + // LPAD(3), + // RPAD(3), + LTRIM(1), RTRIM(1), + // LEFT(2), https://github.com/cwida/Databend/issues/633 + // REPEAT(2), + REPLACE(3), UNICODE(1), + + BIT_COUNT(1), BIT_LENGTH(1), LAST_DAY(1), MONTHNAME(1), DAYNAME(1), YEARWEEK(1), DAYOFMONTH(1), WEEKDAY(1), + WEEKOFYEAR(1), + + IFNULL(2), IF(3); + + private int nrArgs; + private boolean isVariadic; + + DBFunction(int nrArgs) { + this(nrArgs, false); + } + + DBFunction(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static DBFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + + } + + public enum DatabendUnaryPostfixOperator implements Operator { + + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + DatabendUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static DatabendUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public static final class DatabendCollate implements Operator { + + private final String textRepr; + + private DatabendCollate(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return "COLLATE " + textRepr; + } + + public static DatabendCollate getRandom() { + return new DatabendCollate(DatabendTableGenerator.getRandomCollate()); + } + + } + + public enum DatabendUnaryPrefixOperator implements Operator { + + NOT("NOT"), PLUS("+"), MINUS("-"); + + private String textRepr; + + DatabendUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static DatabendUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum DatabendBinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum DatabendBinaryArithmeticOperator implements Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); + + private String textRepr; + + DatabendBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum DatabendBinaryComparisonOperator implements Operator { + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), + REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); + + private String textRepr; + + DatabendBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public NewFunctionNode generateArgsForAggregate( + DatabendAggregateFunction aggregateFunction) { + return new NewFunctionNode( + generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + } + + public Node generateAggregate() { + DatabendAggregateFunction aggrFunc = DatabendAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, DatabendUnaryPrefixOperator.NOT); + } + + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, DatabendUnaryPostfixOperator.IS_NULL); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendIndexGenerator.java b/src/sqlancer/databend/gen/DatabendIndexGenerator.java new file mode 100644 index 000000000..de6820bcc --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendIndexGenerator.java @@ -0,0 +1,59 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.util.List; + +public final class DatabendIndexGenerator { + + private DatabendIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + if (Randomly.getBoolean()) { + errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + sb.append("UNIQUE "); + } + sb.append("INDEX "); + sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information + sb.append(" ON "); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append("("); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + } + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + Node expr = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(); + sb.append(DatabendToStringVisitor.asString(expr)); + } + errors.add("already exists!"); + if (globalState.getDbmsSpecificOptions().testRowid) { + errors.add("Cannot create an index on the rowid!"); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java new file mode 100644 index 000000000..065b2e3a3 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendInsertGenerator extends AbstractInsertGenerator { + + private final DatabendGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public DatabendInsertGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + return new DatabendInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + DatabendErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertValue(DatabendColumn tiDBColumn) { + // TODO: select a more meaningful value + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append("DEFAULT"); + } else { + sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); + } + } + +} diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java new file mode 100644 index 000000000..fcc6dbd7e --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -0,0 +1,70 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public final class DatabendRandomQuerySynthesizer { + + private DatabendRandomQuerySynthesizer() { + } + + public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { + DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); + DatabendSelect select = new DatabendSelect(); + // TODO: distinct + // select.setDistinct(Randomly.getBoolean()); + // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); + List> columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + // if (allowAggregates && Randomly.getBoolean()) { + Node expression = gen.generateExpression(); + columns.add(expression); + // } else { + // columns.add(gen()); + // } + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DatabendJoin.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + + if (Randomly.getBoolean()) { + select.setLimitClause(DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause( + DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + return select; + } + +} diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java new file mode 100644 index 000000000..cd0c15b7f --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -0,0 +1,89 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendTableGenerator { + + public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(); + UntypedExpressionGenerator, DatabendColumn> gen = new DatabendExpressionGenerator( + globalState).setColumns(columns); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + sb.append(columns.get(i).getType()); + if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { + sb.append(" COLLATE "); + sb.append(getRandomCollate()); + } + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" UNIQUE"); + } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } + if (globalState.getDbmsSpecificOptions().testCheckConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" CHECK("); + sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); + DatabendErrors.addExpressionErrors(errors); + sb.append(")"); + } + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DatabendToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + } + } + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { + errors.add("Invalid type for index"); + List primaryKeyColumns = Randomly.nonEmptySubset(columns); + sb.append(", PRIMARY KEY("); + sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + } + sb.append(")"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + public static String getRandomCollate() { + return Randomly.fromOptions("NOCASE", "NOACCENT", "NOACCENT.NOCASE", "C", "POSIX"); + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + DatabendCompositeDataType columnType = DatabendCompositeDataType.getRandomWithoutNull(); + columns.add(new DatabendColumn(columnName, columnType, false, false)); + } + return columns; + } + +} diff --git a/src/sqlancer/databend/gen/DatabendUpdateGenerator.java b/src/sqlancer/databend/gen/DatabendUpdateGenerator.java new file mode 100644 index 000000000..e2f396c4b --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendUpdateGenerator.java @@ -0,0 +1,48 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.util.List; + +public final class DatabendUpdateGenerator { + + private DatabendUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + StringBuilder sb = new StringBuilder("UPDATE "); + ExpectedErrors errors = new ExpectedErrors(); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(" SET "); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append("="); + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + DatabendErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(DatabendToStringVisitor.asString(expr)); + } + DatabendErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendViewGenerator.java b/src/sqlancer/databend/gen/DatabendViewGenerator.java new file mode 100644 index 000000000..55a7c6366 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendViewGenerator.java @@ -0,0 +1,37 @@ +package sqlancer.databend.gen; + +import sqlancer.GlobalState; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; + +public final class DatabendViewGenerator { + + private DatabendViewGenerator() { + } + + public static SQLQueryAdapter generate(DatabendGlobalState globalState) { + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(DatabendToStringVisitor.asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + DatabendErrors.addExpressionErrors(errors); + DatabendErrors.addGroupByErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java new file mode 100644 index 000000000..21453005f --- /dev/null +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -0,0 +1,133 @@ +package sqlancer.databend.test; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.*; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendNoRECOracle extends NoRECBase implements TestOracle { + + private final DatabendSchema s; + + public DatabendNoRECOracle(DatabendGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + DatabendErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + DatabendTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); + Node randomWhereCondition = gen.generateExpression(); + List tables = randomTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DatabendJoin.getJoins(tableList, state); + int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); + int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, + randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getSecondQuery(List> tableList, Node randomWhereCondition, + List> joins) throws SQLException { + DatabendSelect select = new DatabendSelect(); + // select.setGroupByClause(groupBys); + // DatabendExpression isTrue = DatabendPostfixOperation.create(randomWhereCondition, + // PostfixOperator.IS_TRUE); + Node asText = new NewPostfixTextNode<>(new DatabendCastOperation( + new NewPostfixTextNode(randomWhereCondition, + " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), + new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + select.setFromList(tableList); + // select.setSelectType(SelectType.ALL); + select.setJoinList(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + DatabendToStringVisitor.asString(select) + ") as res"; + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, List> joins) + throws SQLException { + DatabendSelect select = new DatabendSelect(); + // select.setGroupByClause(groupBys); + // DatabendAggregate aggr = new DatabendAggregate( + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + // DatabendAggregateFunction.COUNT); + // select.setFetchColumns(Arrays.asList(aggr)); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new DatabendExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + // select.setSelectType(SelectType.ALL); + select.setJoinList(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = DatabendToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java new file mode 100644 index 000000000..e584da545 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -0,0 +1,188 @@ +package sqlancer.databend.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendAggregateFunction; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendBinaryArithmeticOperator; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPrefixOperator; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPostfixOperator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public DatabendQueryPartitioningAggregateTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + DatabendAggregateFunction aggregateFunction = Randomly.fromOptions(DatabendAggregateFunction.MAX, + DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT, + DatabendAggregateFunction.AVG/* , DatabendAggregateFunction.STDDEV_POP */); + NewFunctionNode aggregate = gen + .generateArgsForAggregate(aggregateFunction); + List> fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + originalQuery = DatabendToStringVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResult == null && secondResult != null + || firstResult != null && (!firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { + if (secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + throw new AssertionError(); + } + + } + + private String createMetamorphicUnionQuery(DatabendSelect select, + NewFunctionNode aggregate, List> from) { + String metamorphicQuery; + Node whereClause = gen.generateExpression(); + Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + DatabendUnaryPrefixOperator.NOT); + Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + DatabendUnaryPostfixOperator.IS_NULL); + List> mappedAggregate = mapped(aggregate); + DatabendSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + DatabendSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + DatabendSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += DatabendToStringVisitor.asString(leftSelect) + " UNION ALL " + + DatabendToStringVisitor.asString(middleSelect) + " UNION ALL " + + DatabendToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List> mapped(NewFunctionNode aggregate) { + DatabendCastOperation count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(Arrays.asList(aggregate)); + case AVG: + NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + DatabendAggregateFunction.SUM); + count = new DatabendCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), + new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); + return aliasArgs(Arrays.asList(sum, count)); + case STDDEV_POP: + NewFunctionNode sumSquared = new NewFunctionNode<>( + Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DatabendBinaryArithmeticOperator.MULT)), + DatabendAggregateFunction.SUM); + count = new DatabendCastOperation( + new NewFunctionNode(aggregate.getArgs(), + DatabendAggregateFunction.COUNT), + new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); + NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), + DatabendAggregateFunction.AVG); + return aliasArgs(Arrays.asList(sumSquared, count, avg)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List> aliasArgs(List> originalAggregateArgs) { + List> args = new ArrayList<>(); + int i = 0; + for (Node expr : originalAggregateArgs) { + args.add(new NewAliasNode(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(NewFunctionNode aggregate) { + switch (aggregate.getFunc()) { + case STDDEV_POP: + return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; + case AVG: + return "SUM(agg0::FLOAT)/SUM(agg1)::FLOAT"; + case COUNT: + return DatabendAggregateFunction.SUM.toString() + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private DatabendSelect getSelect(List> aggregates, List> from, + Node whereClause, List> joinList) { + DatabendSelect leftSelect = new DatabendSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java new file mode 100644 index 000000000..a806dd2ca --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -0,0 +1,82 @@ +package sqlancer.databend.test; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class DatabendQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { + + DatabendSchema s; + DatabendTables targetTables; + DatabendExpressionGenerator gen; + DatabendSelect select; + + public DatabendQueryPartitioningBase(DatabendGlobalState state) { + super(state); + DatabendErrors.addExpressionErrors(errors); + } + + public static String canonicalizeResultValue(String value) { + // Rule: -0.0 should be canonicalized to 0.0 + if (Objects.equals(value, "-0.0")) { + return "0.0"; + } + + return value; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new DatabendExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new DatabendSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DatabendJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..23572b110 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java @@ -0,0 +1,44 @@ +package sqlancer.databend.test; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class DatabendQueryPartitioningDistinctTester extends DatabendQueryPartitioningBase { + + public DatabendQueryPartitioningDistinctTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + if (Randomly.getBoolean()) { + select.setDistinct(false); + } + select.setWhereClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DatabendQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..0dbdda637 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java @@ -0,0 +1,53 @@ +package sqlancer.databend.test; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendQueryPartitioningGroupByTester extends DatabendQueryPartitioningBase { + + public DatabendQueryPartitioningGroupByTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setGroupByExpressions(select.getFetchColumns()); + select.setWhereClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DatabendQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + List> generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java new file mode 100644 index 000000000..019eeace7 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -0,0 +1,62 @@ +package sqlancer.databend.test; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase implements TestOracle { + + public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setHavingClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DatabendQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + protected Node generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List> generateFetchColumns() { + return Arrays.asList(gen.generateHavingClause()); + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java new file mode 100644 index 000000000..74f9a22c1 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java @@ -0,0 +1,45 @@ +package sqlancer.databend.test; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class DatabendQueryPartitioningWhereTester extends DatabendQueryPartitioningBase { + + public DatabendQueryPartitioningWhereTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setWhereClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DatabendQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java new file mode 100644 index 000000000..a88ad740b --- /dev/null +++ b/test/sqlancer/dbms/TestDatabend.java @@ -0,0 +1,18 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestDatabend { + + @Test + public void testDatabend() { + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "databend", "--oracle", + "NoREC" })); + } + +} From e09edfaaccdba5ad446040f9197084653a1cbb45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jun 2022 09:12:36 +0000 Subject: [PATCH 0570/1284] Bump checkstyle from 10.3 to 10.3.1 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3 to 10.3.1. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3...checkstyle-10.3.1) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e09cc0954..2f69dc63f 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.3 + 10.3.1 From d4c43ea5aadd7b9d6d846915d1f7a1cd61de04c0 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 30 Jun 2022 15:32:22 +0200 Subject: [PATCH 0571/1284] Fix ClickHouse connection string --- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 ++ src/sqlancer/clickhouse/ClickHouseProvider.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 08df43612..17993c7e4 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -35,6 +35,8 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 errors.add("Cannot parse number with a sign character but without any numeric character"); errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); + errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); + errors.add("Positional argument out of bounds"); } public static void addExpressionHavingErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 3585aa9df..7ff2de11b 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -118,7 +118,7 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ ClickHouseOptions clickHouseOptions = globalState.getDbmsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); - String url = String.format("jdbc:clickhouse://%s:%d/default", host, port); + String url = String.format("jdbc:clickhouse://%s:%d/%s", host, port, "default"); String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); @@ -139,7 +139,7 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ e.printStackTrace(); } con.close(); - con = DriverManager.getConnection("jdbc:clickhouse://localhost:8123/" + databaseName, + con = DriverManager.getConnection(String.format("jdbc:clickhouse://%s:%d/%s", host, port, databaseName), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); return new SQLConnection(con); } From 87fb11d5e7ae7b97e3586edf86a92916a8b640d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Jul 2022 08:24:38 +0000 Subject: [PATCH 0572/1284] Bump mariadb-java-client from 3.0.5 to 3.0.6 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.5 to 3.0.6. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.5...3.0.6) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2f69dc63f..71dc5e90e 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.5 + 3.0.6 org.duckdb From 54bdd811d82fa560658f51bc961fa8e812d82065 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 1 Jul 2022 21:17:14 +0800 Subject: [PATCH 0573/1284] feat: implement databend connection --- .settings/org.eclipse.jdt.core.prefs | 9 ++++-- src/sqlancer/Main.java | 6 ++++ src/sqlancer/databend/DatabendOptions.java | 2 ++ src/sqlancer/databend/DatabendProvider.java | 30 ++++++++++++------- .../databend/TestDatabendConnection.java | 19 ++++++++++++ test/sqlancer/dbms/TestDatabend.java | 5 ++-- 6 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 test/sqlancer/databend/TestDatabendConnection.java diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 0a659c1a7..3af1c5ad8 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.compliance=11 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,6 +28,7 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -73,6 +74,7 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -110,5 +112,6 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=1.8 +org.eclipse.jdt.core.compiler.source=11 diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a4404a6fa..6fc4aeacf 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -440,6 +440,12 @@ private String formatInteger(long intValue) { try { executorFactory.getDBMSExecutor(options.getDatabasePrefix() + "connectiontest", new Randomly()) .testConnection(); + //临时测试databend connection TODO + if(executorFactory.provider.getDBMSName() == "databend"){ + System.out.println("Databend连接成功！"); + return 0; //正常退出 + } + //----------------- } catch (Exception e) { System.err.println( "SQLancer failed creating a test database, indicating that SQLancer might have failed connecting to the DBMS. In order to change the username, password, host and port, you can use the --username, --password, --host and --port options.\n\n"); diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index 1e70bcb66..de6940a76 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -17,6 +17,8 @@ @Parameters(commandDescription = "Databend") public class DatabendOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 3307; @Parameter(names = "--test-collate", arity = 1) public boolean testCollate = true; diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index fd1f2d88f..faf136a3c 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -7,7 +7,6 @@ import sqlancer.common.query.SQLQueryProvider; import sqlancer.databend.gen.*; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.mysql.MySQLOptions; import java.io.File; import java.sql.Connection; @@ -131,24 +130,35 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE String host = globalState.getOptions().getHost(); int port = globalState.getOptions().getPort(); if (host == null) { - host = MySQLOptions.DEFAULT_HOST; + host = DatabendOptions.DEFAULT_HOST; } if (port == MainOptions.NO_SET_PORT) { - port = MySQLOptions.DEFAULT_PORT; + port = DatabendOptions.DEFAULT_PORT; } + String databaseName = globalState.getDatabaseName(); + //记录日志和执行语句处放一起或许比较合适？ TODO + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + //-------------------------------------------- String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", host, port); - Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); - Statement stmt = conn.createStatement(); - stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); - stmt.close(); - return new SQLConnection(conn); + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); } @Override public String getDBMSName() { - return "Databend"; + return "databend"; } } diff --git a/test/sqlancer/databend/TestDatabendConnection.java b/test/sqlancer/databend/TestDatabendConnection.java new file mode 100644 index 000000000..d05b33a2b --- /dev/null +++ b/test/sqlancer/databend/TestDatabendConnection.java @@ -0,0 +1,19 @@ +package sqlancer.databend; + +import com.mongodb.annotations.ThreadSafe; +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestDatabendConnection { + @Test + void testConnection(){ + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", + "--host","192.168.81.133","--port","3307","--username","user1" + ,"databend","--oracle","NoREC"})); + } +} diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index a88ad740b..269093349 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -11,8 +11,9 @@ public class TestDatabend { public void testDatabend() { assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "databend", "--oracle", - "NoREC" })); + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", + "--host","192.168.81.133","--port","3307","--username","user1" + ,"databend","--oracle","NoREC"})); } } From d4594c0876d015f2dadc143be5f0564beb842e0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 09:25:56 +0000 Subject: [PATCH 0574/1284] Bump spotbugs-maven-plugin from 4.7.0.0 to 4.7.1.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.0.0 to 4.7.1.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.0.0...spotbugs-maven-plugin-4.7.1.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 71dc5e90e..423a3ae94 100644 --- a/pom.xml +++ b/pom.xml @@ -250,7 +250,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.0.0 + 4.7.1.0 spotbugs From 23a960e9955846997cad3870eb2c182a8ac60ec1 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 22 Jul 2022 17:52:53 +0800 Subject: [PATCH 0575/1284] fix: fix readSchema Exception and error insert value --- src/sqlancer/GlobalState.java | 9 ++- src/sqlancer/Main.java | 6 -- src/sqlancer/databend/DatabendProvider.java | 34 +++------ src/sqlancer/databend/DatabendSchema.java | 76 ++++++++++++------- .../databend/ast/DatabendConstant.java | 8 +- .../gen/DatabendExpressionGenerator.java | 4 +- .../databend/gen/DatabendInsertGenerator.java | 16 ++-- .../databend/TestDatabendConnection.java | 3 +- test/sqlancer/dbms/TestDatabend.java | 19 ----- 9 files changed, 91 insertions(+), 84 deletions(-) delete mode 100644 test/sqlancer/dbms/TestDatabend.java diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 64b5c731e..20efcdcc8 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -5,6 +5,8 @@ import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; +import java.sql.SQLException; + public abstract class GlobalState, S extends AbstractSchema, C extends SQLancerDBConnection> { protected C databaseConnection; @@ -142,7 +144,12 @@ protected void setSchema(S schema) { } public void updateSchema() throws Exception { - setSchema(readSchema()); + try{ + setSchema(readSchema()); + } catch (SQLException sqlException) { + System.out.println(String.format("%s readSchema SQLException",databaseName)); + sqlException.printStackTrace(); + } for (AbstractTable table : schema.getDatabaseTables()) { table.recomputeCount(); } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 6fc4aeacf..a4404a6fa 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -440,12 +440,6 @@ private String formatInteger(long intValue) { try { executorFactory.getDBMSExecutor(options.getDatabasePrefix() + "connectiontest", new Randomly()) .testConnection(); - //临时测试databend connection TODO - if(executorFactory.provider.getDBMSName() == "databend"){ - System.out.println("Databend连接成功！"); - return 0; //正常退出 - } - //----------------- } catch (Exception e) { System.err.println( "SQLancer failed creating a test database, indicating that SQLancer might have failed connecting to the DBMS. In order to change the username, password, host and port, you can use the --username, --password, --host and --port options.\n\n"); diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index faf136a3c..ffb23124a 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -24,11 +24,9 @@ public DatabendProvider() { public enum Action implements AbstractAction { INSERT(DatabendInsertGenerator::getQuery), // - CREATE_INDEX(DatabendIndexGenerator::getQuery), // - VACUUM((g) -> new SQLQueryAdapter("VACUUM;")), // - ANALYZE((g) -> new SQLQueryAdapter("ANALYZE;")), // - DELETE(DatabendDeleteGenerator::generate), // - UPDATE(DatabendUpdateGenerator::getQuery), // + //TODO 等待databend实现update && delete +// DELETE(DatabendDeleteGenerator::generate), // +// UPDATE(DatabendUpdateGenerator::getQuery), // CREATE_VIEW(DatabendViewGenerator::generate), // EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); @@ -57,19 +55,13 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { switch (a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case CREATE_INDEX: - if (!globalState.getDbmsSpecificOptions().testIndexes) { - return 0; - } - // fall through - case UPDATE: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); - case VACUUM: // seems to be ignored - case ANALYZE: // seems to be ignored case EXPLAIN: return r.getInteger(0, 2); - case DELETE: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + //TODO 等待databend实现update && delete +// case UPDATE: +// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); +// case DELETE: +// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); case CREATE_VIEW: return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: @@ -136,29 +128,27 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE port = DatabendOptions.DEFAULT_PORT; } String databaseName = globalState.getDatabaseName(); - //记录日志和执行语句处放一起或许比较合适？ TODO - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - globalState.getState().logStatement("CREATE DATABASE " + databaseName); - globalState.getState().logStatement("USE " + databaseName); - //-------------------------------------------- String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", host, port); Connection con = DriverManager.getConnection(url, username, password); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); } try (Statement s = con.createStatement()) { s.execute("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); } try (Statement s = con.createStatement()) { s.execute("USE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); } return new SQLConnection(con); } @Override public String getDBMSName() { - return "databend"; + return "databend"; //用于DatabendOptions } } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index d0da77efd..5cc09914b 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -29,6 +29,14 @@ public static DatabendDataType getRandomWithoutNull() { return dt; } + public static DatabendDataType getRandomWithoutNullAndVarchar() { + DatabendDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DatabendDataType.NULL || dt == DatabendDataType.VARCHAR); + return dt; + } + } public static class DatabendCompositeDataType { @@ -82,24 +90,24 @@ public String toString() { case INT: switch (size) { case 8: - return Randomly.fromOptions("BIGINT", "INT8"); + return Randomly.fromOptions("BIGINT", "INT64"); case 4: - return Randomly.fromOptions("INTEGER", "INT", "INT4", "SIGNED"); + return Randomly.fromOptions("INT", "INT32"); case 2: - return Randomly.fromOptions("SMALLINT", "INT2"); + return Randomly.fromOptions("SMALLINT", "INT16"); case 1: - return Randomly.fromOptions("TINYINT", "INT1"); + return Randomly.fromOptions("TINYINT", "INT8"); default: throw new AssertionError(size); } case VARCHAR: - return "VARCHAR"; + return Randomly.fromOptions("VARCHAR"); case FLOAT: switch (size) { case 8: return Randomly.fromOptions("DOUBLE"); case 4: - return Randomly.fromOptions("REAL", "FLOAT4"); + return Randomly.fromOptions("FLOAT"); default: throw new AssertionError(size); } @@ -162,7 +170,7 @@ private static DatabendCompositeDataType getColumnType(String typeString) { return new DatabendCompositeDataType(DatabendDataType.FLOAT, 8); } switch (typeString) { - case "INTEGER": + case "INT": primitiveType = DatabendDataType.INT; size = 4; break; @@ -171,7 +179,6 @@ private static DatabendCompositeDataType getColumnType(String typeString) { size = 2; break; case "BIGINT": - case "HUGEINT": // TODO: 16-bit int primitiveType = DatabendDataType.INT; size = 8; break; @@ -222,12 +229,12 @@ public DatabendTable(String tableName, List columns, boolean isV public static DatabendSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); - List tableNames = getTableNames(con); + List tableNames = getTableNames(con,databaseName); for (String tableName : tableNames) { if (DBMSCommon.matchesIndexName(tableName)) { continue; // TODO: unexpected? } - List databaseColumns = getTableColumns(con, tableName); + List databaseColumns = getTableColumns(con, tableName,databaseName); boolean isView = tableName.startsWith("v"); DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); for (DatabendColumn c : databaseColumns) { @@ -239,37 +246,52 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa return new DatabendSchema(databaseTables); } - private static List getTableNames(SQLConnection con) throws SQLException { - List tableNames = new ArrayList<>(); + private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { + List tableNames = null; + tableNames = new ArrayList<>(); + //SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema != 'system' and table_schema != 'INFORMATION_SCHEMA' and table_type='BASE TABLE' + //"SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName + final String sqlStatement = String.format( + "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master WHERE type='table' or type='view'")) { - while (rs.next()) { - tableNames.add(rs.getString("name")); + try (ResultSet rs = s.executeQuery(sqlStatement)) { + try{ //没有catch的话rs.next()会报SQLException：Not a navigable ResultSet + while (rs.next()) { + tableNames.add(rs.getString("table_name")); + } + } catch (Exception e){ +// e.printStackTrace(); + System.out.println("TableNames->SQLException：Not a navigable ResultSet"); } } } return tableNames; } - private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery(String.format("SELECT * FROM pragma_table_info('%s');", tableName))) { - while (rs.next()) { - String columnName = rs.getString("name"); - String dataType = rs.getString("type"); - boolean isNullable = rs.getString("notnull").contentEquals("false"); - boolean isPrimaryKey = rs.getString("pk").contains("true"); - DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); - columns.add(c); + try (ResultSet rs = s.executeQuery(String.format( + "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = '%s' and table_name ='%s'", + databaseName,tableName))) { + try{ //没有catch的话rs.next()会报SQLException：Not a navigable ResultSet + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + boolean isNullable = rs.getBoolean("is_nullable"); +// boolean isPrimaryKey = rs.getString("pk").contains("true"); + boolean isPrimaryKey = false; //没找到主键元数据 + DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); + columns.add(c); + } + } catch (Exception e) { + System.out.println("TableColumns->SQLException：Not a navigable ResultSet"); } } } if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { - // https://github.com/cwida/Databend/issues/589 - // https://github.com/cwida/Databend/issues/588 // TODO: implement an option to enable/disable rowids - columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, false)); +// columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, false)); } return columns; } diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 5e8d959c6..57ab52dba 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -57,7 +57,13 @@ public String toString() { } else if (value == Double.NEGATIVE_INFINITY) { return "'-Inf'"; } - return String.valueOf(value); + + +// DecimalFormat doubleFormat = new DecimalFormat(); +// doubleFormat.setMaximumFractionDigits(2); //小数点后2位 +// System.out.println(String.valueOf(doubleFormat.format(value))); + //TODO value先返回0.3来取消指数形式表达，等待Databend支持指数形式表达 + return String.valueOf(0.3); } } diff --git a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java index 0e7722175..1b7f5a36c 100644 --- a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java @@ -123,7 +123,9 @@ public Node generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return DatabendConstant.createNullConstant(); } - DatabendDataType type = DatabendDataType.getRandomWithoutNull(); +// DatabendDataType type = DatabendDataType.getRandomWithoutNull(); + //TODO 先跳过varchar等待databend对其更好的支持，或自己改写case VARCHAR的代码 + DatabendDataType type = DatabendDataType.getRandomWithoutNullAndVarchar(); switch (type) { case INT: if (!globalState.getDbmsSpecificOptions().testIntConstants) { diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 065b2e3a3..a61e56263 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -42,12 +42,16 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(DatabendColumn tiDBColumn) { - // TODO: select a more meaningful value - if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append("DEFAULT"); - } else { - sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); - } + // TODO: 等Databend实现NULL 和 DEFAULT ,暂时注入普通的value +// if (Randomly.getBooleanWithRatherLowProbability()) { +// sb.append("DEFAULT"); +// } else { +// sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); +// } + + String value = DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant()); + sb.append(value); + } } diff --git a/test/sqlancer/databend/TestDatabendConnection.java b/test/sqlancer/databend/TestDatabendConnection.java index d05b33a2b..91e0a1d53 100644 --- a/test/sqlancer/databend/TestDatabendConnection.java +++ b/test/sqlancer/databend/TestDatabendConnection.java @@ -3,6 +3,7 @@ import com.mongodb.annotations.ThreadSafe; import org.junit.jupiter.api.Test; import sqlancer.Main; +import sqlancer.Randomly; import sqlancer.dbms.TestConfig; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -13,7 +14,7 @@ void testConnection(){ assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", - "--host","192.168.81.133","--port","3307","--username","user1" + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC),"--host","192.168.81.134","--port","3307","--username","user1" ,"databend","--oracle","NoREC"})); } } diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java deleted file mode 100644 index 269093349..000000000 --- a/test/sqlancer/dbms/TestDatabend.java +++ /dev/null @@ -1,19 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class TestDatabend { - - @Test - public void testDatabend() { - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", - "--host","192.168.81.133","--port","3307","--username","user1" - ,"databend","--oracle","NoREC"})); - } - -} From f356d11e7a9a2ff6e40b4dbd2f185e1214f9f7ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jul 2022 09:15:47 +0000 Subject: [PATCH 0576/1284] Bump spotbugs-maven-plugin from 4.7.1.0 to 4.7.1.1 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.1.0 to 4.7.1.1. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.1.0...spotbugs-maven-plugin-4.7.1.1) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 423a3ae94..c86498379 100644 --- a/pom.xml +++ b/pom.xml @@ -250,7 +250,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.1.0 + 4.7.1.1 spotbugs From a34d67a26f6e79f6906e1f1c9906617d32eb459a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 07:07:38 +0000 Subject: [PATCH 0577/1284] Bump formatter-maven-plugin from 2.19.0 to 2.20.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.19.0 to 2.20.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.19.0...formatter-maven-plugin-2.20.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c86498379..f4221c827 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.19.0 + 2.20.0 eclipseformat From c0332ca5a9880d8ebc4cf8e9dc754e4755da3e1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 09:24:46 +0000 Subject: [PATCH 0578/1284] Bump mysql-connector-java from 8.0.29 to 8.0.30 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.29 to 8.0.30. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.29...8.0.30) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f4221c827..28051f42b 100644 --- a/pom.xml +++ b/pom.xml @@ -292,7 +292,7 @@ mysql mysql-connector-java - 8.0.29 + 8.0.30 org.mariadb.jdbc From c18c6f0166a4d17f0565dd38d39adf4c754594ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 14:12:28 +0000 Subject: [PATCH 0579/1284] Bump plexus-compiler-api from 2.12.0 to 2.12.1 Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.12.0 to 2.12.1. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.12.0...plexus-compiler-2.12.1) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28051f42b..999821ab6 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.codehaus.plexus plexus-compiler-api - 2.12.0 + 2.12.1 From 879daa6614cc58de6eb2dc9f5212eb71dc890fff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Jul 2022 02:51:35 +0000 Subject: [PATCH 0580/1284] Bump plexus-compiler-eclipse from 2.12.0 to 2.12.1 Bumps plexus-compiler-eclipse from 2.12.0 to 2.12.1. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 999821ab6..6184911c5 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.12.0 + 2.12.1 org.eclipse.jdt From d5d73cab478b59c5aef7cb49db077c3b4eafd60d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Jul 2022 09:15:57 +0000 Subject: [PATCH 0581/1284] Bump junit-jupiter-engine from 5.8.2 to 5.9.0 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.8.2 to 5.9.0. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.8.2...r5.9.0) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6184911c5..e51f86f9a 100644 --- a/pom.xml +++ b/pom.xml @@ -307,7 +307,7 @@ org.junit.jupiter junit-jupiter-engine - 5.8.2 + 5.9.0 test From 6a5dd94f3346c227d93c6a37ea589c5d10f7ce86 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 1 Aug 2022 10:01:16 +0800 Subject: [PATCH 0582/1284] disable EXTRA_FLOAT_DIGITS --- .../cockroachdb/gen/CockroachDBSetSessionGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index 8aa26e88c..48bb42fb7 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -27,7 +27,7 @@ private enum CockroachDBSetting { ENABLE_ZIGZAG_JOIN(CockroachDBSetSessionGenerator::onOff), // EXPERIMENTAL_ENABLE_HASH_SHARDED_INDEXES(CockroachDBSetSessionGenerator::onOff), SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), - EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), + // EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), REORDER_JOINS_LIMIT((g) -> g.getRandomly().getInteger(0, Integer.MAX_VALUE)), // SQL_SAFE_UPDATES((g) -> "off"), // TRACING(CockroachDBSetSessionGenerator::onOff) From a0fb2187430e739f9230372c4795dd763ca24fc0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 09:20:50 +0000 Subject: [PATCH 0583/1284] Bump checkstyle from 10.3.1 to 10.3.2 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3.1 to 10.3.2. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3.1...checkstyle-10.3.2) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e51f86f9a..721236136 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.3.1 + 10.3.2 From d074cb9da966108f7031418e670ec51cb0775b9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Aug 2022 09:14:18 +0000 Subject: [PATCH 0584/1284] Bump postgresql from 42.4.0 to 42.4.1 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.4.0 to 42.4.1. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.4.0...REL42.4.1) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 721236136..9c39ee442 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.4.0 + 42.4.1 org.xerial From 9347a09dfffd54f21f3906d866a3aa2249956989 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Aug 2022 09:11:53 +0000 Subject: [PATCH 0585/1284] Bump mariadb-java-client from 3.0.6 to 3.0.7 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.6 to 3.0.7. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.6...3.0.7) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9c39ee442..f87671fd8 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.6 + 3.0.7 org.duckdb From 9195b29605cbaccacc7de4efe92e264f191c74e7 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 12 Aug 2022 21:11:44 +0800 Subject: [PATCH 0586/1284] fix: comment out some expressions not yet supported by databend --- src/sqlancer/databend/DatabendProvider.java | 8 +- src/sqlancer/databend/DatabendSchema.java | 27 ++-- .../gen/DatabendExpressionGenerator.java | 133 ++++++++++-------- .../gen/DatabendRandomQuerySynthesizer.java | 2 +- .../databend/gen/DatabendTableGenerator.java | 47 ++++--- .../databend/test/DatabendNoRECOracle.java | 8 +- 6 files changed, 121 insertions(+), 104 deletions(-) diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index ffb23124a..241bb136a 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -27,7 +27,7 @@ public enum Action implements AbstractAction { //TODO 等待databend实现update && delete // DELETE(DatabendDeleteGenerator::generate), // // UPDATE(DatabendUpdateGenerator::getQuery), // - CREATE_VIEW(DatabendViewGenerator::generate), // +// CREATE_VIEW(DatabendViewGenerator::generate), // TODO 等待databend的create view语法 更加贴近mysql EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); @@ -62,8 +62,8 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); // case DELETE: // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); - case CREATE_VIEW: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); +// case CREATE_VIEW: +// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); } @@ -96,7 +96,7 @@ public void generateDatabase(DatabendGlobalState globalState) throws Exception { throw new IgnoreMeException(); } }); - se.executeStatements(); + se.executeStatements(); //在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 } public void tryDeleteFile(String fname) { diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 5cc09914b..ad92932d2 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -19,7 +19,8 @@ public class DatabendSchema extends AbstractSchema generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { +// if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { +// return generateLeafNode(); +// } + if (depth >= 1) { return generateLeafNode(); } - if (allowAggregates && Randomly.getBoolean()) { - DatabendAggregateFunction aggregate = DatabendAggregateFunction.getRandom(); - allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); - } +// if (allowAggregates && Randomly.getBoolean()) { +// DatabendAggregateFunction aggregate = DatabendAggregateFunction.getRandom(); +// allowAggregates = false; +// return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); +// } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - if (!globalState.getDbmsSpecificOptions().testCollate) { - possibleOptions.remove(Expression.COLLATE); - } - if (!globalState.getDbmsSpecificOptions().testFunctions) { - possibleOptions.remove(Expression.FUNC); - } - if (!globalState.getDbmsSpecificOptions().testCasts) { - possibleOptions.remove(Expression.CAST); - } +// if (!globalState.getDbmsSpecificOptions().testCollate) { +// possibleOptions.remove(Expression.COLLATE); +// } +// if (!globalState.getDbmsSpecificOptions().testFunctions) { +// possibleOptions.remove(Expression.FUNC); +// } +// if (!globalState.getDbmsSpecificOptions().testCasts) { +// possibleOptions.remove(Expression.CAST); +// } if (!globalState.getDbmsSpecificOptions().testBetween) { possibleOptions.remove(Expression.BETWEEN); } if (!globalState.getDbmsSpecificOptions().testIn) { possibleOptions.remove(Expression.IN); } - if (!globalState.getDbmsSpecificOptions().testCase) { - possibleOptions.remove(Expression.CASE); - } +// if (!globalState.getDbmsSpecificOptions().testCase) { +// possibleOptions.remove(Expression.CASE); +// } if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { possibleOptions.remove(Expression.BINARY_COMPARISON); } @@ -67,9 +77,9 @@ protected Node generateExpression(int depth) { } Expression expr = Randomly.fromList(possibleOptions); switch (expr) { - case COLLATE: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), - DatabendCollate.getRandom()); +// case COLLATE: +// return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), +// DatabendCollate.getRandom()); case UNARY_PREFIX: return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), DatabendUnaryPrefixOperator.getRandom()); @@ -84,30 +94,31 @@ protected Node generateExpression(int depth) { op = DatabendBinaryLogicalOperator.getRandom(); return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), op); - case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), DatabendBinaryArithmeticOperator.getRandom()); - case CAST: - return new DatabendCastOperation(generateExpression(depth + 1), - DatabendCompositeDataType.getRandomWithoutNull()); - case FUNC: - DBFunction func = DBFunction.getRandom(); - return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); +// case BINARY_ARITHMETIC: +// return new NewBinaryOperatorNode(generateExpression(depth + 1), +// generateExpression(depth + 1), DatabendBinaryArithmeticOperator.getRandom()); +// case CAST: +// return new DatabendCastOperation(generateExpression(depth + 1), +// DatabendCompositeDataType.getRandomWithoutNull()); +// case FUNC: +// DBFunction func = DBFunction.getRandom(); +// return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); case BETWEEN: return new NewBetweenOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); case IN: return new NewInOperatorNode(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); - case CASE: - int nr = Randomly.smallNumber() + 1; - return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), - generateExpression(depth + 1)); - case LIKE_ESCAPE: - return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), - generateExpression(depth + 1), "LIKE", "ESCAPE"); +// case CASE: +// int nr = Randomly.smallNumber() + 1; +// return new NewCaseOperatorNode(generateExpression(depth + 1), +// generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), +// generateExpression(depth + 1)); +// case LIKE_ESCAPE: +// return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), +// generateExpression(depth + 1), "LIKE", "ESCAPE"); default: + System.out.println(expr.toString()); throw new AssertionError(); } } @@ -132,16 +143,16 @@ public Node generateConstant() { throw new IgnoreMeException(); } return DatabendConstant.createIntConstant(globalState.getRandomly().getInteger()); - case DATE: - if (!globalState.getDbmsSpecificOptions().testDateConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createDateConstant(globalState.getRandomly().getInteger()); - case TIMESTAMP: - if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createTimestampConstant(globalState.getRandomly().getInteger()); +// case DATE: +// if (!globalState.getDbmsSpecificOptions().testDateConstants) { +// throw new IgnoreMeException(); +// } +// return DatabendConstant.createDateConstant(globalState.getRandomly().getInteger()); +// case TIMESTAMP: +// if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { +// throw new IgnoreMeException(); +// } +// return DatabendConstant.createTimestampConstant(globalState.getRandomly().getInteger()); case VARCHAR: if (!globalState.getDbmsSpecificOptions().testStringConstants) { throw new IgnoreMeException(); @@ -190,8 +201,9 @@ public String getTextRepresentation() { } public enum DatabendAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1), STRING_AGG(1), FIRST(1), SUM(1), STDDEV_SAMP(1), STDDEV_POP(1), VAR_POP(1), - VAR_SAMP(1), COVAR_POP(1), COVAR_SAMP(1); + MAX(1), MIN(1), AVG(1), COUNT(1),SUM(1), STDDEV_POP(1), + COVAR_POP(1), COVAR_SAMP(2); + //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) private int nrArgs; @@ -210,7 +222,7 @@ public int getNrArgs() { } public enum DBFunction { - // trigonometric functions + // trigonometric functions, nrArgs表示参数个数 ACOS(1), // ASIN(1), // ATAN(1), // @@ -218,7 +230,7 @@ public enum DBFunction { SIN(1), // TAN(1), // COT(1), // - ATAN2(1), // + ATAN2(2), // // math functions ABS(1), // CEIL(1), // @@ -265,8 +277,9 @@ public enum DBFunction { // REPEAT(2), REPLACE(3), UNICODE(1), - BIT_COUNT(1), BIT_LENGTH(1), LAST_DAY(1), MONTHNAME(1), DAYNAME(1), YEARWEEK(1), DAYOFMONTH(1), WEEKDAY(1), + BIT_COUNT(1), BIT_LENGTH(1), MONTHNAME(1), DAYNAME(1), YEARWEEK(1), DAYOFMONTH(1), WEEKDAY(1), WEEKOFYEAR(1), + //, LAST_DAY(1) IFNULL(2), IF(3); @@ -338,7 +351,8 @@ public static DatabendCollate getRandom() { public enum DatabendUnaryPrefixOperator implements Operator { - NOT("NOT"), PLUS("+"), MINUS("-"); + NOT("NOT"); + //PLUS("+"), MINUS("-"); //等待boolean支持数学运算 private String textRepr; @@ -373,7 +387,7 @@ public static Operator getRandom() { } public enum DatabendBinaryArithmeticOperator implements Operator { - CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); + ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"); private String textRepr; @@ -393,9 +407,8 @@ public String getTextRepresentation() { } public enum DatabendBinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), - LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), - REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); + //LIKE("LIKE"), NOT_LIKE("NOT LIKE"); private String textRepr; diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index fcc6dbd7e..eebb9ffe3 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -25,7 +25,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); - // TODO: distinct + // TODO distinct // select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); List> columns = new ArrayList<>(); diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index cd0c15b7f..0dd75375a 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -36,38 +36,41 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType()); - if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() - && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { - sb.append(" COLLATE "); - sb.append(getRandomCollate()); - } - if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" UNIQUE"); - } +// if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() +// && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { +// sb.append(" COLLATE "); +// sb.append(getRandomCollate()); +// } +// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { +// sb.append(" UNIQUE"); +// } if (globalState.getDbmsSpecificOptions().testNotNullConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); + } else { + sb.append(" NULL"); //Databend 默认字段为非空，这个将它默认设置为允许空 } - if (globalState.getDbmsSpecificOptions().testCheckConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" CHECK("); - sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); - DatabendErrors.addExpressionErrors(errors); - sb.append(")"); - } +// if (globalState.getDbmsSpecificOptions().testCheckConstraints //databend 无check约束 +// && Randomly.getBooleanWithRatherLowProbability()) { +// sb.append(" CHECK("); +// sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); +// DatabendErrors.addExpressionErrors(errors); +// sb.append(")"); +// } if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DatabendToStringVisitor.asString(gen.generateConstant())); sb.append(")"); } } - if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { - errors.add("Invalid type for index"); - List primaryKeyColumns = Randomly.nonEmptySubset(columns); - sb.append(", PRIMARY KEY("); - sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - } + //databend并没有索引 +// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { +// errors.add("Invalid type for index"); +// List primaryKeyColumns = Randomly.nonEmptySubset(columns); +// sb.append(", PRIMARY KEY("); +// sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); +// sb.append(")"); +// } sb.append(")"); return new SQLQueryAdapter(sb.toString(), errors, true); } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 21453005f..858f411a0 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -41,15 +41,15 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { @Override public void check() throws SQLException { - DatabendTables randomTables = s.getRandomTableNonEmptyTables(); + DatabendTables randomTables = s.getRandomTableNonEmptyTables(); //随机获得nr张表 List columns = randomTables.getColumns(); DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); + Node randomWhereCondition = gen.generateExpression(); //生成随机where条件，形式为ast List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); List> joins = DatabendJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); + int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); //禁用优化 int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { @@ -71,7 +71,7 @@ private int getSecondQuery(List> tableList, Node(randomWhereCondition, " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); - select.setFetchColumns(Arrays.asList(asText)); + select.setFetchColumns(Arrays.asList(asText)); // ? select.setFromList(tableList); // select.setSelectType(SelectType.ALL); select.setJoinList(joins); From abd932a2154f574af413fe6531c81bd6f3924328 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:01:19 +0800 Subject: [PATCH 0587/1284] feat: implement the binary comparison operation --- .../DatabendBinaryComparisonOperation.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java diff --git a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java new file mode 100644 index 000000000..f76579ca3 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java @@ -0,0 +1,37 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; + + +public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode { + + public DatabendBinaryComparisonOperation(Node left, Node right, + DatabendBinaryComparisonOperator op) { + super(left, right, op); + } + + public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Operator{ + EQUALS("="), +// IS_DISTINCT("IS DISTINCT FROM"), +// IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), + NOT_EQUALS("!="), + LESS("<"), + LESS_EQUALS("<="), + GREATER(">"), + GREATER_EQUALS(">="); + + DatabendBinaryComparisonOperator(String text) { + textRepresentation = text; + } + private String textRepresentation; + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + + +} From c3da03ec3722325f52179164bdc64a1e2ad89457 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:02:28 +0800 Subject: [PATCH 0588/1284] feat: implement the binary logical operation --- .../ast/DatabendBinaryLogicalOperation.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java diff --git a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java new file mode 100644 index 000000000..32c8df0ca --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java @@ -0,0 +1,76 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; + +public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode { + + public DatabendBinaryLogicalOperation(Node left, Node right, + DatabendBinaryLogicalOperator op) { + super(left,right,op); + } + + public enum DatabendBinaryLogicalOperator implements BinaryOperatorNode.Operator { + AND("AND", "and") { +// @Override +// public DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right) { +// if (left.isNull() && right.isNull()) { +// return DatabendNoRECConstant.createNullConstant(); +// } else if (left.isNull()) { +// if (right.asBooleanNotNull()) { +// return DatabendNoRECConstant.createNullConstant(); +// } else { +// return DatabendNoRECConstant.createFalse(); +// } +// } else if (right.isNull()) { +// if (left.asBooleanNotNull()) { +// return DatabendNoRECConstant.createNullConstant(); +// } else { +// return DatabendNoRECConstant.createFalse(); +// } +// } else { +// return left.asBooleanNotNull() && right.asBooleanNotNull() ? DatabendNoRECConstant.createTrue() +// : DatabendNoRECConstant.createFalse(); +// } +// } + }, + OR("OR", "or") { +// @Override +// public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { +// if (!left.isNull() && left.asBooleanNotNull()) { +// return ClickHouseConstant.createTrue(); +// } else if (!right.isNull() && right.asBooleanNotNull()) { +// return ClickHouseConstant.createTrue(); +// } else if (left.isNull() || right.isNull()) { +// return ClickHouseConstant.createNullConstant(); +// } else { +// return ClickHouseConstant.createFalse(); +// } +// } + }; + + private final String[] textRepresentations; + + DatabendBinaryLogicalOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + + public DatabendBinaryLogicalOperator getRandomOp() { + return Randomly.fromOptions(values()); + } + +// public abstract DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right); + + public static DatabendBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + +} From a717c5df34969d59d6c8d15e3011dd8103a94f31 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:03:30 +0800 Subject: [PATCH 0589/1284] feat: implement the like operation --- .../databend/ast/DatabendLikeOperation.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendLikeOperation.java diff --git a/src/sqlancer/databend/ast/DatabendLikeOperation.java b/src/sqlancer/databend/ast/DatabendLikeOperation.java new file mode 100644 index 000000000..e76219b1a --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendLikeOperation.java @@ -0,0 +1,31 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; + +public class DatabendLikeOperation extends NewBinaryOperatorNode { + + public DatabendLikeOperation(Node left, Node right + , DatabendLikeOperator op) { + super(left, right, op); + } + + + public enum DatabendLikeOperator implements BinaryOperatorNode.Operator { + LIKE_OPERATOR("LIKE","like"); + + private final String[] textRepresentations; + + DatabendLikeOperator(String... text){ + textRepresentations = text; + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + } + +} From 9b3aac126ac18a26fab3aaaecd8dd121728a10a0 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:04:07 +0800 Subject: [PATCH 0590/1284] feat: implement the unary postfix operation --- .../ast/DatabendUnaryPostfixOperation.java | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java new file mode 100644 index 000000000..e20a6be6b --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -0,0 +1,77 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendSchema.DatabendDataType; + + +public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { + +// private final Node expr; +// private final DatabendUnaryPostfixOperator op; + private boolean negate; + + public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op, boolean negate) { + super(expr,op); + setNegate(negate); + } + + public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL"){ + @Override + public DatabendDataType[] getInputDataTypes() { + return DatabendDataType.values(); + } + }, + IS_NOT_NULL("IS NOT NULL"){ + @Override + public DatabendDataType[] getInputDataTypes() { + return DatabendDataType.values(); + } + }; + //IS + + private final String textRepresentations; + + DatabendUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static DatabendUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract DatabendDataType[] getInputDataTypes(); + + } + + public boolean isNegated() { + return negate; + } + + public void setNegate(boolean negate) { + this.negate = negate; + } + +// @Override + public Node getExpression() { + return getExpr(); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + +// @Override +// public OperatorKind getOperatorKind() { +// return OperatorKind.POSTFIX; +// } +} From e2d52ff4a06fed7af3ea868e9675d65477d6d98f Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:05:18 +0800 Subject: [PATCH 0591/1284] feat: implement the unary prefix operation --- .../ast/DatabendUnaryPrefixOperation.java | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java new file mode 100644 index 000000000..eabb4eabd --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -0,0 +1,77 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendSchema.DatabendDataType; + + +public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { + +// private final Node expr; +// private final DatabendUnaryPrefixOperator op; + private boolean negate; + + public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op, boolean negate) { + super(expr,op); + setNegate(negate); + } + + void setNegate(boolean negate){ + this.negate = negate; + } + +// @Override + public Node getExpression() { + return getExpr(); + } + + @Override + public String getOperatorRepresentation() { + return null; + } + +// @Override +// public OperatorKind getOperatorKind() { +// return OperatorKind.PREFIX; +// } + + + + public enum DatabendUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", DatabendDataType.BOOLEAN, DatabendDataType.INT) { + @Override + public DatabendDataType getExpressionType() { + return DatabendDataType.BOOLEAN; + } + + @Override + protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { + return null; // TODO + } + }; + + + private String textRepresentation; + private DatabendDataType[] dataTypes; + + DatabendUnaryPrefixOperator(String textRepresentation, DatabendDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract DatabendDataType getExpressionType(); + + public DatabendDataType[] getInputDataTypes(){ + return dataTypes; + } + + protected abstract DatabendConstant getExpectedValue(DatabendConstant expectedValue); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + } + +} From ccf79d8cd6f38e2c83cbd1166357d6d8c048d0dd Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:08:20 +0800 Subject: [PATCH 0592/1284] feat: refactored untypde expression generator to typed expression generator --- .../gen/DatabendNoRECExpressionGenerator.java | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java new file mode 100644 index 000000000..4c65ccf8b --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -0,0 +1,213 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.ast.*; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; +import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; +import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendNoRECExpressionGenerator extends + TypedExpressionGenerator, DatabendColumn, DatabendDataType> { + + private final DatabendGlobalState globalState; + + public DatabendNoRECExpressionGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; +// SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; + } + + public Node generateLeafNode(DatabendDataType dataType) { + return generateConstant(dataType); + } + + + @Override + protected Node generateExpression(DatabendDataType type, int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } + + switch (type) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + case FLOAT: + case VARCHAR: + case NULL: + return generateConstant(type); + default: + throw new AssertionError(); + } + } + + private Node generateIntExpression(int depth) { + + return null; + } + + Node generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + getPostfix(depth + 1); + case NOT: + getNOT(depth + 1); + case BETWEEN: + return getBetween(depth + 1); + case IN_OPERATION: + return getIn(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return getBinaryLogical(depth + 1,DatabendDataType.BOOLEAN); + case BINARY_COMPARISON: + return getComparison(depth + 1); + case LIKE: + return getLike(depth + 1,DatabendDataType.VARCHAR); + default: + throw new AssertionError(); + } + + } + + Node getPostfix(int depth) { + DatabendUnaryPostfixOperator randomOp = DatabendUnaryPostfixOperator.getRandom(); +// System.out.println("getPostfix:" + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth)); + return new DatabendUnaryPostfixOperation( + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), + randomOp,Randomly.getBoolean()); + } + + Node getNOT(int depth) { + DatabendUnaryPrefixOperator op = DatabendUnaryPrefixOperator.NOT; + return new DatabendUnaryPrefixOperation( + generateExpression(Randomly.fromOptions(op.getInputDataTypes()), depth), + op,Randomly.getBoolean()); + } + + Node getBetween(int depth){ + //跳过boolean + DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() + .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); + + return new NewBetweenOperatorNode(generateExpression(dataType,depth), + generateExpression(dataType,depth), generateExpression(dataType,depth), + Randomly.getBoolean()); + } + + Node getIn(int depth) { + DatabendDataType dataType = Randomly.fromOptions(DatabendDataType.values()); + Node leftExpr = generateExpression(dataType,depth); + List> rightExprs = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + for(int i = 0; i < nr; i++) { + rightExprs.add(generateExpression(dataType,depth)); + } + return new NewInOperatorNode(leftExpr,rightExprs, Randomly.getBoolean()); + } + + Node getBinaryLogical(int depth, DatabendDataType dataType){ + Node expr = generateExpression(dataType,depth); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + expr = new DatabendBinaryLogicalOperation(expr, + generateExpression(DatabendDataType.BOOLEAN, depth), + DatabendBinaryLogicalOperator.getRandom()); + } + return expr; + } + + Node getComparison(int depth) { + //跳过boolean + DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() + .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); + Node leftExpr = generateExpression(dataType,depth); + Node rightExpr = generateExpression(dataType,depth); + DatabendBinaryComparisonOperation op = new DatabendBinaryComparisonOperation(leftExpr,rightExpr, + Randomly.fromOptions(DatabendBinaryComparisonOperator.values())); + return op; + } + + Node getLike(int depth, DatabendDataType dataType) { + return new DatabendLikeOperation(generateExpression(dataType,depth) + ,generateExpression(dataType,depth), DatabendLikeOperation.DatabendLikeOperator.LIKE_OPERATOR); + } + + @Override + public Node generatePredicate() { + return null; + } + + @Override + public Node negatePredicate(Node predicate) { + return null; + } + + @Override + public Node isNull(Node expr) { + return null; + } + + @Override + public Node generateConstant(DatabendDataType type) { + Randomly r = globalState.getRandomly(); + if (Randomly.getBooleanWithSmallProbability()) { + return DatabendConstant.createNullConstant(); + } + + switch (type) { + case INT: + if (Randomly.getBooleanWithSmallProbability()) { + return DatabendConstant.createStringConstant(String.valueOf(r.getInteger())); + } else { + return DatabendConstant.createIntConstant(r.getInteger()); + } + case BOOLEAN: + if (Randomly.getBooleanWithSmallProbability()) { + return DatabendConstant + .createStringConstant(Randomly.fromOptions("TRUE", "FALSE")); //TODO databend中非0为false，0为false + } else { + return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); + } + case FLOAT: + return DatabendConstant.createFloatConstant((float) r.getDouble()); + case VARCHAR: + return DatabendConstant.createStringConstant(r.getString()); + case NULL: + return DatabendConstant.createNullConstant(); + default: + throw new AssertionError(type); + } + } + + @Override + protected Node generateColumn(DatabendDataType type) { + return null; + } + + @Override + protected DatabendDataType getRandomType() { + return Randomly.fromOptions(DatabendDataType.values()); + } + + @Override + protected boolean canGenerateColumnOfType(DatabendDataType type) { + return false; + } +} From fb86d9bd7670c10dfb3be3311ef2084aac26914c Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:12:07 +0800 Subject: [PATCH 0593/1284] fix: replace the expression generator --- src/sqlancer/databend/ast/DatabendJoin.java | 20 +++++++++++-------- .../gen/DatabendRandomQuerySynthesizer.java | 16 +++++++++------ .../databend/test/DatabendNoRECOracle.java | 14 +++++++++---- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index ddc72ed53..886f3824c 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -3,10 +3,11 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; import java.util.ArrayList; import java.util.List; @@ -28,7 +29,7 @@ public static JoinType getRandom() { } public enum OuterType { - FULL, LEFT, RIGHT; + LEFT, RIGHT; public static OuterType getRandom() { return Randomly.fromOptions(values()); @@ -76,21 +77,24 @@ public static List> getJoins( TableReferenceNode rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); - DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); +// DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); + DatabendNoRECExpressionGenerator joinGen = new DatabendNoRECExpressionGenerator(globalState).setColumns(columns); + switch (JoinType.getRandom()) { case INNER: - joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case NATURAL: joinExpressions.add(DatabendJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); break; case LEFT: - joinExpressions - .add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + joinExpressions.add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case RIGHT: - joinExpressions - .add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + joinExpressions.add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; default: throw new AssertionError(); diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index eebb9ffe3..a0b4c087b 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -3,6 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -22,7 +23,9 @@ private DatabendRandomQuerySynthesizer() { public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); - DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) +// DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) +// .setColumns(targetTables.getColumns()); + DatabendNoRECExpressionGenerator gen = new DatabendNoRECExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); // TODO distinct @@ -31,7 +34,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int List> columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = gen.generateExpression(); + Node expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); columns.add(expression); // } else { // columns.add(gen()); @@ -45,7 +48,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } if (Randomly.getBoolean()) { select.setOrderByExpressions(gen.generateOrderBys()); @@ -61,9 +64,10 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setOffsetClause( DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } - if (Randomly.getBoolean()) { - select.setHavingClause(gen.generateHavingClause()); - } + // TODO 待添加HavingClause +// if (Randomly.getBoolean()) { +// select.setHavingClause(gen.generateHavingClause()); +// } return select; } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 858f411a0..d514164ae 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -21,6 +21,7 @@ import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.*; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; +import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; import java.sql.ResultSet; import java.sql.SQLException; @@ -43,8 +44,13 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { public void check() throws SQLException { DatabendTables randomTables = s.getRandomTableNonEmptyTables(); //随机获得nr张表 List columns = randomTables.getColumns(); - DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); //生成随机where条件，形式为ast +// DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); + DatabendNoRECExpressionGenerator gen = new DatabendNoRECExpressionGenerator(state).setColumns(columns); + + Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); //生成随机where条件，形式为ast + +// System.out.println(DatabendToStringVisitor.asString(randomWhereCondition)); + List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -71,13 +77,14 @@ private int getSecondQuery(List> tableList, Node(randomWhereCondition, " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); // ? select.setFromList(tableList); - // select.setSelectType(SelectType.ALL); select.setJoinList(joins); int secondCount = 0; unoptimizedQueryString = "SELECT SUM(count) FROM (" + DatabendToStringVisitor.asString(select) + ") as res"; errors.add("canceling statement due to statement timeout"); +// System.out.println(unoptimizedQueryString); SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { @@ -111,7 +118,6 @@ private int getFirstQueryCount(SQLConnection con, List> if (Randomly.getBooleanWithSmallProbability()) { select.setOrderByExpressions(new DatabendExpressionGenerator(state).setColumns(columns).generateOrderBys()); } - // select.setSelectType(SelectType.ALL); select.setJoinList(joins); int firstCount = 0; try (Statement stat = con.createStatement()) { From 9f260f3f04dcb6a8ac5f79c7ccec8d01aee2fb51 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:28:46 +0800 Subject: [PATCH 0594/1284] fix: update double constant --- src/sqlancer/databend/ast/DatabendConstant.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 57ab52dba..7b0d09164 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -59,11 +59,7 @@ public String toString() { } -// DecimalFormat doubleFormat = new DecimalFormat(); -// doubleFormat.setMaximumFractionDigits(2); //小数点后2位 -// System.out.println(String.valueOf(doubleFormat.format(value))); - //TODO value先返回0.3来取消指数形式表达，等待Databend支持指数形式表达 - return String.valueOf(0.3); + return String.valueOf(value).toLowerCase(); // TODO 待databend支持大写E再去掉小写转换 } } From 71a27693b802860bb8c9ff8fb78c15201d039928 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 16:22:39 +0800 Subject: [PATCH 0595/1284] fix: insert null on not null column --- src/sqlancer/databend/gen/DatabendInsertGenerator.java | 10 ++++++---- .../databend/gen/DatabendNoRECExpressionGenerator.java | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index a61e56263..1e61d5a8c 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -5,6 +5,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -41,15 +42,16 @@ private SQLQueryAdapter generate() { } @Override - protected void insertValue(DatabendColumn tiDBColumn) { - // TODO: 等Databend实现NULL 和 DEFAULT ,暂时注入普通的value + protected void insertValue(DatabendColumn column) { + // TODO: 等Databend实现DEFAULT ,暂时注入普通的value // if (Randomly.getBooleanWithRatherLowProbability()) { // sb.append("DEFAULT"); // } else { // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); // } - - String value = DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant()); + String value = DatabendToStringVisitor.asString( + new DatabendNoRECExpressionGenerator(globalState). + generateConstant(column.isNullable())); sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java index 4c65ccf8b..53e2d526e 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -1,5 +1,6 @@ package sqlancer.databend.gen; +import com.google.common.collect.RangeMap; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.common.ast.newast.NewInOperatorNode; @@ -164,6 +165,15 @@ public Node isNull(Node expr) { return null; } + public Node generateConstant(boolean isNullable) { + DatabendDataType type; + do { + type = Randomly.fromOptions(DatabendDataType.values()); + } while(!isNullable && type.equals(DatabendDataType.NULL)); + + return generateConstant(type); + } + @Override public Node generateConstant(DatabendDataType type) { Randomly r = globalState.getRandomly(); From 55f8d81fab5f0633122d702189a1bea52b1eb397 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Aug 2022 09:12:36 +0000 Subject: [PATCH 0596/1284] Bump postgresql from 42.4.1 to 42.4.2 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.4.1 to 42.4.2. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.4.1...REL42.4.2) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f87671fd8..86e589805 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.4.1 + 42.4.2 org.xerial From 81c37555941a42aeb9fa4fa3fa943721571342a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 09:13:43 +0000 Subject: [PATCH 0597/1284] Bump maven-jxr-plugin from 3.2.0 to 3.3.0 Bumps [maven-jxr-plugin](https://github.com/apache/maven-jxr) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/apache/maven-jxr/releases) - [Commits](https://github.com/apache/maven-jxr/compare/jxr-3.2.0...jxr-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jxr-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 86e589805..28a057022 100644 --- a/pom.xml +++ b/pom.xml @@ -341,7 +341,7 @@ org.apache.maven.plugins maven-jxr-plugin - 3.2.0 + 3.3.0 From d4138a9b32eea3dd7544cafb7d56d6361a6c6283 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Tue, 23 Aug 2022 09:49:25 +0800 Subject: [PATCH 0598/1284] fix: remove int and boolean constants of type string --- .../gen/DatabendNoRECExpressionGenerator.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java index 53e2d526e..c24a9b246 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -183,18 +183,11 @@ public Node generateConstant(DatabendDataType type) { switch (type) { case INT: - if (Randomly.getBooleanWithSmallProbability()) { - return DatabendConstant.createStringConstant(String.valueOf(r.getInteger())); - } else { - return DatabendConstant.createIntConstant(r.getInteger()); - } + //不支持string转化故直接返回int constant + return DatabendConstant.createIntConstant(r.getInteger()); case BOOLEAN: - if (Randomly.getBooleanWithSmallProbability()) { - return DatabendConstant - .createStringConstant(Randomly.fromOptions("TRUE", "FALSE")); //TODO databend中非0为false，0为false - } else { - return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); - } + //不支持string转化故直接返回boolean constant + return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); case FLOAT: return DatabendConstant.createFloatConstant((float) r.getDouble()); case VARCHAR: From 282f712c2a2948a096ff313ea62da0e61d4ed451 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Aug 2022 02:28:17 +0000 Subject: [PATCH 0599/1284] Bump maven-javadoc-plugin from 3.4.0 to 3.4.1 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.4.0 to 3.4.1. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.4.0...maven-javadoc-plugin-3.4.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28a057022..8d4ce21f1 100644 --- a/pom.xml +++ b/pom.xml @@ -397,7 +397,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.4.0 + 3.4.1 8 From 091fc3c1f7007fba53c7fc46d34aa7faf8cd9561 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Thu, 25 Aug 2022 01:02:52 +0800 Subject: [PATCH 0600/1284] fix: insert appropriate constants --- src/sqlancer/databend/DatabendProvider.java | 6 ++++++ .../databend/gen/DatabendInsertGenerator.java | 4 ++-- .../gen/DatabendNoRECExpressionGenerator.java | 16 ++++++++-------- .../databend/gen/DatabendTableGenerator.java | 9 ++++++--- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 241bb136a..dd54a56db 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -143,6 +143,12 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE s.execute("USE " + databaseName); globalState.getState().logStatement("USE " + databaseName); } + +// try (Statement s = con.createStatement()) { +// s.execute("set enable_planner_v2 = 0;"); +// globalState.getState().logStatement("set enable_planner_v2 = 0;"); +// } + return new SQLConnection(con); } diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 1e61d5a8c..d05f65c14 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -43,7 +43,7 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(DatabendColumn column) { - // TODO: 等Databend实现DEFAULT ,暂时注入普通的value + // TODO: 等Databend实现DEFAULT关键字,暂时注入普通的value // if (Randomly.getBooleanWithRatherLowProbability()) { // sb.append("DEFAULT"); // } else { @@ -51,7 +51,7 @@ protected void insertValue(DatabendColumn column) { // } String value = DatabendToStringVisitor.asString( new DatabendNoRECExpressionGenerator(globalState). - generateConstant(column.isNullable())); + generateConstant(column.getType().getPrimitiveDataType())); //生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java index c24a9b246..391790804 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -165,14 +165,14 @@ public Node isNull(Node expr) { return null; } - public Node generateConstant(boolean isNullable) { - DatabendDataType type; - do { - type = Randomly.fromOptions(DatabendDataType.values()); - } while(!isNullable && type.equals(DatabendDataType.NULL)); - - return generateConstant(type); - } +// public Node generateConstant(boolean isNullable) { //TODO 极小概率生成NULL值 +// DatabendDataType type; +// do { +// type = Randomly.fromOptions(DatabendDataType.values()); +// } while(!isNullable && type.equals(DatabendDataType.NULL)); //isNullable为假，同时获得NULL就重新获取 +// +// return generateConstant(type); +// } @Override public Node generateConstant(DatabendDataType type) { diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 0dd75375a..6e9eeb57c 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -2,6 +2,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -12,6 +13,7 @@ import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.test.DatabendNoRECOracle; import java.util.ArrayList; import java.util.List; @@ -27,8 +29,8 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(tableName); sb.append("("); List columns = getNewColumns(); - UntypedExpressionGenerator, DatabendColumn> gen = new DatabendExpressionGenerator( - globalState).setColumns(columns); + TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = + new DatabendNoRECExpressionGenerator(globalState).setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); @@ -59,7 +61,8 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { // } if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); - sb.append(DatabendToStringVisitor.asString(gen.generateConstant())); + sb.append(DatabendToStringVisitor.asString( //常量类型于字段类型等同 + gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); sb.append(")"); } } From c2c82b944d4e11d960035a1c8b7d6ae6deb04233 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Thu, 25 Aug 2022 01:16:38 +0800 Subject: [PATCH 0601/1284] fix: extreme value representation of a double constant --- src/sqlancer/databend/ast/DatabendConstant.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 7b0d09164..56a4661da 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -53,13 +53,12 @@ public double getValue() { @Override public String toString() { if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; + return "3.40282347e+38"; } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; + return "-3.40282347e+38"; } - - return String.valueOf(value).toLowerCase(); // TODO 待databend支持大写E再去掉小写转换 + return String.valueOf(value); } } From 18df9540f4f96eb23a3367b7144f37a343596c0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Aug 2022 09:09:29 +0000 Subject: [PATCH 0602/1284] Bump postgresql from 42.4.2 to 42.5.0 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.4.2 to 42.5.0. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.4.2...REL42.5.0) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8d4ce21f1..a6d7d16db 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.4.2 + 42.5.0 org.xerial From 7b76fec60ccc43dc8c4eb5c1c3ddc7430993061d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Aug 2022 03:14:00 +0000 Subject: [PATCH 0603/1284] Bump maven-checkstyle-plugin from 3.1.2 to 3.2.0 Bumps [maven-checkstyle-plugin](https://github.com/apache/maven-checkstyle-plugin) from 3.1.2 to 3.2.0. - [Release notes](https://github.com/apache/maven-checkstyle-plugin/releases) - [Commits](https://github.com/apache/maven-checkstyle-plugin/compare/maven-checkstyle-plugin-3.1.2...maven-checkstyle-plugin-3.2.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-checkstyle-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a6d7d16db..2cd50e527 100644 --- a/pom.xml +++ b/pom.xml @@ -202,7 +202,7 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.1.2 + 3.2.0 com.puppycrawl.tools From b55486b29c2021f59e6274c078b75cbf9294b9c0 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sat, 27 Aug 2022 16:26:31 +0800 Subject: [PATCH 0604/1284] fix: add the corresponding type name to the original column name --- src/sqlancer/databend/gen/DatabendTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 6e9eeb57c..3fc403157 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -85,8 +85,8 @@ public static String getRandomCollate() { private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - String columnName = String.format("c%d", i); DatabendCompositeDataType columnType = DatabendCompositeDataType.getRandomWithoutNull(); + String columnName = String.format("c%d%s", i,columnType.getPrimitiveDataType().toString()); columns.add(new DatabendColumn(columnName, columnType, false, false)); } return columns; From 8b14856965a9fadaa194926b004be7c72fc8cf79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 09:15:33 +0000 Subject: [PATCH 0605/1284] Bump checkstyle from 10.3.2 to 10.3.3 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3.2 to 10.3.3. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3.2...checkstyle-10.3.3) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2cd50e527..db142a0d6 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.3.2 + 10.3.3 From c8d0e63e3f87e872c2287f035eec1f1889ef2d94 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Mon, 29 Aug 2022 18:08:50 +0800 Subject: [PATCH 0606/1284] fix: implement where tester of tlp --- .../ast/DatabendUnaryPostfixOperation.java | 4 ++ .../ast/DatabendUnaryPrefixOperation.java | 11 ++-- .../databend/gen/DatabendIndexGenerator.java | 59 ------------------- .../gen/DatabendNoRECExpressionGenerator.java | 53 ++++++++++++++--- .../test/DatabendQueryPartitioningBase.java | 5 +- ...DatabendQueryPartitioningHavingTester.java | 3 +- 6 files changed, 59 insertions(+), 76 deletions(-) delete mode 100644 src/sqlancer/databend/gen/DatabendIndexGenerator.java diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java index e20a6be6b..f516f88a2 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -18,6 +18,10 @@ public DatabendUnaryPostfixOperation(Node expr, DatabendUnar setNegate(negate); } + public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op) { + super(expr,op); + } + public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator { IS_NULL("IS NULL"){ @Override diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index eabb4eabd..64c9328d8 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -17,6 +17,10 @@ public DatabendUnaryPrefixOperation(Node expr, DatabendUnary setNegate(negate); } + public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op) { + super(expr,op); + } + void setNegate(boolean negate){ this.negate = negate; } @@ -26,18 +30,11 @@ public Node getExpression() { return getExpr(); } - @Override - public String getOperatorRepresentation() { - return null; - } - // @Override // public OperatorKind getOperatorKind() { // return OperatorKind.PREFIX; // } - - public enum DatabendUnaryPrefixOperator implements BinaryOperatorNode.Operator { NOT("NOT", DatabendDataType.BOOLEAN, DatabendDataType.INT) { @Override diff --git a/src/sqlancer/databend/gen/DatabendIndexGenerator.java b/src/sqlancer/databend/gen/DatabendIndexGenerator.java deleted file mode 100644 index de6820bcc..000000000 --- a/src/sqlancer/databend/gen/DatabendIndexGenerator.java +++ /dev/null @@ -1,59 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendTable; - -import java.util.List; - -public final class DatabendIndexGenerator { - - private DatabendIndexGenerator() { - } - - public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { - errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); - sb.append("UNIQUE "); - } - sb.append("INDEX "); - sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information - sb.append(" ON "); - DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); - sb.append("("); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(Randomly.fromOptions("ASC", "DESC")); - } - } - sb.append(")"); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - Node expr = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()) - .generateExpression(); - sb.append(DatabendToStringVisitor.asString(expr)); - } - errors.add("already exists!"); - if (globalState.getDbmsSpecificOptions().testRowid) { - errors.add("Cannot create an index on the rowid!"); - } - return new SQLQueryAdapter(sb.toString(), errors, true); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java index 391790804..79939a1bf 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -1,8 +1,8 @@ package sqlancer.databend.gen; -import com.google.common.collect.RangeMap; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.NewInOperatorNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; @@ -14,7 +14,6 @@ import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -24,6 +23,7 @@ public class DatabendNoRECExpressionGenerator extends TypedExpressionGenerator, DatabendColumn, DatabendDataType> { private final DatabendGlobalState globalState; + private boolean allowAggregateFunctions; public DatabendNoRECExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; @@ -89,7 +89,6 @@ Node generateBooleanExpression(int depth) { Node getPostfix(int depth) { DatabendUnaryPostfixOperator randomOp = DatabendUnaryPostfixOperator.getRandom(); -// System.out.println("getPostfix:" + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth)); return new DatabendUnaryPostfixOperation( generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp,Randomly.getBoolean()); @@ -152,17 +151,17 @@ Node getLike(int depth, DatabendDataType dataType) { @Override public Node generatePredicate() { - return null; + return generateExpression(DatabendDataType.BOOLEAN); } @Override public Node negatePredicate(Node predicate) { - return null; + return new DatabendUnaryPrefixOperation(predicate,DatabendUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return null; + public Node isNull(Node predicate) { + return new DatabendUnaryPostfixOperation(predicate,DatabendUnaryPostfixOperator.IS_NULL); } // public Node generateConstant(boolean isNullable) { //TODO 极小概率生成NULL值 @@ -213,4 +212,44 @@ protected DatabendDataType getRandomType() { protected boolean canGenerateColumnOfType(DatabendDataType type) { return false; } + + public enum DatabendAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1),SUM(1), STDDEV_POP(1), + COVAR_POP(1), COVAR_SAMP(2); + //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) + + private int nrArgs; + + DatabendAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static DatabendAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } + + public NewFunctionNode generateArgsForAggregate( + DatabendAggregateFunction aggregateFunction) { + return new NewFunctionNode( + generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + } + + public Node generateAggregate() { + DatabendAggregateFunction aggrFunc = DatabendAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + public Node generateHavingClause() { + this.allowAggregateFunctions = true; + Node expression = generateExpression(DatabendDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java index a806dd2ca..afa8b2202 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -17,6 +17,7 @@ import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; import java.sql.SQLException; import java.util.ArrayList; @@ -29,7 +30,7 @@ public class DatabendQueryPartitioningBase DatabendSchema s; DatabendTables targetTables; - DatabendExpressionGenerator gen; + DatabendNoRECExpressionGenerator gen; DatabendSelect select; public DatabendQueryPartitioningBase(DatabendGlobalState state) { @@ -50,7 +51,7 @@ public static String canonicalizeResultValue(String value) { public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); - gen = new DatabendExpressionGenerator(state).setColumns(targetTables.getColumns()); + gen = new DatabendNoRECExpressionGenerator(state).setColumns(targetTables.getColumns()); initializeTernaryPredicateVariants(); select = new DatabendSelect(); select.setFetchColumns(generateFetchColumns()); diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java index 019eeace7..9ee9715cd 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -5,6 +5,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendProvider.DatabendGlobalState; @@ -25,7 +26,7 @@ public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } boolean orderBy = Randomly.getBoolean(); if (orderBy) { From 6b14210374413336d9c796462dfd98e843716d35 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 4 Sep 2022 22:53:55 +0800 Subject: [PATCH 0607/1284] fix: implement all tester of tlp Signed-off-by: Yisong Han --- .../databend/DatabendToStringVisitor.java | 10 ++++++ src/sqlancer/databend/ast/DatabendJoin.java | 4 +-- .../databend/gen/DatabendInsertGenerator.java | 4 +-- ...va => DatabendNewExpressionGenerator.java} | 36 +++++++++++++++---- .../gen/DatabendRandomQuerySynthesizer.java | 2 +- .../databend/gen/DatabendTableGenerator.java | 6 +--- .../databend/test/DatabendNoRECOracle.java | 4 +-- ...abendQueryPartitioningAggregateTester.java | 24 +++++++------ .../test/DatabendQueryPartitioningBase.java | 21 ++++++----- ...DatabendQueryPartitioningHavingTester.java | 16 ++++++--- 10 files changed, 86 insertions(+), 41 deletions(-) rename src/sqlancer/databend/gen/{DatabendNoRECExpressionGenerator.java => DatabendNewExpressionGenerator.java} (88%) diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java index 50a37aeec..628d0502a 100644 --- a/src/sqlancer/databend/DatabendToStringVisitor.java +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -2,6 +2,7 @@ import sqlancer.common.ast.newast.NewToStringVisitor; import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -42,6 +43,15 @@ private void visit(DatabendConstant constant) { sb.append(constant.toString()); } +// private void visitFromList(List> fromList) { +// for (int i = 0; i < fromList.size(); i++) { +// if (i != 0) { +// sb.append(" INNER JOIN "); +// } +// visit(fromList.get(i)); +// } +// } + private void visit(DatabendSelect select) { sb.append("SELECT "); if (select.isDistinct()) { diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index 886f3824c..8797e1508 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -7,7 +7,7 @@ import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; import java.util.ArrayList; import java.util.List; @@ -78,7 +78,7 @@ public static List> getJoins( List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); // DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); - DatabendNoRECExpressionGenerator joinGen = new DatabendNoRECExpressionGenerator(globalState).setColumns(columns); + DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); switch (JoinType.getRandom()) { case INNER: diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index d05f65c14..f0f767d8f 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.databend.gen; -import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -50,7 +48,7 @@ protected void insertValue(DatabendColumn column) { // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); // } String value = DatabendToStringVisitor.asString( - new DatabendNoRECExpressionGenerator(globalState). + new DatabendNewExpressionGenerator(globalState). generateConstant(column.getType().getPrimitiveDataType())); //生成一个与column相同的常量类型 sb.append(value); diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java similarity index 88% rename from src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java rename to src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 79939a1bf..67d35b06b 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -19,13 +19,13 @@ import java.util.List; import java.util.stream.Collectors; -public class DatabendNoRECExpressionGenerator extends +public class DatabendNewExpressionGenerator extends TypedExpressionGenerator, DatabendColumn, DatabendDataType> { private final DatabendGlobalState globalState; private boolean allowAggregateFunctions; - public DatabendNoRECExpressionGenerator(DatabendGlobalState globalState) { + public DatabendNewExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; } @@ -58,6 +58,14 @@ protected Node generateExpression(DatabendDataType type, int } } + public List> generateExpressions(int nr, DatabendDataType type) { + List> expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(type)); + } + return expressions; + } + private Node generateIntExpression(int depth) { return null; @@ -71,7 +79,7 @@ Node generateBooleanExpression(int depth) { getPostfix(depth + 1); case NOT: getNOT(depth + 1); - case BETWEEN: + case BETWEEN: //TODO (NULL BETWEEN NULL AND NULL) 返回的是 NULL 需要注意 return getBetween(depth + 1); case IN_OPERATION: return getIn(depth + 1); @@ -214,20 +222,35 @@ protected boolean canGenerateColumnOfType(DatabendDataType type) { } public enum DatabendAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1),SUM(1), STDDEV_POP(1), + MAX(1), + MIN(1), + AVG(1,DatabendDataType.INT,DatabendDataType.FLOAT), + COUNT(1), + SUM(1,DatabendDataType.INT,DatabendDataType.FLOAT), + STDDEV_POP(1), COVAR_POP(1), COVAR_SAMP(2); //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) private int nrArgs; + private DatabendDataType[] dataTypes; - DatabendAggregateFunction(int nrArgs) { + DatabendAggregateFunction(int nrArgs, DatabendDataType ...dataTypes) { this.nrArgs = nrArgs; + this.dataTypes = dataTypes; } public static DatabendAggregateFunction getRandom() { return Randomly.fromOptions(values()); } + public DatabendDataType getRandomType() { + if(dataTypes.length == 0) { + return Randomly.fromOptions(DatabendDataType.values()); + } else { + return Randomly.fromOptions(dataTypes); + } + } + public int getNrArgs() { return nrArgs; } @@ -237,7 +260,8 @@ public int getNrArgs() { public NewFunctionNode generateArgsForAggregate( DatabendAggregateFunction aggregateFunction) { return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + generateExpressions(aggregateFunction.getNrArgs(),aggregateFunction.getRandomType()), + aggregateFunction); } public Node generateAggregate() { diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index a0b4c087b..9ac3499b3 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -25,7 +25,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) // .setColumns(targetTables.getColumns()); - DatabendNoRECExpressionGenerator gen = new DatabendNoRECExpressionGenerator(globalState) + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); // TODO distinct diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 3fc403157..ba08f8acc 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -3,21 +3,17 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; -import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; -import sqlancer.databend.test.DatabendNoRECOracle; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class DatabendTableGenerator { @@ -30,7 +26,7 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append("("); List columns = getNewColumns(); TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = - new DatabendNoRECExpressionGenerator(globalState).setColumns(columns); + new DatabendNewExpressionGenerator(globalState).setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index d514164ae..451940a6e 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -21,7 +21,7 @@ import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.*; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; -import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; import java.sql.ResultSet; import java.sql.SQLException; @@ -45,7 +45,7 @@ public void check() throws SQLException { DatabendTables randomTables = s.getRandomTableNonEmptyTables(); //随机获得nr张表 List columns = randomTables.getColumns(); // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); - DatabendNoRECExpressionGenerator gen = new DatabendNoRECExpressionGenerator(state).setColumns(columns); + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); //生成随机where条件，形式为ast diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index e584da545..9ce498234 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -23,12 +23,11 @@ import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; -import sqlancer.databend.gen.DatabendExpressionGenerator; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendAggregateFunction; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.gen.DatabendNewExpressionGenerator.DatabendAggregateFunction; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendBinaryArithmeticOperator; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPrefixOperator; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPostfixOperator; import sqlancer.databend.DatabendProvider.DatabendGlobalState; public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { @@ -56,9 +55,9 @@ public void check() throws SQLException { fetchColumns.add(gen.generateAggregate()); } select.setFetchColumns(Arrays.asList(aggregate)); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); - } +// if (Randomly.getBooleanWithRatherLowProbability()) { +// select.setOrderByExpressions(gen.generateOrderBys()); +// } originalQuery = DatabendToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); @@ -80,7 +79,7 @@ public void check() throws SQLException { private String createMetamorphicUnionQuery(DatabendSelect select, NewFunctionNode aggregate, List> from) { String metamorphicQuery; - Node whereClause = gen.generateExpression(); + Node whereClause = gen.generateExpression(DatabendDataType.BOOLEAN); Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, DatabendUnaryPrefixOperator.NOT); Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, @@ -98,7 +97,7 @@ private String createMetamorphicUnionQuery(DatabendSelect select, } private String getAggregateResult(String queryString) throws SQLException { - String resultString; + String resultString = null; SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { @@ -107,7 +106,11 @@ private String getAggregateResult(String queryString) throws SQLException { if (!result.next()) { resultString = null; } else { - resultString = result.getString(1); + try { + resultString = result.getString(1); + } catch (Exception e) { + System.out.println("Invalid integer format for value"); //TODO 超过integer范围无法格式化异常，还未有解决方案 + } } return resultString; } catch (SQLException e) { @@ -180,7 +183,8 @@ private DatabendSelect getSelect(List> aggregates, List leftSelect.setWhereClause(whereClause); leftSelect.setJoinList(joinList); if (Randomly.getBooleanWithSmallProbability()) { - leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); +// leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + leftSelect.setGroupByExpressions(select.getFetchColumns()); } return leftSelect; } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java index afa8b2202..c0d090649 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -12,12 +12,11 @@ import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.gen.DatabendExpressionGenerator; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; -import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; import java.sql.SQLException; import java.util.ArrayList; @@ -30,7 +29,7 @@ public class DatabendQueryPartitioningBase DatabendSchema s; DatabendTables targetTables; - DatabendNoRECExpressionGenerator gen; + DatabendNewExpressionGenerator gen; DatabendSelect select; public DatabendQueryPartitioningBase(DatabendGlobalState state) { @@ -51,10 +50,10 @@ public static String canonicalizeResultValue(String value) { public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); - gen = new DatabendNoRECExpressionGenerator(state).setColumns(targetTables.getColumns()); + gen = new DatabendNewExpressionGenerator(state).setColumns(targetTables.getColumns()); initializeTernaryPredicateVariants(); select = new DatabendSelect(); - select.setFetchColumns(generateFetchColumns()); + select.setFetchColumns(generateRandomColumns()); List tables = targetTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -66,15 +65,21 @@ public void check() throws SQLException { List> generateFetchColumns() { List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { //TODO 为什么会返回 false 或 true 字段 columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + columns = generateRandomColumns(); } return columns; } + List> generateRandomColumns() { + List> columns; + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + return columns; + } + @Override protected ExpressionGenerator> getGen() { return gen; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java index 9ee9715cd..18c17b452 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -7,6 +7,7 @@ import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendProvider.DatabendGlobalState; @@ -14,6 +15,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Random; public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase implements TestOracle { @@ -28,13 +30,19 @@ public void check() throws SQLException { if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - boolean orderBy = Randomly.getBoolean(); - if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); +// boolean orderBy = Randomly.getBoolean(); + boolean orderBy = false; //关闭order by + if (orderBy) { //TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select +// select.setOrderByExpressions(gen.generateOrderBys()); + List> constants = new ArrayList<>(); + constants.add(new DatabendConstant.DatabendIntConstant(new Random().nextInt(select.getFetchColumns().size()) + 1)); + select.setOrderByExpressions(constants); } - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); +// select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); +// System.out.println(originalQueryString); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setHavingClause(predicate); From 7bd59dc3c9e749f14c11b5cb7cdcf05fe831c3d8 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Mon, 5 Sep 2022 18:17:21 +0800 Subject: [PATCH 0608/1284] feat(databend): implement binary arithmetic operation Signed-off-by: Yisong Han --- .../DatabendBinaryArithmeticOperation.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java new file mode 100644 index 000000000..3c4caabb9 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -0,0 +1,31 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; + +public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode { + + public DatabendBinaryArithmeticOperation(Node left, Node right, BinaryOperatorNode.Operator op) { + super(left, right, op); + } + + public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Operator{ + ADDITION("+"), + SUBTRACTION("-"), + MULTIPLICATION("*"), + DIVISION("/"), + MODULO("%"); + + DatabendBinaryArithmeticOperator(String text) { + textRepresentation = text; + } + private String textRepresentation; + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + +} From 66a439283298e5a4ea5529ae10a7bfdddaa9c73c Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Mon, 5 Sep 2022 18:17:59 +0800 Subject: [PATCH 0609/1284] feat(databend): add int expression generator Signed-off-by: Yisong Han --- .../ast/DatabendUnaryPrefixOperation.java | 23 +++++++++ .../databend/gen/DatabendInsertGenerator.java | 2 +- .../gen/DatabendNewExpressionGenerator.java | 51 ++++++++++++------- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index 64c9328d8..b86bebe6b 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -46,6 +46,29 @@ public DatabendDataType getExpressionType() { protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { return null; // TODO } + }, + + UNARY_PLUS("+",DatabendDataType.INT) { + @Override + public DatabendDataType getExpressionType() { + return DatabendDataType.INT; + } + + @Override + protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { + return expectedValue; + } + }, + UNARY_MINUS("-", DatabendDataType.INT) { + @Override + public DatabendDataType getExpressionType() { + return DatabendDataType.INT; + } + + @Override + protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { + return null; + } }; diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index f0f767d8f..ade2840e4 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -49,7 +49,7 @@ protected void insertValue(DatabendColumn column) { // } String value = DatabendToStringVisitor.asString( new DatabendNewExpressionGenerator(globalState). - generateConstant(column.getType().getPrimitiveDataType())); //生成一个与column相同的常量类型 + generateConstant(column.getType().getPrimitiveDataType(),column.isNullable())); //生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 67d35b06b..9c953cb5e 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -14,6 +14,7 @@ import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -29,11 +30,6 @@ public DatabendNewExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; } - private enum BooleanExpression { - POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; -// SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; - } - public Node generateLeafNode(DatabendDataType dataType) { return generateConstant(dataType); } @@ -49,6 +45,7 @@ protected Node generateExpression(DatabendDataType type, int case BOOLEAN: return generateBooleanExpression(depth); case INT: + return generateIntExpression(depth); case FLOAT: case VARCHAR: case NULL: @@ -66,9 +63,28 @@ public List> generateExpressions(int nr, DatabendDataTy return expressions; } + private enum IntExpression{ + UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION + } + private Node generateIntExpression(int depth) { + IntExpression intExpression = Randomly.fromOptions(IntExpression.values()); + switch (intExpression) { + case UNARY_OPERATION: + return new DatabendUnaryPrefixOperation(generateExpression(DatabendDataType.INT,depth+1), + Randomly.getBoolean()? DatabendUnaryPrefixOperator.UNARY_PLUS : DatabendUnaryPrefixOperator.UNARY_MINUS); + case BINARY_ARITHMETIC_OPERATION: + return new DatabendBinaryArithmeticOperation(generateExpression(DatabendDataType.INT,depth+1), + generateExpression(DatabendDataType.INT,depth+1), + Randomly.fromOptions(DatabendBinaryArithmeticOperator.values())); + default: + throw new AssertionError(); + } + } - return null; + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; +// SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; } Node generateBooleanExpression(int depth) { @@ -172,28 +188,29 @@ public Node isNull(Node predicate) { return new DatabendUnaryPostfixOperation(predicate,DatabendUnaryPostfixOperator.IS_NULL); } -// public Node generateConstant(boolean isNullable) { //TODO 极小概率生成NULL值 -// DatabendDataType type; -// do { -// type = Randomly.fromOptions(DatabendDataType.values()); -// } while(!isNullable && type.equals(DatabendDataType.NULL)); //isNullable为假，同时获得NULL就重新获取 -// -// return generateConstant(type); -// } + public Node generateConstant(DatabendDataType type,boolean isNullable) { + if(isNullable && Randomly.getBooleanWithSmallProbability()) { + createConstant(DatabendDataType.NULL); + } + return createConstant(type); + } @Override public Node generateConstant(DatabendDataType type) { - Randomly r = globalState.getRandomly(); if (Randomly.getBooleanWithSmallProbability()) { return DatabendConstant.createNullConstant(); } + return createConstant(type); + } + public Node createConstant(DatabendDataType type) { + Randomly r = globalState.getRandomly(); switch (type) { case INT: - //不支持string转化故直接返回int constant + //TODO 已支持数值型string转化，待添加 return DatabendConstant.createIntConstant(r.getInteger()); case BOOLEAN: - //不支持string转化故直接返回boolean constant + //TODO 已支持boolean型string转化，待添加 return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); case FLOAT: return DatabendConstant.createFloatConstant((float) r.getDouble()); From 5f07a2dde8134e2937c83228bf8c312359cb953f Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sat, 10 Sep 2022 00:09:46 +0800 Subject: [PATCH 0610/1284] fix(databend): mvn verify succeed Signed-off-by: Yisong Han --- dependency-reduced-pom.xml | 433 +++++++++++++++++ src/sqlancer/GlobalState.java | 8 +- src/sqlancer/databend/DatabendOptions.java | 18 +- src/sqlancer/databend/DatabendProvider.java | 62 ++- src/sqlancer/databend/DatabendSchema.java | 90 ++-- .../databend/DatabendToStringVisitor.java | 17 +- .../DatabendBinaryArithmeticOperation.java | 14 +- .../DatabendBinaryComparisonOperation.java | 19 +- .../ast/DatabendBinaryLogicalOperation.java | 72 +-- .../databend/ast/DatabendCastOperation.java | 19 + .../databend/ast/DatabendConstant.java | 4 +- src/sqlancer/databend/ast/DatabendJoin.java | 29 +- .../databend/ast/DatabendLikeOperation.java | 11 +- .../ast/DatabendUnaryPostfixOperation.java | 28 +- .../ast/DatabendUnaryPrefixOperation.java | 44 +- .../gen/DatabendAlterTableGenerator.java | 70 --- .../databend/gen/DatabendDeleteGenerator.java | 30 -- .../gen/DatabendExpressionGenerator.java | 451 ------------------ .../databend/gen/DatabendInsertGenerator.java | 26 +- .../gen/DatabendNewExpressionGenerator.java | 215 +++++---- .../gen/DatabendRandomQuerySynthesizer.java | 27 +- .../databend/gen/DatabendTableGenerator.java | 66 +-- .../databend/gen/DatabendUpdateGenerator.java | 48 -- .../databend/gen/DatabendViewGenerator.java | 6 +- .../databend/test/DatabendNoRECOracle.java | 50 +- ...abendQueryPartitioningAggregateTester.java | 42 +- .../test/DatabendQueryPartitioningBase.java | 26 +- ...tabendQueryPartitioningDistinctTester.java | 10 +- ...atabendQueryPartitioningGroupByTester.java | 14 +- ...DatabendQueryPartitioningHavingTester.java | 31 +- .../DatabendQueryPartitioningWhereTester.java | 20 +- .../databend/TestDatabendConnection.java | 10 +- test/sqlancer/dbms/TestClickHouse.java | 2 +- test/sqlancer/dbms/TestDuckDB.java | 2 +- test/sqlancer/dbms/TestMySQLTLP.java | 2 +- 35 files changed, 953 insertions(+), 1063 deletions(-) create mode 100644 dependency-reduced-pom.xml create mode 100644 src/sqlancer/databend/ast/DatabendCastOperation.java delete mode 100644 src/sqlancer/databend/gen/DatabendAlterTableGenerator.java delete mode 100644 src/sqlancer/databend/gen/DatabendDeleteGenerator.java delete mode 100644 src/sqlancer/databend/gen/DatabendExpressionGenerator.java delete mode 100644 src/sqlancer/databend/gen/DatabendUpdateGenerator.java diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml new file mode 100644 index 000000000..357833ccb --- /dev/null +++ b/dependency-reduced-pom.xml @@ -0,0 +1,433 @@ + + + 4.0.0 + com.sqlancer + sqlancer + SQLancer + 2.0.0 + SQLancer finds logic bugs in Database Management Systems through automatic testing + http://www.sqlancer.com/ + + + mrigger + Manuel Rigger + manuel.rigger@inf.ethz.ch + https://www.manuelrigger.at/ + ETH Zurich + https://ethz.ch/ + + architect + developer + + + + + + MIT License + https://github.com/sqlancer/sqlancer/blob/master/LICENSE.md + repo + + + + scm:git:git://github.com/sqlancer/sqlancer.git + scm:git:ssh://github.com:sqlancer/sqlancer.git + https://github.com/sqlancer/sqlancer/ + + + src + test + + + maven-shade-plugin + 3.3.0 + + + package + + shade + + + + + com.beust:jcommander + + + + + + + + maven-antrun-plugin + 3.1.0 + + + package + + run + + + + + + + + + + + + + maven-surefire-plugin + 2.22.2 + + + org.jacoco + jacoco-maven-plugin + 0.8.8 + + + + prepare-agent + + + + report + test + + report + + + + + + maven-compiler-plugin + 3.10.1 + + + org.codehaus.plexus + plexus-compiler-eclipse + 2.12.0 + + + org.eclipse.jdt + ecj + 3.28.0 + + + org.codehaus.plexus + plexus-compiler-api + 2.12.0 + + + + 11 + 11 + + ${project.basedir}/.settings/org.eclipse.jdt.core.prefs + + eclipse + true + true + + + + net.revelc.code.formatter + formatter-maven-plugin + 2.19.0 + + + eclipseformat + verify + + validate + + + + + + maven-dependency-plugin + 3.2.0 + + + copy-dependencies + prepare-package + + copy-dependencies + + + ${project.build.directory}/lib + false + false + true + jcommander + + + + + + maven-jar-plugin + 3.2.2 + + true + + + true + lib/ + sqlancer.Main + + + + + + org.codehaus.mojo + xml-maven-plugin + 1.0.2 + + + xml-check-format + verify + + check-format + + + + + + maven-checkstyle-plugin + 3.1.2 + + + checkstyle + verify + + check + + + true + + + + + + com.puppycrawl.tools + checkstyle + 10.3 + + + + configs/checkstyle.xml + + + + maven-pmd-plugin + 3.14.0 + + + pmd + verify + + check + + + 2 + true + + configs/pmd-rules.xml + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + 4.7.0.0 + + + spotbugs + verify + + check + + + configs/spotbugs-exclude.xml + High + + + + + + + + + jdk-8-config + + ${java.home}/../bin/javadoc + + + + jdk-11-config + + ${java.home}/bin/javadoc + + + + release-steps + + + + maven-source-plugin + 3.2.1 + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + 3.4.0 + + + attach-javadocs + + jar + + + + + 8 + + + + maven-gpg-plugin + 3.0.1 + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + + + + + + com.google.auto.service + auto-service + 1.0.1 + compile + + + org.postgresql + postgresql + 42.4.0 + compile + + + org.xerial + sqlite-jdbc + 3.36.0.3 + compile + + + mysql + mysql-connector-java + 8.0.29 + compile + + + org.mariadb.jdbc + mariadb-java-client + 3.0.5 + compile + + + org.duckdb + duckdb_jdbc + 0.4.0 + compile + + + org.junit.jupiter + junit-jupiter-engine + 5.8.2 + test + + + junit-platform-engine + org.junit.platform + + + junit-jupiter-api + org.junit.jupiter + + + apiguardian-api + org.apiguardian + + + + + org.slf4j + slf4j-simple + 1.7.36 + compile + + + ru.yandex.clickhouse + clickhouse-jdbc + 0.3.2 + compile + + + com.h2database + h2 + 2.1.214 + compile + + + org.mongodb + mongodb-driver-sync + 4.1.1 + compile + + + com.arangodb + arangodb-java-driver + 6.9.0 + compile + + + + + + maven-jxr-plugin + 3.2.0 + + + + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + UTF-8 + + diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 20efcdcc8..9c48dd900 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,12 +1,12 @@ package sqlancer; +import java.sql.SQLException; + import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; -import java.sql.SQLException; - public abstract class GlobalState, S extends AbstractSchema, C extends SQLancerDBConnection> { protected C databaseConnection; @@ -144,10 +144,10 @@ protected void setSchema(S schema) { } public void updateSchema() throws Exception { - try{ + try { setSchema(readSchema()); } catch (SQLException sqlException) { - System.out.println(String.format("%s readSchema SQLException",databaseName)); + System.out.printf("%s readSchema SQLException%n", databaseName); sqlException.printStackTrace(); } for (AbstractTable table : schema.getDatabaseTables()) { diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index de6940a76..ebacd21cf 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -1,19 +1,25 @@ package sqlancer.databend; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; + import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.databend.test.*; import sqlancer.databend.DatabendOptions.DatabendOracleFactory; -import sqlancer.databend.test.DatabendQueryPartitioningAggregateTester; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import sqlancer.databend.test.DatabendNoRECOracle; +import sqlancer.databend.test.DatabendQueryPartitioningAggregateTester; +import sqlancer.databend.test.DatabendQueryPartitioningDistinctTester; +import sqlancer.databend.test.DatabendQueryPartitioningGroupByTester; +import sqlancer.databend.test.DatabendQueryPartitioningHavingTester; +import sqlancer.databend.test.DatabendQueryPartitioningWhereTester; @Parameters(commandDescription = "Databend") public class DatabendOptions implements DBMSSpecificOptions { diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index dd54a56db..793650798 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -1,19 +1,30 @@ package sqlancer.databend; -import com.google.auto.service.AutoService; -import sqlancer.*; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLQueryProvider; -import sqlancer.databend.gen.*; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; - import java.io.File; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.gen.DatabendInsertGenerator; +import sqlancer.databend.gen.DatabendRandomQuerySynthesizer; +import sqlancer.databend.gen.DatabendTableGenerator; + @AutoService(DatabaseProvider.class) public class DatabendProvider extends SQLProviderAdapter { @@ -24,10 +35,11 @@ public DatabendProvider() { public enum Action implements AbstractAction { INSERT(DatabendInsertGenerator::getQuery), // - //TODO 等待databend实现update && delete -// DELETE(DatabendDeleteGenerator::generate), // -// UPDATE(DatabendUpdateGenerator::getQuery), // -// CREATE_VIEW(DatabendViewGenerator::generate), // TODO 等待databend的create view语法 更加贴近mysql + // TODO 等待databend实现update && delete + // DELETE(DatabendDeleteGenerator::generate), // + // UPDATE(DatabendUpdateGenerator::getQuery), // + + // CREATE_VIEW(DatabendViewGenerator::generate), //TODO 等待databend的create view语法 更加贴近mysql EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); @@ -57,13 +69,13 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case EXPLAIN: return r.getInteger(0, 2); - //TODO 等待databend实现update && delete -// case UPDATE: -// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); -// case DELETE: -// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); -// case CREATE_VIEW: -// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + // TODO 等待databend实现update && delete + // case UPDATE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + // case DELETE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + // case CREATE_VIEW: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); } @@ -96,7 +108,7 @@ public void generateDatabase(DatabendGlobalState globalState) throws Exception { throw new IgnoreMeException(); } }); - se.executeStatements(); //在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 + se.executeStatements(); // 在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 } public void tryDeleteFile(String fname) { @@ -144,17 +156,17 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE globalState.getState().logStatement("USE " + databaseName); } -// try (Statement s = con.createStatement()) { -// s.execute("set enable_planner_v2 = 0;"); -// globalState.getState().logStatement("set enable_planner_v2 = 0;"); -// } + // try (Statement s = con.createStatement()) { + // s.execute("set enable_planner_v2 = 0;"); + // globalState.getState().logStatement("set enable_planner_v2 = 0;"); + // } return new SQLConnection(con); } @Override public String getDBMSName() { - return "databend"; //用于DatabendOptions + return "databend"; // 用于DatabendOptions } } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index ad92932d2..643f53dd9 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -1,13 +1,5 @@ package sqlancer.databend; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.DBMSCommon; -import sqlancer.common.schema.*; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendTable; - import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -15,12 +7,24 @@ import java.util.Collections; import java.util.List; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendTable; + public class DatabendSchema extends AbstractSchema { public enum DatabendDataType { INT, VARCHAR, BOOLEAN, FLOAT, NULL; - //, DATE, TIMESTAMP + // , DATE, TIMESTAMP public static DatabendDataType getRandomWithoutNull() { DatabendDataType dt; @@ -74,8 +78,8 @@ public static DatabendCompositeDataType getRandomWithoutNull() { break; case BOOLEAN: case VARCHAR: -// case DATE: -// case TIMESTAMP: + // case DATE: + // case TIMESTAMP: size = 0; break; default: @@ -114,10 +118,10 @@ public String toString() { } case BOOLEAN: return Randomly.fromOptions("BOOLEAN", "BOOL"); -// case TIMESTAMP: -// return Randomly.fromOptions("TIMESTAMP", "DATETIME"); -// case DATE: -// return Randomly.fromOptions("DATE"); + // case TIMESTAMP: + // return Randomly.fromOptions("TIMESTAMP", "DATETIME"); + // case DATE: + // return Randomly.fromOptions("DATE"); case NULL: return Randomly.fromOptions("NULL"); default: @@ -132,7 +136,8 @@ public static class DatabendColumn extends AbstractTableColumn columns, boolean isV public static DatabendSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); - List tableNames = getTableNames(con,databaseName); + List tableNames = getTableNames(con, databaseName); for (String tableName : tableNames) { if (DBMSCommon.matchesIndexName(tableName)) { continue; // TODO: unexpected? } - List databaseColumns = getTableColumns(con, tableName,databaseName); + List databaseColumns = getTableColumns(con, tableName, databaseName); boolean isView = tableName.startsWith("v"); DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); for (DatabendColumn c : databaseColumns) { @@ -250,18 +255,20 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { List tableNames = null; tableNames = new ArrayList<>(); - //SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema != 'system' and table_schema != 'INFORMATION_SCHEMA' and table_type='BASE TABLE' - //"SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName + // SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema != 'system' and table_schema != + // 'INFORMATION_SCHEMA' and table_type='BASE TABLE' + // "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName final String sqlStatement = String.format( - "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName); + "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ", + databaseName); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(sqlStatement)) { - try{ //没有catch的话rs.next()会报SQLException：Not a navigable ResultSet + try { // 没有catch的话rs.next()会报SQLException：Not a navigable ResultSet while (rs.next()) { tableNames.add(rs.getString("table_name")); } - } catch (Exception e){ -// e.printStackTrace(); + } catch (Exception e) { + // e.printStackTrace(); System.out.println("TableNames->SQLException：Not a navigable ResultSet"); } } @@ -269,20 +276,22 @@ private static List getTableNames(SQLConnection con, String databaseName return tableNames; } - private static List getTableColumns(SQLConnection con, String tableName, String databaseName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) + throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format( "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = '%s' and table_name ='%s'", - databaseName,tableName))) { - try{ //没有catch的话rs.next()会报SQLException：Not a navigable ResultSet + databaseName, tableName))) { + try { // 没有catch的话rs.next()会报SQLException：Not a navigable ResultSet while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); boolean isNullable = rs.getBoolean("is_nullable"); -// boolean isPrimaryKey = rs.getString("pk").contains("true"); - boolean isPrimaryKey = false; //没找到主键元数据 - DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); + // boolean isPrimaryKey = rs.getString("pk").contains("true"); + boolean isPrimaryKey = false; // 没找到主键元数据 + DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, + isNullable); columns.add(c); } } catch (Exception e) { @@ -290,10 +299,11 @@ private static List getTableColumns(SQLConnection con, String ta } } } - if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { - // TODO: implement an option to enable/disable rowids -// columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, false)); - } + // if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { + // TODO: implement an option to enable/disable rowids + // columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, + // false)); + // } return columns; } diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java index 628d0502a..b3669e4b7 100644 --- a/src/sqlancer/databend/DatabendToStringVisitor.java +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -2,7 +2,6 @@ import sqlancer.common.ast.newast.NewToStringVisitor; import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -43,14 +42,14 @@ private void visit(DatabendConstant constant) { sb.append(constant.toString()); } -// private void visitFromList(List> fromList) { -// for (int i = 0; i < fromList.size(); i++) { -// if (i != 0) { -// sb.append(" INNER JOIN "); -// } -// visit(fromList.get(i)); -// } -// } + // private void visitFromList(List> fromList) { + // for (int i = 0; i < fromList.size(); i++) { + // if (i != 0) { + // sb.append(" INNER JOIN "); + // } + // visit(fromList.get(i)); + // } + // } private void visit(DatabendSelect select) { sb.append("SELECT "); diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java index 3c4caabb9..dfc44ee22 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -6,21 +6,19 @@ public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode { - public DatabendBinaryArithmeticOperation(Node left, Node right, BinaryOperatorNode.Operator op) { + public DatabendBinaryArithmeticOperation(Node left, Node right, + BinaryOperatorNode.Operator op) { super(left, right, op); } - public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Operator{ - ADDITION("+"), - SUBTRACTION("-"), - MULTIPLICATION("*"), - DIVISION("/"), - MODULO("%"); + public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + ADDITION("+"), SUBTRACTION("-"), MULTIPLICATION("*"), DIVISION("/"), MODULO("%"); + + private final String textRepresentation; DatabendBinaryArithmeticOperator(String text) { textRepresentation = text; } - private String textRepresentation; @Override public String getTextRepresentation() { diff --git a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java index f76579ca3..9e8e43a51 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java @@ -4,28 +4,24 @@ import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.Node; - public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode { public DatabendBinaryComparisonOperation(Node left, Node right, - DatabendBinaryComparisonOperator op) { + DatabendBinaryComparisonOperator op) { super(left, right, op); } - public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Operator{ + public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Operator { EQUALS("="), -// IS_DISTINCT("IS DISTINCT FROM"), -// IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), - NOT_EQUALS("!="), - LESS("<"), - LESS_EQUALS("<="), - GREATER(">"), - GREATER_EQUALS(">="); + // IS_DISTINCT("IS DISTINCT FROM"), + // IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), + NOT_EQUALS("!="), LESS("<"), LESS_EQUALS("<="), GREATER(">"), GREATER_EQUALS(">="); + + private final String textRepresentation; DatabendBinaryComparisonOperator(String text) { textRepresentation = text; } - private String textRepresentation; @Override public String getTextRepresentation() { @@ -33,5 +29,4 @@ public String getTextRepresentation() { } } - } diff --git a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java index 32c8df0ca..14ff984e6 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java @@ -8,47 +8,47 @@ public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode { public DatabendBinaryLogicalOperation(Node left, Node right, - DatabendBinaryLogicalOperator op) { - super(left,right,op); + DatabendBinaryLogicalOperator op) { + super(left, right, op); } public enum DatabendBinaryLogicalOperator implements BinaryOperatorNode.Operator { AND("AND", "and") { -// @Override -// public DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right) { -// if (left.isNull() && right.isNull()) { -// return DatabendNoRECConstant.createNullConstant(); -// } else if (left.isNull()) { -// if (right.asBooleanNotNull()) { -// return DatabendNoRECConstant.createNullConstant(); -// } else { -// return DatabendNoRECConstant.createFalse(); -// } -// } else if (right.isNull()) { -// if (left.asBooleanNotNull()) { -// return DatabendNoRECConstant.createNullConstant(); -// } else { -// return DatabendNoRECConstant.createFalse(); -// } -// } else { -// return left.asBooleanNotNull() && right.asBooleanNotNull() ? DatabendNoRECConstant.createTrue() -// : DatabendNoRECConstant.createFalse(); -// } -// } + // @Override + // public DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right) { + // if (left.isNull() && right.isNull()) { + // return DatabendNoRECConstant.createNullConstant(); + // } else if (left.isNull()) { + // if (right.asBooleanNotNull()) { + // return DatabendNoRECConstant.createNullConstant(); + // } else { + // return DatabendNoRECConstant.createFalse(); + // } + // } else if (right.isNull()) { + // if (left.asBooleanNotNull()) { + // return DatabendNoRECConstant.createNullConstant(); + // } else { + // return DatabendNoRECConstant.createFalse(); + // } + // } else { + // return left.asBooleanNotNull() && right.asBooleanNotNull() ? DatabendNoRECConstant.createTrue() + // : DatabendNoRECConstant.createFalse(); + // } + // } }, OR("OR", "or") { -// @Override -// public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { -// if (!left.isNull() && left.asBooleanNotNull()) { -// return ClickHouseConstant.createTrue(); -// } else if (!right.isNull() && right.asBooleanNotNull()) { -// return ClickHouseConstant.createTrue(); -// } else if (left.isNull() || right.isNull()) { -// return ClickHouseConstant.createNullConstant(); -// } else { -// return ClickHouseConstant.createFalse(); -// } -// } + // @Override + // public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { + // if (!left.isNull() && left.asBooleanNotNull()) { + // return ClickHouseConstant.createTrue(); + // } else if (!right.isNull() && right.asBooleanNotNull()) { + // return ClickHouseConstant.createTrue(); + // } else if (left.isNull() || right.isNull()) { + // return ClickHouseConstant.createNullConstant(); + // } else { + // return ClickHouseConstant.createFalse(); + // } + // } }; private final String[] textRepresentations; @@ -66,7 +66,7 @@ public DatabendBinaryLogicalOperator getRandomOp() { return Randomly.fromOptions(values()); } -// public abstract DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right); + // public abstract DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right); public static DatabendBinaryLogicalOperator getRandom() { return Randomly.fromOptions(values()); diff --git a/src/sqlancer/databend/ast/DatabendCastOperation.java b/src/sqlancer/databend/ast/DatabendCastOperation.java new file mode 100644 index 000000000..c1b3ffe66 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendCastOperation.java @@ -0,0 +1,19 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendSchema; + +public class DatabendCastOperation extends NewUnaryPostfixOperatorNode { + + public DatabendCastOperation(Node expr, DatabendSchema.DatabendCompositeDataType type) { + super(expr, new BinaryOperatorNode.Operator() { + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 56a4661da..39056aff9 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -1,10 +1,10 @@ package sqlancer.databend.ast; -import sqlancer.common.ast.newast.Node; - import java.sql.Timestamp; import java.text.SimpleDateFormat; +import sqlancer.common.ast.newast.Node; + public class DatabendConstant implements Node { private DatabendConstant() { diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index 8797e1508..8694608cd 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -1,17 +1,17 @@ package sqlancer.databend.ast; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -import java.util.ArrayList; -import java.util.List; - public class DatabendJoin implements Node { private final TableReferenceNode leftTable; @@ -37,8 +37,8 @@ public static OuterType getRandom() { } public DatabendJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; @@ -77,8 +77,9 @@ public static List> getJoins( TableReferenceNode rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); -// DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); - DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + // DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); + DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState) + .setColumns(columns); switch (JoinType.getRandom()) { case INNER: @@ -90,11 +91,11 @@ public static List> getJoins( break; case LEFT: joinExpressions.add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case RIGHT: joinExpressions.add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; default: throw new AssertionError(); @@ -104,22 +105,22 @@ public static List> getJoins( } public static DatabendJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new DatabendJoin(left, right, JoinType.RIGHT, predicate); } public static DatabendJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new DatabendJoin(left, right, JoinType.LEFT, predicate); } public static DatabendJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new DatabendJoin(left, right, JoinType.INNER, predicate); } public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, OuterType naturalJoinType) { + TableReferenceNode right, OuterType naturalJoinType) { DatabendJoin join = new DatabendJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; diff --git a/src/sqlancer/databend/ast/DatabendLikeOperation.java b/src/sqlancer/databend/ast/DatabendLikeOperation.java index e76219b1a..2755970e7 100644 --- a/src/sqlancer/databend/ast/DatabendLikeOperation.java +++ b/src/sqlancer/databend/ast/DatabendLikeOperation.java @@ -7,19 +7,18 @@ public class DatabendLikeOperation extends NewBinaryOperatorNode { - public DatabendLikeOperation(Node left, Node right - , DatabendLikeOperator op) { + public DatabendLikeOperation(Node left, Node right, + DatabendLikeOperator op) { super(left, right, op); } - public enum DatabendLikeOperator implements BinaryOperatorNode.Operator { - LIKE_OPERATOR("LIKE","like"); + LIKE_OPERATOR("LIKE", "like"); private final String[] textRepresentations; - DatabendLikeOperator(String... text){ - textRepresentations = text; + DatabendLikeOperator(String... text) { + textRepresentations = text.clone(); } @Override diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java index f516f88a2..e01c35282 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -6,36 +6,36 @@ import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendSchema.DatabendDataType; - public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { -// private final Node expr; -// private final DatabendUnaryPostfixOperator op; + // private final Node expr; + // private final DatabendUnaryPostfixOperator op; private boolean negate; - public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op, boolean negate) { - super(expr,op); + public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op, + boolean negate) { + super(expr, op); setNegate(negate); } public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op) { - super(expr,op); + super(expr, op); } public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator { - IS_NULL("IS NULL"){ + IS_NULL("IS NULL") { @Override public DatabendDataType[] getInputDataTypes() { return DatabendDataType.values(); } }, - IS_NOT_NULL("IS NOT NULL"){ + IS_NOT_NULL("IS NOT NULL") { @Override public DatabendDataType[] getInputDataTypes() { return DatabendDataType.values(); } }; - //IS + // IS private final String textRepresentations; @@ -64,7 +64,7 @@ public void setNegate(boolean negate) { this.negate = negate; } -// @Override + // @Override public Node getExpression() { return getExpr(); } @@ -74,8 +74,8 @@ public String getOperatorRepresentation() { return this.op.getTextRepresentation(); } -// @Override -// public OperatorKind getOperatorKind() { -// return OperatorKind.POSTFIX; -// } + // @Override + // public OperatorKind getOperatorKind() { + // return OperatorKind.POSTFIX; + // } } diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index b86bebe6b..69c95a745 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -1,39 +1,40 @@ package sqlancer.databend.ast; +import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendSchema.DatabendDataType; - public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { -// private final Node expr; -// private final DatabendUnaryPrefixOperator op; - private boolean negate; - - public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op, boolean negate) { - super(expr,op); - setNegate(negate); - } + // private final Node expr; + // private final DatabendUnaryPrefixOperator op; + // private boolean negate; public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op) { - super(expr,op); + super(expr, op); } - void setNegate(boolean negate){ - this.negate = negate; - } + // public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op, boolean + // negate) { + // super(expr,op); + // setNegate(negate); + // } + + // void setNegate(boolean negate){ + // this.negate = negate; + // } -// @Override + // @Override public Node getExpression() { return getExpr(); } -// @Override -// public OperatorKind getOperatorKind() { -// return OperatorKind.PREFIX; -// } + // @Override + // public OperatorKind getOperatorKind() { + // return OperatorKind.PREFIX; + // } public enum DatabendUnaryPrefixOperator implements BinaryOperatorNode.Operator { NOT("NOT", DatabendDataType.BOOLEAN, DatabendDataType.INT) { @@ -48,7 +49,7 @@ protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { } }, - UNARY_PLUS("+",DatabendDataType.INT) { + UNARY_PLUS("+", DatabendDataType.INT) { @Override public DatabendDataType getExpressionType() { return DatabendDataType.INT; @@ -71,7 +72,6 @@ protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { } }; - private String textRepresentation; private DatabendDataType[] dataTypes; @@ -82,8 +82,8 @@ protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { public abstract DatabendDataType getExpressionType(); - public DatabendDataType[] getInputDataTypes(){ - return dataTypes; + public DatabendDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); } protected abstract DatabendConstant getExpectedValue(DatabendConstant expectedValue); diff --git a/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java b/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java deleted file mode 100644 index 90920c819..000000000 --- a/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java +++ /dev/null @@ -1,70 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; -import sqlancer.databend.DatabendSchema.DatabendTable; - -public final class DatabendAlterTableGenerator { - - private DatabendAlterTableGenerator() { - } - - enum Action { - ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN - } - - public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - errors.add(" does not have a column with name \"rowid\""); - errors.add("Table does not contain column rowid referenced in alter statement"); - StringBuilder sb = new StringBuilder("ALTER TABLE "); - DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()); - sb.append(table.getName()); - sb.append(" "); - Action action = Randomly.fromOptions(Action.values()); - switch (action) { - case ADD_COLUMN: - sb.append("ADD COLUMN "); - String columnName = table.getFreeColumnName(); - sb.append(columnName); - sb.append(" "); - sb.append(DatabendCompositeDataType.getRandomWithoutNull().toString()); - break; - case ALTER_COLUMN: - sb.append("ALTER COLUMN "); - sb.append(table.getRandomColumn().getName()); - sb.append(" SET DATA TYPE "); - sb.append(DatabendCompositeDataType.getRandomWithoutNull().toString()); - if (Randomly.getBoolean()) { - sb.append(" USING "); - DatabendErrors.addExpressionErrors(errors); - sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); - } - errors.add("Cannot change the type of this column: an index depends on it!"); - errors.add("Cannot change the type of a column that has a UNIQUE or PRIMARY KEY constraint specified"); - errors.add("Unimplemented type for cast"); - errors.add("Conversion:"); - errors.add("Cannot change the type of a column that has a CHECK constraint specified"); - break; - case DROP_COLUMN: - sb.append("DROP COLUMN "); - sb.append(table.getRandomColumn().getName()); - errors.add("named in key does not exist"); // TODO - errors.add("Cannot drop this column:"); - errors.add("Cannot drop column: table only has one column remaining!"); - errors.add("because there is a CHECK constraint that depends on it"); - errors.add("because there is a UNIQUE constraint that depends on it"); - break; - default: - throw new AssertionError(action); - } - return new SQLQueryAdapter(sb.toString(), errors, true); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java deleted file mode 100644 index 25dd83f45..000000000 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ /dev/null @@ -1,30 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendTable; - -public final class DatabendDeleteGenerator { - - private DatabendDeleteGenerator() { - } - - public static SQLQueryAdapter generate(DatabendGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); - DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DatabendToStringVisitor.asString( - new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); - } - DatabendErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java deleted file mode 100644 index a1a0fd3d9..000000000 --- a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java +++ /dev/null @@ -1,451 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.*; -import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; -import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.databend.ast.DatabendConstant; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; -import sqlancer.databend.DatabendSchema.DatabendDataType; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public final class DatabendExpressionGenerator extends UntypedExpressionGenerator, DatabendColumn> { - - private final DatabendGlobalState globalState; - - public DatabendExpressionGenerator(DatabendGlobalState globalState) { - this.globalState = globalState; - } - - private enum Expression { - UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL, BETWEEN, IN - // COLLATE, LIKE_ESCAPE, CAST, CASE - //TODO 数学运算时去掉Boolean，或等待databend允许boolean参与数学运算 - //FUNC(数学函数不支持boolean),BINARY_ARITHMETIC(不支持boolean参与二元运算 - - } - - private enum BooleanExpression { - UNARY_POSTFIX, BINARY_COMPARISON,BINARY_LOGICAL - } - - @Override - protected Node generateExpression(int depth) { -// if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { -// return generateLeafNode(); -// } - if (depth >= 1) { - return generateLeafNode(); - } -// if (allowAggregates && Randomly.getBoolean()) { -// DatabendAggregateFunction aggregate = DatabendAggregateFunction.getRandom(); -// allowAggregates = false; -// return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); -// } - List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); -// if (!globalState.getDbmsSpecificOptions().testCollate) { -// possibleOptions.remove(Expression.COLLATE); -// } -// if (!globalState.getDbmsSpecificOptions().testFunctions) { -// possibleOptions.remove(Expression.FUNC); -// } -// if (!globalState.getDbmsSpecificOptions().testCasts) { -// possibleOptions.remove(Expression.CAST); -// } - if (!globalState.getDbmsSpecificOptions().testBetween) { - possibleOptions.remove(Expression.BETWEEN); - } - if (!globalState.getDbmsSpecificOptions().testIn) { - possibleOptions.remove(Expression.IN); - } -// if (!globalState.getDbmsSpecificOptions().testCase) { -// possibleOptions.remove(Expression.CASE); -// } - if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { - possibleOptions.remove(Expression.BINARY_COMPARISON); - } - if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { - possibleOptions.remove(Expression.BINARY_LOGICAL); - } - Expression expr = Randomly.fromList(possibleOptions); - switch (expr) { -// case COLLATE: -// return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), -// DatabendCollate.getRandom()); - case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), - DatabendUnaryPrefixOperator.getRandom()); - case UNARY_POSTFIX: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), - DatabendUnaryPostfixOperator.getRandom()); - case BINARY_COMPARISON: - Operator op = DatabendBinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); - case BINARY_LOGICAL: - op = DatabendBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); -// case BINARY_ARITHMETIC: -// return new NewBinaryOperatorNode(generateExpression(depth + 1), -// generateExpression(depth + 1), DatabendBinaryArithmeticOperator.getRandom()); -// case CAST: -// return new DatabendCastOperation(generateExpression(depth + 1), -// DatabendCompositeDataType.getRandomWithoutNull()); -// case FUNC: -// DBFunction func = DBFunction.getRandom(); -// return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); - case BETWEEN: - return new NewBetweenOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); - case IN: - return new NewInOperatorNode(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); -// case CASE: -// int nr = Randomly.smallNumber() + 1; -// return new NewCaseOperatorNode(generateExpression(depth + 1), -// generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), -// generateExpression(depth + 1)); -// case LIKE_ESCAPE: -// return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), -// generateExpression(depth + 1), "LIKE", "ESCAPE"); - default: - System.out.println(expr.toString()); - throw new AssertionError(); - } - } - - @Override - protected Node generateColumn() { - DatabendColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode(column); - } - - @Override - public Node generateConstant() { - if (Randomly.getBooleanWithSmallProbability()) { - return DatabendConstant.createNullConstant(); - } -// DatabendDataType type = DatabendDataType.getRandomWithoutNull(); - //TODO 先跳过varchar等待databend对其更好的支持，或自己改写case VARCHAR的代码 - DatabendDataType type = DatabendDataType.getRandomWithoutNullAndVarchar(); - switch (type) { - case INT: - if (!globalState.getDbmsSpecificOptions().testIntConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createIntConstant(globalState.getRandomly().getInteger()); -// case DATE: -// if (!globalState.getDbmsSpecificOptions().testDateConstants) { -// throw new IgnoreMeException(); -// } -// return DatabendConstant.createDateConstant(globalState.getRandomly().getInteger()); -// case TIMESTAMP: -// if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { -// throw new IgnoreMeException(); -// } -// return DatabendConstant.createTimestampConstant(globalState.getRandomly().getInteger()); - case VARCHAR: - if (!globalState.getDbmsSpecificOptions().testStringConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createStringConstant(globalState.getRandomly().getString()); - case BOOLEAN: - if (!globalState.getDbmsSpecificOptions().testBooleanConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); - case FLOAT: - if (!globalState.getDbmsSpecificOptions().testFloatConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createFloatConstant(globalState.getRandomly().getDouble()); - default: - throw new AssertionError(); - } - } - - @Override - public List> generateOrderBys() { - List> expr = super.generateOrderBys(); - List> newExpr = new ArrayList<>(expr.size()); - for (Node curExpr : expr) { - if (Randomly.getBoolean()) { - curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); - } - newExpr.add(curExpr); - } - return newExpr; - }; - - public static class DatabendCastOperation extends NewUnaryPostfixOperatorNode { - - public DatabendCastOperation(Node expr, DatabendCompositeDataType type) { - super(expr, new Operator() { - - @Override - public String getTextRepresentation() { - return "::" + type.toString(); - } - }); - } - - } - - public enum DatabendAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1),SUM(1), STDDEV_POP(1), - COVAR_POP(1), COVAR_SAMP(2); - //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) - - private int nrArgs; - - DatabendAggregateFunction(int nrArgs) { - this.nrArgs = nrArgs; - } - - public static DatabendAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - - } - - public enum DBFunction { - // trigonometric functions, nrArgs表示参数个数 - ACOS(1), // - ASIN(1), // - ATAN(1), // - COS(1), // - SIN(1), // - TAN(1), // - COT(1), // - ATAN2(2), // - // math functions - ABS(1), // - CEIL(1), // - CEILING(1), // - FLOOR(1), // - LOG(1), // - LOG10(1), LOG2(1), // - LN(1), // - PI(0), // - SQRT(1), // - POWER(1), // - CBRT(1), // - ROUND(2), // - SIGN(1), // - DEGREES(1), // - RADIANS(1), // - MOD(2), // - XOR(2), // - // string functions - LENGTH(1), // - LOWER(1), // - UPPER(1), // - SUBSTRING(3), // - REVERSE(1), // - CONCAT(1, true), // - CONCAT_WS(1, true), CONTAINS(2), // - PREFIX(2), // - SUFFIX(2), // - INSTR(2), // - PRINTF(1, true), // - REGEXP_MATCHES(2), // - REGEXP_REPLACE(3), // - STRIP_ACCENTS(1), // - - // date functions - DATE_PART(2), AGE(2), - - COALESCE(3), NULLIF(2), - - // LPAD(3), - // RPAD(3), - LTRIM(1), RTRIM(1), - // LEFT(2), https://github.com/cwida/Databend/issues/633 - // REPEAT(2), - REPLACE(3), UNICODE(1), - - BIT_COUNT(1), BIT_LENGTH(1), MONTHNAME(1), DAYNAME(1), YEARWEEK(1), DAYOFMONTH(1), WEEKDAY(1), - WEEKOFYEAR(1), - //, LAST_DAY(1) - - IFNULL(2), IF(3); - - private int nrArgs; - private boolean isVariadic; - - DBFunction(int nrArgs) { - this(nrArgs, false); - } - - DBFunction(int nrArgs, boolean isVariadic) { - this.nrArgs = nrArgs; - this.isVariadic = isVariadic; - } - - public static DBFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - if (isVariadic) { - return Randomly.smallNumber() + nrArgs; - } else { - return nrArgs; - } - } - - } - - public enum DatabendUnaryPostfixOperator implements Operator { - - IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); - - private String textRepr; - - DatabendUnaryPostfixOperator(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - public static DatabendUnaryPostfixOperator getRandom() { - return Randomly.fromOptions(values()); - } - - } - - public static final class DatabendCollate implements Operator { - - private final String textRepr; - - private DatabendCollate(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return "COLLATE " + textRepr; - } - - public static DatabendCollate getRandom() { - return new DatabendCollate(DatabendTableGenerator.getRandomCollate()); - } - - } - - public enum DatabendUnaryPrefixOperator implements Operator { - - NOT("NOT"); - //PLUS("+"), MINUS("-"); //等待boolean支持数学运算 - - private String textRepr; - - DatabendUnaryPrefixOperator(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - public static DatabendUnaryPrefixOperator getRandom() { - return Randomly.fromOptions(values()); - } - - } - - public enum DatabendBinaryLogicalOperator implements Operator { - - AND, OR; - - @Override - public String getTextRepresentation() { - return toString(); - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - } - - public enum DatabendBinaryArithmeticOperator implements Operator { - ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"); - - private String textRepr; - - DatabendBinaryArithmeticOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - public enum DatabendBinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); - //LIKE("LIKE"), NOT_LIKE("NOT LIKE"); - - private String textRepr; - - DatabendBinaryComparisonOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - public NewFunctionNode generateArgsForAggregate( - DatabendAggregateFunction aggregateFunction) { - return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); - } - - public Node generateAggregate() { - DatabendAggregateFunction aggrFunc = DatabendAggregateFunction.getRandom(); - return generateArgsForAggregate(aggrFunc); - } - - @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, DatabendUnaryPrefixOperator.NOT); - } - - @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, DatabendUnaryPostfixOperator.IS_NULL); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index ade2840e4..340e494ef 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -1,16 +1,17 @@ package sqlancer.databend.gen; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; - -import java.util.List; -import java.util.stream.Collectors; +import sqlancer.databend.DatabendToStringVisitor; public class DatabendInsertGenerator extends AbstractInsertGenerator { @@ -31,7 +32,7 @@ private SQLQueryAdapter generate() { List columns = table.getRandomNonEmptyColumnSubset(); sb.append(table.getName()); sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); sb.append(")"); sb.append(" VALUES "); insertColumns(columns); @@ -42,14 +43,13 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(DatabendColumn column) { // TODO: 等Databend实现DEFAULT关键字,暂时注入普通的value -// if (Randomly.getBooleanWithRatherLowProbability()) { -// sb.append("DEFAULT"); -// } else { -// sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); -// } - String value = DatabendToStringVisitor.asString( - new DatabendNewExpressionGenerator(globalState). - generateConstant(column.getType().getPrimitiveDataType(),column.isNullable())); //生成一个与column相同的常量类型 + // if (Randomly.getBooleanWithRatherLowProbability()) { + // sb.append("DEFAULT"); + // } else { + // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); + // } + String value = DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 9c953cb5e..f565a9be7 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -1,5 +1,10 @@ package sqlancer.databend.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.common.ast.newast.NewFunctionNode; @@ -9,19 +14,22 @@ import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; -import sqlancer.databend.ast.*; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; +import sqlancer.databend.ast.DatabendBinaryComparisonOperation; +import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; +import sqlancer.databend.ast.DatabendBinaryLogicalOperation; +import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendLikeOperation; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; -import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; -import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; -import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -public class DatabendNewExpressionGenerator extends - TypedExpressionGenerator, DatabendColumn, DatabendDataType> { +public class DatabendNewExpressionGenerator + extends TypedExpressionGenerator, DatabendColumn, DatabendDataType> { private final DatabendGlobalState globalState; private boolean allowAggregateFunctions; @@ -30,11 +38,11 @@ public DatabendNewExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; } + @Override public Node generateLeafNode(DatabendDataType dataType) { return generateConstant(dataType); } - @Override protected Node generateExpression(DatabendDataType type, int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { @@ -42,16 +50,16 @@ protected Node generateExpression(DatabendDataType type, int } switch (type) { - case BOOLEAN: - return generateBooleanExpression(depth); - case INT: - return generateIntExpression(depth); - case FLOAT: - case VARCHAR: - case NULL: - return generateConstant(type); - default: - throw new AssertionError(); + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case FLOAT: + case VARCHAR: + case NULL: + return generateConstant(type); + default: + throw new AssertionError(); } } @@ -63,50 +71,57 @@ public List> generateExpressions(int nr, DatabendDataTy return expressions; } - private enum IntExpression{ + private enum IntExpression { UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION } private Node generateIntExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } IntExpression intExpression = Randomly.fromOptions(IntExpression.values()); switch (intExpression) { - case UNARY_OPERATION: - return new DatabendUnaryPrefixOperation(generateExpression(DatabendDataType.INT,depth+1), - Randomly.getBoolean()? DatabendUnaryPrefixOperator.UNARY_PLUS : DatabendUnaryPrefixOperator.UNARY_MINUS); - case BINARY_ARITHMETIC_OPERATION: - return new DatabendBinaryArithmeticOperation(generateExpression(DatabendDataType.INT,depth+1), - generateExpression(DatabendDataType.INT,depth+1), - Randomly.fromOptions(DatabendBinaryArithmeticOperator.values())); - default: - throw new AssertionError(); + case UNARY_OPERATION: + return new DatabendUnaryPrefixOperation(generateExpression(DatabendDataType.INT, depth + 1), + Randomly.getBoolean() ? DatabendUnaryPrefixOperator.UNARY_PLUS + : DatabendUnaryPrefixOperator.UNARY_MINUS); + case BINARY_ARITHMETIC_OPERATION: + return new DatabendBinaryArithmeticOperation(generateExpression(DatabendDataType.INT, depth + 1), + generateExpression(DatabendDataType.INT, depth + 1), + Randomly.fromOptions(DatabendBinaryArithmeticOperator.values())); + default: + throw new AssertionError(); } } private enum BooleanExpression { POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; -// SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; + // SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; } Node generateBooleanExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); BooleanExpression option = Randomly.fromList(validOptions); switch (option) { - case POSTFIX_OPERATOR: - getPostfix(depth + 1); - case NOT: - getNOT(depth + 1); - case BETWEEN: //TODO (NULL BETWEEN NULL AND NULL) 返回的是 NULL 需要注意 - return getBetween(depth + 1); - case IN_OPERATION: - return getIn(depth + 1); - case BINARY_LOGICAL_OPERATOR: - return getBinaryLogical(depth + 1,DatabendDataType.BOOLEAN); - case BINARY_COMPARISON: - return getComparison(depth + 1); - case LIKE: - return getLike(depth + 1,DatabendDataType.VARCHAR); - default: - throw new AssertionError(); + case POSTFIX_OPERATOR: + getPostfix(depth + 1); + case NOT: + getNOT(depth + 1); + case BETWEEN: // TODO (NULL BETWEEN NULL AND NULL) 返回的是 NULL 需要注意 + return getBetween(depth + 1); + case IN_OPERATION: + return getIn(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return getBinaryLogical(depth + 1, DatabendDataType.BOOLEAN); + case BINARY_COMPARISON: + return getComparison(depth + 1); + case LIKE: + return getLike(depth + 1, DatabendDataType.VARCHAR); + default: + throw new AssertionError(); } } @@ -114,63 +129,58 @@ Node generateBooleanExpression(int depth) { Node getPostfix(int depth) { DatabendUnaryPostfixOperator randomOp = DatabendUnaryPostfixOperator.getRandom(); return new DatabendUnaryPostfixOperation( - generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), - randomOp,Randomly.getBoolean()); + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp, + Randomly.getBoolean()); } Node getNOT(int depth) { DatabendUnaryPrefixOperator op = DatabendUnaryPrefixOperator.NOT; - return new DatabendUnaryPrefixOperation( - generateExpression(Randomly.fromOptions(op.getInputDataTypes()), depth), - op,Randomly.getBoolean()); + return new DatabendUnaryPrefixOperation(generateExpression(op.getRandomInputDataTypes(), depth), op); } - Node getBetween(int depth){ - //跳过boolean + Node getBetween(int depth) { + // 跳过boolean DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); - return new NewBetweenOperatorNode(generateExpression(dataType,depth), - generateExpression(dataType,depth), generateExpression(dataType,depth), - Randomly.getBoolean()); + return new NewBetweenOperatorNode(generateExpression(dataType, depth), + generateExpression(dataType, depth), generateExpression(dataType, depth), Randomly.getBoolean()); } Node getIn(int depth) { DatabendDataType dataType = Randomly.fromOptions(DatabendDataType.values()); - Node leftExpr = generateExpression(dataType,depth); + Node leftExpr = generateExpression(dataType, depth); List> rightExprs = new ArrayList<>(); int nr = Randomly.smallNumber() + 1; - for(int i = 0; i < nr; i++) { - rightExprs.add(generateExpression(dataType,depth)); + for (int i = 0; i < nr; i++) { + rightExprs.add(generateExpression(dataType, depth)); } - return new NewInOperatorNode(leftExpr,rightExprs, Randomly.getBoolean()); + return new NewInOperatorNode(leftExpr, rightExprs, Randomly.getBoolean()); } - Node getBinaryLogical(int depth, DatabendDataType dataType){ - Node expr = generateExpression(dataType,depth); + Node getBinaryLogical(int depth, DatabendDataType dataType) { + Node expr = generateExpression(dataType, depth); int nr = Randomly.smallNumber() + 1; for (int i = 0; i < nr; i++) { - expr = new DatabendBinaryLogicalOperation(expr, - generateExpression(DatabendDataType.BOOLEAN, depth), + expr = new DatabendBinaryLogicalOperation(expr, generateExpression(DatabendDataType.BOOLEAN, depth), DatabendBinaryLogicalOperator.getRandom()); } return expr; } Node getComparison(int depth) { - //跳过boolean + // 跳过boolean DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); - Node leftExpr = generateExpression(dataType,depth); - Node rightExpr = generateExpression(dataType,depth); - DatabendBinaryComparisonOperation op = new DatabendBinaryComparisonOperation(leftExpr,rightExpr, + Node leftExpr = generateExpression(dataType, depth); + Node rightExpr = generateExpression(dataType, depth); + return new DatabendBinaryComparisonOperation(leftExpr, rightExpr, Randomly.fromOptions(DatabendBinaryComparisonOperator.values())); - return op; } Node getLike(int depth, DatabendDataType dataType) { - return new DatabendLikeOperation(generateExpression(dataType,depth) - ,generateExpression(dataType,depth), DatabendLikeOperation.DatabendLikeOperator.LIKE_OPERATOR); + return new DatabendLikeOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + DatabendLikeOperation.DatabendLikeOperator.LIKE_OPERATOR); } @Override @@ -180,16 +190,16 @@ public Node generatePredicate() { @Override public Node negatePredicate(Node predicate) { - return new DatabendUnaryPrefixOperation(predicate,DatabendUnaryPrefixOperator.NOT); + return new DatabendUnaryPrefixOperation(predicate, DatabendUnaryPrefixOperator.NOT); } @Override public Node isNull(Node predicate) { - return new DatabendUnaryPostfixOperation(predicate,DatabendUnaryPostfixOperator.IS_NULL); + return new DatabendUnaryPostfixOperation(predicate, DatabendUnaryPostfixOperator.IS_NULL); } - public Node generateConstant(DatabendDataType type,boolean isNullable) { - if(isNullable && Randomly.getBooleanWithSmallProbability()) { + public Node generateConstant(DatabendDataType type, boolean isNullable) { + if (isNullable && Randomly.getBooleanWithSmallProbability()) { createConstant(DatabendDataType.NULL); } return createConstant(type); @@ -206,20 +216,20 @@ public Node generateConstant(DatabendDataType type) { public Node createConstant(DatabendDataType type) { Randomly r = globalState.getRandomly(); switch (type) { - case INT: - //TODO 已支持数值型string转化，待添加 - return DatabendConstant.createIntConstant(r.getInteger()); - case BOOLEAN: - //TODO 已支持boolean型string转化，待添加 - return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); - case FLOAT: - return DatabendConstant.createFloatConstant((float) r.getDouble()); - case VARCHAR: - return DatabendConstant.createStringConstant(r.getString()); - case NULL: - return DatabendConstant.createNullConstant(); - default: - throw new AssertionError(type); + case INT: + // TODO 已支持数值型string转化但仍然不支持运算符计算，待添加 + return DatabendConstant.createIntConstant(r.getInteger()); + case BOOLEAN: + // TODO 已支持boolean型string转化但仍然不支持运算符计算，待添加 + return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + return DatabendConstant.createFloatConstant((float) r.getDouble()); + case VARCHAR: + return DatabendConstant.createStringConstant(r.getString()); + case NULL: + return DatabendConstant.createNullConstant(); + default: + throw new AssertionError(type); } } @@ -239,21 +249,16 @@ protected boolean canGenerateColumnOfType(DatabendDataType type) { } public enum DatabendAggregateFunction { - MAX(1), - MIN(1), - AVG(1,DatabendDataType.INT,DatabendDataType.FLOAT), - COUNT(1), - SUM(1,DatabendDataType.INT,DatabendDataType.FLOAT), - STDDEV_POP(1), - COVAR_POP(1), COVAR_SAMP(2); - //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) + MAX(1), MIN(1), AVG(1, DatabendDataType.INT, DatabendDataType.FLOAT), COUNT(1), + SUM(1, DatabendDataType.INT, DatabendDataType.FLOAT), STDDEV_POP(1), COVAR_POP(1), COVAR_SAMP(2); + // , STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) private int nrArgs; private DatabendDataType[] dataTypes; - DatabendAggregateFunction(int nrArgs, DatabendDataType ...dataTypes) { + DatabendAggregateFunction(int nrArgs, DatabendDataType... dataTypes) { this.nrArgs = nrArgs; - this.dataTypes = dataTypes; + this.dataTypes = dataTypes.clone(); } public static DatabendAggregateFunction getRandom() { @@ -261,7 +266,7 @@ public static DatabendAggregateFunction getRandom() { } public DatabendDataType getRandomType() { - if(dataTypes.length == 0) { + if (dataTypes.length == 0) { return Randomly.fromOptions(DatabendDataType.values()); } else { return Randomly.fromOptions(dataTypes); @@ -277,7 +282,7 @@ public int getNrArgs() { public NewFunctionNode generateArgsForAggregate( DatabendAggregateFunction aggregateFunction) { return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs(),aggregateFunction.getRandomType()), + generateExpressions(aggregateFunction.getNrArgs(), aggregateFunction.getRandomType()), aggregateFunction); } @@ -287,9 +292,9 @@ public Node generateAggregate() { } public Node generateHavingClause() { - this.allowAggregateFunctions = true; + allowAggregateFunctions = true; Node expression = generateExpression(DatabendDataType.BOOLEAN); - this.allowAggregateFunctions = false; + allowAggregateFunctions = false; return expression; } diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 9ac3499b3..099c0aa9e 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -1,20 +1,20 @@ package sqlancer.databend.gen; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.DatabendSchema.DatabendTables; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; public final class DatabendRandomQuerySynthesizer { @@ -23,8 +23,8 @@ private DatabendRandomQuerySynthesizer() { public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); -// DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) -// .setColumns(targetTables.getColumns()); + // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) + // .setColumns(targetTables.getColumns()); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); @@ -58,16 +58,17 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int } if (Randomly.getBoolean()) { - select.setLimitClause(DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + select.setLimitClause( + DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } if (Randomly.getBoolean()) { select.setOffsetClause( DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } // TODO 待添加HavingClause -// if (Randomly.getBoolean()) { -// select.setHavingClause(gen.generateHavingClause()); -// } + // if (Randomly.getBoolean()) { + // select.setHavingClause(gen.generateHavingClause()); + // } return select; } diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index ba08f8acc..c83cd93e9 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -1,19 +1,19 @@ package sqlancer.databend.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; - -import java.util.ArrayList; -import java.util.List; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; public class DatabendTableGenerator { @@ -25,8 +25,8 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(tableName); sb.append("("); List columns = getNewColumns(); - TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = - new DatabendNewExpressionGenerator(globalState).setColumns(columns); + TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = new DatabendNewExpressionGenerator( + globalState).setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); @@ -34,42 +34,42 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType()); -// if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() -// && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { -// sb.append(" COLLATE "); -// sb.append(getRandomCollate()); -// } -// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { -// sb.append(" UNIQUE"); -// } + // if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + // && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { + // sb.append(" COLLATE "); + // sb.append(getRandomCollate()); + // } + // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + // sb.append(" UNIQUE"); + // } if (globalState.getDbmsSpecificOptions().testNotNullConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); } else { - sb.append(" NULL"); //Databend 默认字段为非空，这个将它默认设置为允许空 + sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 } -// if (globalState.getDbmsSpecificOptions().testCheckConstraints //databend 无check约束 -// && Randomly.getBooleanWithRatherLowProbability()) { -// sb.append(" CHECK("); -// sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); -// DatabendErrors.addExpressionErrors(errors); -// sb.append(")"); -// } + // if (globalState.getDbmsSpecificOptions().testCheckConstraints //databend 无check约束 + // && Randomly.getBooleanWithRatherLowProbability()) { + // sb.append(" CHECK("); + // sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); + // DatabendErrors.addExpressionErrors(errors); + // sb.append(")"); + // } if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); - sb.append(DatabendToStringVisitor.asString( //常量类型于字段类型等同 + sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); sb.append(")"); } } - //databend并没有索引 -// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { -// errors.add("Invalid type for index"); -// List primaryKeyColumns = Randomly.nonEmptySubset(columns); -// sb.append(", PRIMARY KEY("); -// sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); -// sb.append(")"); -// } + // databend并没有索引 + // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { + // errors.add("Invalid type for index"); + // List primaryKeyColumns = Randomly.nonEmptySubset(columns); + // sb.append(", PRIMARY KEY("); + // sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + // sb.append(")"); + // } sb.append(")"); return new SQLQueryAdapter(sb.toString(), errors, true); } @@ -82,7 +82,7 @@ private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { DatabendCompositeDataType columnType = DatabendCompositeDataType.getRandomWithoutNull(); - String columnName = String.format("c%d%s", i,columnType.getPrimitiveDataType().toString()); + String columnName = String.format("c%d%s", i, columnType.getPrimitiveDataType().toString()); columns.add(new DatabendColumn(columnName, columnType, false, false)); } return columns; diff --git a/src/sqlancer/databend/gen/DatabendUpdateGenerator.java b/src/sqlancer/databend/gen/DatabendUpdateGenerator.java deleted file mode 100644 index e2f396c4b..000000000 --- a/src/sqlancer/databend/gen/DatabendUpdateGenerator.java +++ /dev/null @@ -1,48 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendTable; - -import java.util.List; - -public final class DatabendUpdateGenerator { - - private DatabendUpdateGenerator() { - } - - public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); - DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); - DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()); - sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - Node expr; - if (Randomly.getBooleanWithSmallProbability()) { - expr = gen.generateExpression(); - DatabendErrors.addExpressionErrors(errors); - } else { - expr = gen.generateConstant(); - } - sb.append(DatabendToStringVisitor.asString(expr)); - } - DatabendErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendViewGenerator.java b/src/sqlancer/databend/gen/DatabendViewGenerator.java index 55a7c6366..6daff1694 100644 --- a/src/sqlancer/databend/gen/DatabendViewGenerator.java +++ b/src/sqlancer/databend/gen/DatabendViewGenerator.java @@ -1,12 +1,11 @@ package sqlancer.databend.gen; -import sqlancer.GlobalState; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendToStringVisitor; public final class DatabendViewGenerator { @@ -27,7 +26,8 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { sb.append(i); } sb.append(") AS "); - sb.append(DatabendToStringVisitor.asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + sb.append(DatabendToStringVisitor + .asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); DatabendErrors.addGroupByErrors(errors); diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 451940a6e..7d315bbee 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -1,5 +1,12 @@ package sqlancer.databend.test; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -12,24 +19,20 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendCastOperation; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.gen.DatabendExpressionGenerator; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.*; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - public class DatabendNoRECOracle extends NoRECBase implements TestOracle { private final DatabendSchema s; @@ -42,20 +45,20 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { @Override public void check() throws SQLException { - DatabendTables randomTables = s.getRandomTableNonEmptyTables(); //随机获得nr张表 + DatabendTables randomTables = s.getRandomTableNonEmptyTables(); // 随机获得nr张表 List columns = randomTables.getColumns(); -// DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); + // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); //生成随机where条件，形式为ast + Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件，形式为ast -// System.out.println(DatabendToStringVisitor.asString(randomWhereCondition)); + // System.out.println(DatabendToStringVisitor.asString(randomWhereCondition)); List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); List> joins = DatabendJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); //禁用优化 + int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); // 禁用优化 int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { @@ -68,7 +71,7 @@ public void check() throws SQLException { } private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + List> joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); // DatabendExpression isTrue = DatabendPostfixOperation.create(randomWhereCondition, @@ -84,7 +87,7 @@ private int getSecondQuery(List> tableList, Node> tableList, Node> tableList, - List columns, Node randomWhereCondition, List> joins) - throws SQLException { + List columns, Node randomWhereCondition, + List> joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); // DatabendAggregate aggr = new DatabendAggregate( List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + .map((c) -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); // DatabendAggregateFunction.COUNT); // select.setFetchColumns(Arrays.asList(aggr)); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions(new DatabendExpressionGenerator(state).setColumns(columns).generateOrderBys()); + select.setOrderByExpressions( + new DatabendNewExpressionGenerator(state).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); int firstCount = 0; @@ -125,6 +130,7 @@ private int getFirstQueryCount(SQLConnection con, List> if (options.logEachSelect()) { logger.writeCurrent(optimizedQueryString); } + // System.out.println("First: " + optimizedQueryString); try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { while (rs.next()) { firstCount++; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index 9ce498234..f177b2921 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -18,17 +18,18 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; +import sqlancer.databend.ast.DatabendCastOperation; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.DatabendSchema.DatabendDataType; -import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; import sqlancer.databend.gen.DatabendNewExpressionGenerator.DatabendAggregateFunction; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendBinaryArithmeticOperator; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPrefixOperator; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; + public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { private String firstResult; @@ -55,9 +56,9 @@ public void check() throws SQLException { fetchColumns.add(gen.generateAggregate()); } select.setFetchColumns(Arrays.asList(aggregate)); -// if (Randomly.getBooleanWithRatherLowProbability()) { -// select.setOrderByExpressions(gen.generateOrderBys()); -// } + // if (Randomly.getBooleanWithRatherLowProbability()) { + // select.setOrderByExpressions(gen.generateOrderBys()); + // } originalQuery = DatabendToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); @@ -77,7 +78,8 @@ public void check() throws SQLException { } private String createMetamorphicUnionQuery(DatabendSelect select, - NewFunctionNode aggregate, List> from) { + NewFunctionNode aggregate, + List> from) { String metamorphicQuery; Node whereClause = gen.generateExpression(DatabendDataType.BOOLEAN); Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, @@ -109,7 +111,7 @@ private String getAggregateResult(String queryString) throws SQLException { try { resultString = result.getString(1); } catch (Exception e) { - System.out.println("Invalid integer format for value"); //TODO 超过integer范围无法格式化异常，还未有解决方案 + System.out.println("Invalid integer format for value"); // TODO 超过integer范围无法格式化异常，还未有解决方案 } } return resultString; @@ -122,7 +124,8 @@ private String getAggregateResult(String queryString) throws SQLException { } } - private List> mapped(NewFunctionNode aggregate) { + private List> mapped( + NewFunctionNode aggregate) { DatabendCastOperation count; switch (aggregate.getFunc()) { case COUNT: @@ -131,22 +134,23 @@ private List> mapped(NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), - DatabendAggregateFunction.SUM); - count = new DatabendCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), + NewFunctionNode sum = new NewFunctionNode<>( + aggregate.getArgs(), DatabendAggregateFunction.SUM); + count = new DatabendCastOperation( + new NewFunctionNode<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); return aliasArgs(Arrays.asList(sum, count)); case STDDEV_POP: NewFunctionNode sumSquared = new NewFunctionNode<>( Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), - DatabendBinaryArithmeticOperator.MULT)), + DatabendBinaryArithmeticOperator.MULTIPLICATION)), DatabendAggregateFunction.SUM); count = new DatabendCastOperation( new NewFunctionNode(aggregate.getArgs(), DatabendAggregateFunction.COUNT), new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); - NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), - DatabendAggregateFunction.AVG); + NewFunctionNode avg = new NewFunctionNode<>( + aggregate.getArgs(), DatabendAggregateFunction.AVG); return aliasArgs(Arrays.asList(sumSquared, count, avg)); default: throw new AssertionError(aggregate.getFunc()); @@ -183,7 +187,7 @@ private DatabendSelect getSelect(List> aggregates, List leftSelect.setWhereClause(whereClause); leftSelect.setJoinList(joinList); if (Randomly.getBooleanWithSmallProbability()) { -// leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); leftSelect.setGroupByExpressions(select.getFetchColumns()); } return leftSelect; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java index c0d090649..6ef132572 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -1,5 +1,11 @@ package sqlancer.databend.test; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; @@ -8,24 +14,18 @@ import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendSchema; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.ast.DatabendJoin; -import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -public class DatabendQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { +public class DatabendQueryPartitioningBase extends + TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { DatabendSchema s; DatabendTables targetTables; @@ -65,7 +65,7 @@ public void check() throws SQLException { List> generateFetchColumns() { List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { //TODO 为什么会返回 false 或 true 字段 + if (Randomly.getBoolean()) { // TODO 为什么会返回 false 或 true 字段 columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); } else { columns = generateRandomColumns(); diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java index 23572b110..8eae1e502 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java @@ -1,14 +1,14 @@ package sqlancer.databend.test; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; +import sqlancer.databend.DatabendToStringVisitor; public class DatabendQueryPartitioningDistinctTester extends DatabendQueryPartitioningBase { diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java index 0dbdda637..dd25b6d7f 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java @@ -1,19 +1,19 @@ package sqlancer.databend.test; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; -import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; public class DatabendQueryPartitioningGroupByTester extends DatabendQueryPartitioningBase { diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java index 18c17b452..72ad0795b 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -1,21 +1,21 @@ package sqlancer.databend.test; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Random; public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase implements TestOracle { @@ -30,19 +30,20 @@ public void check() throws SQLException { if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } -// boolean orderBy = Randomly.getBoolean(); - boolean orderBy = false; //关闭order by - if (orderBy) { //TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select -// select.setOrderByExpressions(gen.generateOrderBys()); + // boolean orderBy = Randomly.getBoolean(); + boolean orderBy = false; // 关闭order by + if (orderBy) { // TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select + // select.setOrderByExpressions(gen.generateOrderBys()); List> constants = new ArrayList<>(); - constants.add(new DatabendConstant.DatabendIntConstant(new Random().nextInt(select.getFetchColumns().size()) + 1)); + constants.add(new DatabendConstant.DatabendIntConstant( + Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByExpressions(constants); } -// select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + // select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); -// System.out.println(originalQueryString); + // System.out.println(originalQueryString); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setHavingClause(predicate); @@ -65,7 +66,7 @@ protected Node generatePredicate() { @Override List> generateFetchColumns() { - return Arrays.asList(gen.generateHavingClause()); + return Collections.singletonList(gen.generateHavingClause()); } } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java index 74f9a22c1..2b43d0ba0 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java @@ -1,15 +1,14 @@ package sqlancer.databend.test; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; - import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import sqlancer.ComparatorHelper; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendToStringVisitor; + public class DatabendQueryPartitioningWhereTester extends DatabendQueryPartitioningBase { public DatabendQueryPartitioningWhereTester(DatabendGlobalState state) { @@ -25,10 +24,11 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); - } + // boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + boolean orderBy = false; + // if (orderBy) { //TODO 待开启 + // select.setOrderByExpressions(gen.generateOrderBys()); + // } select.setWhereClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/test/sqlancer/databend/TestDatabendConnection.java b/test/sqlancer/databend/TestDatabendConnection.java index 91e0a1d53..db915140f 100644 --- a/test/sqlancer/databend/TestDatabendConnection.java +++ b/test/sqlancer/databend/TestDatabendConnection.java @@ -1,6 +1,5 @@ package sqlancer.databend; -import com.mongodb.annotations.ThreadSafe; import org.junit.jupiter.api.Test; import sqlancer.Main; import sqlancer.Randomly; @@ -10,11 +9,12 @@ public class TestDatabendConnection { @Test - void testConnection(){ + void testConnection() { assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC),"--host","192.168.81.134","--port","3307","--username","user1" - ,"databend","--oracle","NoREC"})); + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + "--host", "192.168.191.151", "--port", "3307", "--username", "user1", "--password", "1234", + "databend", "--oracle", "HAVING" })); } } diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 2cf53eef2..bc7e0ea1c 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -69,4 +69,4 @@ public void testClickHouseTLPAggregate() { "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPAggregate")); } -} +} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDB.java index d36d8610f..da981faca 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDB.java @@ -17,4 +17,4 @@ public void testDuckDB() { "QUERY_PARTITIONING" })); } -} +} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestMySQLTLP.java b/test/sqlancer/dbms/TestMySQLTLP.java index 3d3e7d107..793d0f807 100644 --- a/test/sqlancer/dbms/TestMySQLTLP.java +++ b/test/sqlancer/dbms/TestMySQLTLP.java @@ -21,4 +21,4 @@ public void testMySQL() { "mysql", "--oracle", "TLP_WHERE" })); } -} +} \ No newline at end of file From 4daedd927af1e72fc2c80d21f7b3e99a01799e9f Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 11 Sep 2022 16:39:40 +0800 Subject: [PATCH 0611/1284] feat(databend): add ci test Signed-off-by: Yisong Han --- .github/workflows/main.yml | 29 ++++- databases/database0.db | Bin 0 -> 12288 bytes databases/databaseconnectiontest.db | 0 ...DatabendQueryPartitioningHavingTester.java | 1 - test/sqlancer/dbms/TestDatabend.java | 106 ++++++++++++++++++ 5 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 databases/database0.db create mode 100644 databases/databaseconnectiontest.db create mode 100644 test/sqlancer/dbms/TestDatabend.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e11a8d053..12e0d4bee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,6 +117,33 @@ jobs: - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test + databend: + name: DBMS Tests (Databend) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up Databend + run: | + sudo apt update + sudo apt install mysql-client + LASTEST_TAG=$(curl -s GET https://api.github.com/repos/datafuselabs/databend/tags\?per_page\=1 | jq -r '.[].name') + curl -LJO https://github.com/datafuselabs/databend/releases/download/${LASTEST_TAG}/databend-${LASTEST_TAG}-x86_64-unknown-linux-musl.tar.gz + mkdir ./databend && tar xzvf databend-${LASTEST_TAG}-x86_64-unknown-linux-musl.tar.gz -C ./databend + ./databend/bin/databend-query & + - name: Create SQLancer user + run: mysql -uroot -h127.0.0.1 -P3307 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" + - name: Run Tests + run: | + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test + duckdb: name: DBMS Tests (DuckDB) runs-on: ubuntu-latest @@ -320,4 +347,4 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb \ No newline at end of file diff --git a/databases/database0.db b/databases/database0.db new file mode 100644 index 0000000000000000000000000000000000000000..2072f0a2b682817091df8ed8176e371cfef499dc GIT binary patch literal 12288 zcmeI2&ubGw6vw}t-89;?tR8akAZ($U2HKJ~KTK{WZPh^9(j>LswAus;TB!uvYel?y z_dn@Lyz5cWgWyGP;z>{t1;1}6)7`X!*Me`C*_nAW@6CIkH?xqi?X~81zZ>)(K6NvF4}pWco0_S$80&QT*-z%)nd9GjxGb&fV;W4UL$ ziPXhR=FIaN9&_C;_E_h`GVJY5dborQT9t9Tu3fdwu_ns`Jq#E}BlJ~bG7{sx#JPbR zoF{Qb5lMuzjAV||lTmbzG$eLASK_}M>(8+-F(0=tYg!$#AD?06c}k;I;?l@k8AbY6 zV^RA6Z7Yg%a=VkO%R2%pM3BlZhmt#ggLY8A_r zi03vw*=b@+?rd0u_-dGcy8~N{x<%W|p}jgaI^?$$Mj*OT*0S8h$K#hs>}rxvySw5n zXUJ)uUR79^_$;#&|EVRshB?Xp$W!|YtrYky@+p}R+XdQ`j1*`$VkS1K=pD23Hy_1z zmDa>kA&j#cwkotPYnfcj{+3-HkJDt@oHUbj%X^ROihPB{J4a+Bue^IsY9fgmIRr{-w~bg?8?cq*)=E6>=k*BTxV%ZJsffD%vw wN Date: Sun, 11 Sep 2022 18:19:31 +0800 Subject: [PATCH 0612/1284] fix(databend): code adjustment Signed-off-by: Yisong Han --- .github/workflows/main.yml | 2 +- .gitignore | 4 + .settings/org.eclipse.jdt.core.prefs | 9 +- databases/database0.db | Bin 12288 -> 0 bytes databases/databaseconnectiontest.db | 0 dependency-reduced-pom.xml | 433 ------------------ src/sqlancer/GlobalState.java | 9 +- src/sqlancer/databend/DatabendErrors.java | 2 + src/sqlancer/databend/DatabendProvider.java | 24 +- src/sqlancer/databend/DatabendSchema.java | 13 +- .../DatabendBinaryComparisonOperation.java | 6 +- .../ast/DatabendBinaryLogicalOperation.java | 40 +- src/sqlancer/databend/ast/DatabendJoin.java | 1 - .../gen/DatabendNewExpressionGenerator.java | 4 +- .../gen/DatabendRandomQuerySynthesizer.java | 15 +- .../databend/gen/DatabendViewGenerator.java | 19 +- .../databend/test/DatabendNoRECOracle.java | 5 - ...abendQueryPartitioningAggregateTester.java | 12 +- .../databend/TestDatabendConnection.java | 20 - test/sqlancer/dbms/TestClickHouse.java | 2 +- test/sqlancer/dbms/TestDatabend.java | 123 ++--- test/sqlancer/dbms/TestDuckDB.java | 2 +- test/sqlancer/dbms/TestMySQLTLP.java | 2 +- 23 files changed, 108 insertions(+), 639 deletions(-) delete mode 100644 databases/database0.db delete mode 100644 databases/databaseconnectiontest.db delete mode 100644 dependency-reduced-pom.xml delete mode 100644 test/sqlancer/databend/TestDatabendConnection.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 12e0d4bee..bfbbe60cc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -347,4 +347,4 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb \ No newline at end of file + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb diff --git a/.gitignore b/.gitignore index dcec833f6..48efcf83a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,12 @@ target/ .classpath .settings/org.eclipse.core.resources.prefs .settings/org.eclipse.m2e.core.prefs +.settings/org.eclipse.jdt.core.prefs .project .checkstyle *.DS_Store .idea SQLancer.iml +dependency-reduced-pom.xml +database0.db +databaseconnectiontest.db diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 3af1c5ad8..0a659c1a7 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=11 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,7 +28,6 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore -org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -74,7 +73,6 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore -org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -112,6 +110,5 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=11 +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/databases/database0.db b/databases/database0.db deleted file mode 100644 index 2072f0a2b682817091df8ed8176e371cfef499dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2&ubGw6vw}t-89;?tR8akAZ($U2HKJ~KTK{WZPh^9(j>LswAus;TB!uvYel?y z_dn@Lyz5cWgWyGP;z>{t1;1}6)7`X!*Me`C*_nAW@6CIkH?xqi?X~81zZ>)(K6NvF4}pWco0_S$80&QT*-z%)nd9GjxGb&fV;W4UL$ ziPXhR=FIaN9&_C;_E_h`GVJY5dborQT9t9Tu3fdwu_ns`Jq#E}BlJ~bG7{sx#JPbR zoF{Qb5lMuzjAV||lTmbzG$eLASK_}M>(8+-F(0=tYg!$#AD?06c}k;I;?l@k8AbY6 zV^RA6Z7Yg%a=VkO%R2%pM3BlZhmt#ggLY8A_r zi03vw*=b@+?rd0u_-dGcy8~N{x<%W|p}jgaI^?$$Mj*OT*0S8h$K#hs>}rxvySw5n zXUJ)uUR79^_$;#&|EVRshB?Xp$W!|YtrYky@+p}R+XdQ`j1*`$VkS1K=pD23Hy_1z zmDa>kA&j#cwkotPYnfcj{+3-HkJDt@oHUbj%X^ROihPB{J4a+Bue^IsY9fgmIRr{-w~bg?8?cq*)=E6>=k*BTxV%ZJsffD%vw wN - - 4.0.0 - com.sqlancer - sqlancer - SQLancer - 2.0.0 - SQLancer finds logic bugs in Database Management Systems through automatic testing - http://www.sqlancer.com/ - - - mrigger - Manuel Rigger - manuel.rigger@inf.ethz.ch - https://www.manuelrigger.at/ - ETH Zurich - https://ethz.ch/ - - architect - developer - - - - - - MIT License - https://github.com/sqlancer/sqlancer/blob/master/LICENSE.md - repo - - - - scm:git:git://github.com/sqlancer/sqlancer.git - scm:git:ssh://github.com:sqlancer/sqlancer.git - https://github.com/sqlancer/sqlancer/ - - - src - test - - - maven-shade-plugin - 3.3.0 - - - package - - shade - - - - - com.beust:jcommander - - - - - - - - maven-antrun-plugin - 3.1.0 - - - package - - run - - - - - - - - - - - - - maven-surefire-plugin - 2.22.2 - - - org.jacoco - jacoco-maven-plugin - 0.8.8 - - - - prepare-agent - - - - report - test - - report - - - - - - maven-compiler-plugin - 3.10.1 - - - org.codehaus.plexus - plexus-compiler-eclipse - 2.12.0 - - - org.eclipse.jdt - ecj - 3.28.0 - - - org.codehaus.plexus - plexus-compiler-api - 2.12.0 - - - - 11 - 11 - - ${project.basedir}/.settings/org.eclipse.jdt.core.prefs - - eclipse - true - true - - - - net.revelc.code.formatter - formatter-maven-plugin - 2.19.0 - - - eclipseformat - verify - - validate - - - - - - maven-dependency-plugin - 3.2.0 - - - copy-dependencies - prepare-package - - copy-dependencies - - - ${project.build.directory}/lib - false - false - true - jcommander - - - - - - maven-jar-plugin - 3.2.2 - - true - - - true - lib/ - sqlancer.Main - - - - - - org.codehaus.mojo - xml-maven-plugin - 1.0.2 - - - xml-check-format - verify - - check-format - - - - - - maven-checkstyle-plugin - 3.1.2 - - - checkstyle - verify - - check - - - true - - - - - - com.puppycrawl.tools - checkstyle - 10.3 - - - - configs/checkstyle.xml - - - - maven-pmd-plugin - 3.14.0 - - - pmd - verify - - check - - - 2 - true - - configs/pmd-rules.xml - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - 4.7.0.0 - - - spotbugs - verify - - check - - - configs/spotbugs-exclude.xml - High - - - - - - - - - jdk-8-config - - ${java.home}/../bin/javadoc - - - - jdk-11-config - - ${java.home}/bin/javadoc - - - - release-steps - - - - maven-source-plugin - 3.2.1 - - - attach-sources - - jar-no-fork - - - - - - maven-javadoc-plugin - 3.4.0 - - - attach-javadocs - - jar - - - - - 8 - - - - maven-gpg-plugin - 3.0.1 - - - sign-artifacts - verify - - sign - - - - --pinentry-mode - loopback - - - - - - - - - - - - com.google.auto.service - auto-service - 1.0.1 - compile - - - org.postgresql - postgresql - 42.4.0 - compile - - - org.xerial - sqlite-jdbc - 3.36.0.3 - compile - - - mysql - mysql-connector-java - 8.0.29 - compile - - - org.mariadb.jdbc - mariadb-java-client - 3.0.5 - compile - - - org.duckdb - duckdb_jdbc - 0.4.0 - compile - - - org.junit.jupiter - junit-jupiter-engine - 5.8.2 - test - - - junit-platform-engine - org.junit.platform - - - junit-jupiter-api - org.junit.jupiter - - - apiguardian-api - org.apiguardian - - - - - org.slf4j - slf4j-simple - 1.7.36 - compile - - - ru.yandex.clickhouse - clickhouse-jdbc - 0.3.2 - compile - - - com.h2database - h2 - 2.1.214 - compile - - - org.mongodb - mongodb-driver-sync - 4.1.1 - compile - - - com.arangodb - arangodb-java-driver - 6.9.0 - compile - - - - - - maven-jxr-plugin - 3.2.0 - - - - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - UTF-8 - - diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 9c48dd900..64b5c731e 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,7 +1,5 @@ package sqlancer; -import java.sql.SQLException; - import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractSchema; @@ -144,12 +142,7 @@ protected void setSchema(S schema) { } public void updateSchema() throws Exception { - try { - setSchema(readSchema()); - } catch (SQLException sqlException) { - System.out.printf("%s readSchema SQLException%n", databaseName); - sqlException.printStackTrace(); - } + setSchema(readSchema()); for (AbstractTable table : schema.getDatabaseTables()) { table.recomputeCount(); } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 92c7b17f7..60ea7ca9c 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -51,6 +51,8 @@ public static void addExpressionErrors(ExpectedErrors errors) { // removed column errors.add("Contents of view were altered: types don't match!"); errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + // TODO Databend待修复的bug（union schema error mismatch） + errors.add("unexpected end of file (failed to fill whole buffer)"); } private static void addRegexErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 793650798..9a94241d0 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -1,6 +1,5 @@ package sqlancer.databend; -import java.io.File; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -74,7 +73,7 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); // case DELETE: // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); - // case CREATE_VIEW: + // case CREATE_VIEW: //TODO 暂时关闭create view // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); @@ -111,22 +110,6 @@ public void generateDatabase(DatabendGlobalState globalState) throws Exception { se.executeStatements(); // 在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 } - public void tryDeleteFile(String fname) { - try { - File f = new File(fname); - f.delete(); - } catch (Exception e) { - } - } - - public void tryDeleteDatabase(String dbpath) { - if (dbpath.equals("") || dbpath.equals(":memory:")) { - return; - } - tryDeleteFile(dbpath); - tryDeleteFile(dbpath + ".wal"); - } - @Override public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLException { String username = globalState.getOptions().getUserName(); @@ -156,11 +139,6 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE globalState.getState().logStatement("USE " + databaseName); } - // try (Statement s = con.createStatement()) { - // s.execute("set enable_planner_v2 = 0;"); - // globalState.getState().logStatement("set enable_planner_v2 = 0;"); - // } - return new SQLConnection(con); } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 643f53dd9..98d7acafd 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -255,21 +255,14 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { List tableNames = null; tableNames = new ArrayList<>(); - // SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema != 'system' and table_schema != - // 'INFORMATION_SCHEMA' and table_type='BASE TABLE' - // "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName + final String sqlStatement = String.format( "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ", databaseName); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(sqlStatement)) { - try { // 没有catch的话rs.next()会报SQLException：Not a navigable ResultSet - while (rs.next()) { - tableNames.add(rs.getString("table_name")); - } - } catch (Exception e) { - // e.printStackTrace(); - System.out.println("TableNames->SQLException：Not a navigable ResultSet"); + while (rs.next()) { + tableNames.add(rs.getString("table_name")); } } } diff --git a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java index 9e8e43a51..cf63ad17e 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java @@ -12,10 +12,8 @@ public DatabendBinaryComparisonOperation(Node left, Node"), GREATER_EQUALS(">="); + EQUALS("="), IS_DISTINCT("IS DISTINCT FROM"), IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), NOT_EQUALS("!="), + LESS("<"), LESS_EQUALS("<="), GREATER(">"), GREATER_EQUALS(">="); private final String textRepresentation; diff --git a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java index 14ff984e6..355ec88a5 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java @@ -13,43 +13,7 @@ public DatabendBinaryLogicalOperation(Node left, Node> getJoins( TableReferenceNode rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); - // DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState) .setColumns(columns); diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index f565a9be7..43d38939f 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -107,9 +107,9 @@ Node generateBooleanExpression(int depth) { BooleanExpression option = Randomly.fromList(validOptions); switch (option) { case POSTFIX_OPERATOR: - getPostfix(depth + 1); + return getPostfix(depth + 1); case NOT: - getNOT(depth + 1); + return getNOT(depth + 1); case BETWEEN: // TODO (NULL BETWEEN NULL AND NULL) 返回的是 NULL 需要注意 return getBetween(depth + 1); case IN_OPERATION: diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 099c0aa9e..286764c6e 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -23,8 +23,6 @@ private DatabendRandomQuerySynthesizer() { public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); - // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) - // .setColumns(targetTables.getColumns()); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); @@ -50,12 +48,13 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); - } - if (Randomly.getBoolean()) { - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - } + // if (Randomly.getBoolean()) {//TODO order by超过实际行数 + // select.setOrderByExpressions(gen.generateOrderBys()); + // } + + // if (Randomly.getBoolean()) { //TODO group by超过实际行数 + // select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + // } if (Randomly.getBoolean()) { select.setLimitClause( diff --git a/src/sqlancer/databend/gen/DatabendViewGenerator.java b/src/sqlancer/databend/gen/DatabendViewGenerator.java index 6daff1694..55cc6ffc5 100644 --- a/src/sqlancer/databend/gen/DatabendViewGenerator.java +++ b/src/sqlancer/databend/gen/DatabendViewGenerator.java @@ -17,15 +17,16 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { StringBuilder sb = new StringBuilder("CREATE "); sb.append("VIEW "); sb.append(globalState.getSchema().getFreeViewName()); - sb.append("("); - for (int i = 0; i < nrColumns; i++) { - if (i != 0) { - sb.append(", "); - } - sb.append("c"); - sb.append(i); - } - sb.append(") AS "); + // sb.append("("); + // for (int i = 0; i < nrColumns; i++) { + // if (i != 0) { + // sb.append(", "); + // } + // sb.append("c"); + // sb.append(i); + // } + // sb.append(") AS "); + sb.append(" AS "); sb.append(DatabendToStringVisitor .asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 7d315bbee..cb27fa78b 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -47,13 +47,10 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { public void check() throws SQLException { DatabendTables randomTables = s.getRandomTableNonEmptyTables(); // 随机获得nr张表 List columns = randomTables.getColumns(); - // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件，形式为ast - // System.out.println(DatabendToStringVisitor.asString(randomWhereCondition)); - List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -87,7 +84,6 @@ private int getSecondQuery(List> tableList, Node> if (options.logEachSelect()) { logger.writeCurrent(optimizedQueryString); } - // System.out.println("First: " + optimizedQueryString); try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { while (rs.next()) { firstCount++; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index f177b2921..59d6695ca 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -111,7 +111,7 @@ private String getAggregateResult(String queryString) throws SQLException { try { resultString = result.getString(1); } catch (Exception e) { - System.out.println("Invalid integer format for value"); // TODO 超过integer范围无法格式化异常，还未有解决方案 + throw new IgnoreMeException(); // TODO 超过integer范围无法格式化异常，还未有解决方案 } } return resultString; @@ -171,7 +171,7 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List leftSelect.setFromList(from); leftSelect.setWhereClause(whereClause); leftSelect.setJoinList(joinList); - if (Randomly.getBooleanWithSmallProbability()) { - leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - leftSelect.setGroupByExpressions(select.getFetchColumns()); - } + // if (Randomly.getBooleanWithSmallProbability()) { + // leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); //TODO group by超过实际行数 + // leftSelect.setGroupByExpressions(select.getFetchColumns());// TODO group by不能放入聚合函数 + // } return leftSelect; } diff --git a/test/sqlancer/databend/TestDatabendConnection.java b/test/sqlancer/databend/TestDatabendConnection.java deleted file mode 100644 index db915140f..000000000 --- a/test/sqlancer/databend/TestDatabendConnection.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.databend; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; -import sqlancer.Randomly; -import sqlancer.dbms.TestConfig; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class TestDatabendConnection { - @Test - void testConnection() { - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "192.168.191.151", "--port", "3307", "--username", "user1", "--password", "1234", - "databend", "--oracle", "HAVING" })); - } -} diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index bc7e0ea1c..2cf53eef2 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -69,4 +69,4 @@ public void testClickHouseTLPAggregate() { "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPAggregate")); } -} \ No newline at end of file +} diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index 94a86aef0..a9575fd7d 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -21,66 +21,7 @@ public void testDatabendNoREC() { "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); } - @Test - public void testDatabendTLPWhere() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "WHERE")); - } - - @Test - public void testDatabendTLPGroupBy() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "GROUP_BY")); - } - - @Test - public void testDatabendTLPHaving() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "HAVING")); - } - - @Test - public void testDatabendTLPDistinct() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "DISTINCT")); - } - - @Test - public void testDatabendTLPAggregate() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "AGGREGATE")); - } - + // TODO Databend待修复的bug（union schema error mismatch）https://github.com/datafuselabs/databend/issues/7463 @Test public void testDatabendTLPQueryPartitioning() { String databendAvailable = System.getenv("DATABEND_AVAILABLE"); @@ -93,13 +34,73 @@ public void testDatabendTLPQueryPartitioning() { "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); } + // @Test + // public void testDatabendTLPWhere() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "WHERE")); + // } + // + // @Test + // public void testDatabendTLPGroupBy() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "GROUP_BY")); + // } + // + // @Test + // public void testDatabendTLPHaving() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "HAVING")); + // } + // + // @Test + // public void testDatabendTLPDistinct() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "DISTINCT")); + // } + // + // @Test + // public void testDatabendTLPAggregate() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "AGGREGATE")); + // } + // @Test // void testConnection() { // assertEquals(0, // Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, // "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - // "--host", "192.168.191.151", "--port", "3307", "--username", "user1", "--password", "1234", + // "--host", "127.0.0.1", "--port", "3307", "--username", "user1", "--password", "1234", // "databend", "--oracle", "HAVING" })); // } diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDB.java index da981faca..d36d8610f 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDB.java @@ -17,4 +17,4 @@ public void testDuckDB() { "QUERY_PARTITIONING" })); } -} \ No newline at end of file +} diff --git a/test/sqlancer/dbms/TestMySQLTLP.java b/test/sqlancer/dbms/TestMySQLTLP.java index 793d0f807..3d3e7d107 100644 --- a/test/sqlancer/dbms/TestMySQLTLP.java +++ b/test/sqlancer/dbms/TestMySQLTLP.java @@ -21,4 +21,4 @@ public void testMySQL() { "mysql", "--oracle", "TLP_WHERE" })); } -} \ No newline at end of file +} From 26b893a09adf43eeae5e1f7e6d9e92d3efd1a3cc Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Tue, 4 Oct 2022 13:29:50 +0400 Subject: [PATCH 0613/1284] Added wrapped exceptions support --- src/sqlancer/common/query/SQLQueryAdapter.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index c2e6706cc..14dbd523e 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -96,9 +96,19 @@ public String getUnterminatedQueryString() { } public void checkException(Exception e) throws AssertionError { - if (!expectedErrors.errorIsExpected(e.getMessage())) { - throw new AssertionError(query, e); + Throwable ex = e; + + while (true) { + if (expectedErrors.errorIsExpected(ex.getMessage())) { + return; + } else if (ex.getCause() != null) { + ex = ex.getCause(); + } else { + break; + } } + + throw new AssertionError(query, ex); } @Override From 2265a9a989256db64bfb9b382d8e0f8367904806 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Tue, 4 Oct 2022 15:39:26 +0400 Subject: [PATCH 0614/1284] Reworked checkExceptions method --- src/sqlancer/common/query/SQLQueryAdapter.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 14dbd523e..01fcc8539 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -98,17 +98,15 @@ public String getUnterminatedQueryString() { public void checkException(Exception e) throws AssertionError { Throwable ex = e; - while (true) { + while (ex != null) { if (expectedErrors.errorIsExpected(ex.getMessage())) { return; - } else if (ex.getCause() != null) { - ex = ex.getCause(); } else { - break; + ex = ex.getCause(); } } - throw new AssertionError(query, ex); + throw new AssertionError(query, e); } @Override From 46b7eecfdfabe71ab10431d981e0fcdc4654929d Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Tue, 4 Oct 2022 17:37:30 +0400 Subject: [PATCH 0615/1284] Added Yugabyte API support --- configs/spotbugs-exclude.xml | 7 +- pom.xml | 10 + src/sqlancer/yugabyte/ycql/YCQLErrors.java | 21 + src/sqlancer/yugabyte/ycql/YCQLOptions.java | 47 ++ src/sqlancer/yugabyte/ycql/YCQLProvider.java | 156 +++++ src/sqlancer/yugabyte/ycql/YCQLSchema.java | 267 ++++++++ .../yugabyte/ycql/YCQLToStringVisitor.java | 64 ++ .../yugabyte/ycql/ast/YCQLConstant.java | 173 +++++ .../yugabyte/ycql/ast/YCQLExpression.java | 8 + .../yugabyte/ycql/ast/YCQLSelect.java | 18 + .../ycql/gen/YCQLAlterTableGenerator.java | 48 ++ .../ycql/gen/YCQLDeleteGenerator.java | 31 + .../ycql/gen/YCQLExpressionGenerator.java | 297 +++++++++ .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 56 ++ .../ycql/gen/YCQLInsertGenerator.java | 65 ++ .../ycql/gen/YCQLRandomQuerySynthesizer.java | 55 ++ .../yugabyte/ycql/gen/YCQLTableGenerator.java | 57 ++ .../ycql/gen/YCQLUpdateGenerator.java | 56 ++ .../yugabyte/ycql/test/YCQLFuzzer.java | 73 +++ .../yugabyte/ysql/YSQLCompoundDataType.java | 45 ++ .../ysql/YSQLExpectedValueVisitor.java | 152 +++++ .../yugabyte/ysql/YSQLGlobalState.java | 127 ++++ src/sqlancer/yugabyte/ysql/YSQLOptions.java | 98 +++ src/sqlancer/yugabyte/ysql/YSQLProvider.java | 366 +++++++++++ src/sqlancer/yugabyte/ysql/YSQLSchema.java | 329 ++++++++++ .../yugabyte/ysql/YSQLToStringVisitor.java | 329 ++++++++++ src/sqlancer/yugabyte/ysql/YSQLVisitor.java | 120 ++++ .../yugabyte/ysql/ast/YSQLAggregate.java | 58 ++ src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java | 35 + .../ysql/ast/YSQLBetweenOperation.java | 63 ++ .../ast/YSQLBinaryArithmeticOperation.java | 106 +++ .../ysql/ast/YSQLBinaryBitOperation.java | 46 ++ .../ast/YSQLBinaryComparisonOperation.java | 135 ++++ .../ysql/ast/YSQLBinaryLogicalOperation.java | 88 +++ .../ysql/ast/YSQLBinaryRangeOperation.java | 71 ++ .../yugabyte/ysql/ast/YSQLCastOperation.java | 45 ++ .../yugabyte/ysql/ast/YSQLCollate.java | 33 + .../yugabyte/ysql/ast/YSQLColumnValue.java | 34 + .../ysql/ast/YSQLConcatOperation.java | 37 ++ .../yugabyte/ysql/ast/YSQLConstant.java | 611 ++++++++++++++++++ .../yugabyte/ysql/ast/YSQLExpression.java | 14 + .../yugabyte/ysql/ast/YSQLFunction.java | 283 ++++++++ .../ast/YSQLFunctionWithUnknownResult.java | 174 +++++ .../yugabyte/ysql/ast/YSQLInOperation.java | 65 ++ src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java | 49 ++ .../yugabyte/ysql/ast/YSQLOrderByTerm.java | 42 ++ .../ysql/ast/YSQLPOSIXRegularExpression.java | 65 ++ .../ysql/ast/YSQLPostfixOperation.java | 146 +++++ .../yugabyte/ysql/ast/YSQLPostfixText.java | 36 ++ .../ysql/ast/YSQLPrefixOperation.java | 115 ++++ .../yugabyte/ysql/ast/YSQLSelect.java | 135 ++++ .../yugabyte/ysql/ast/YSQLSimilarTo.java | 39 ++ .../ysql/gen/YSQLAlterTableGenerator.java | 182 ++++++ .../ysql/gen/YSQLAnalyzeGenerator.java | 37 ++ .../ysql/gen/YSQLClusterGenerator.java | 32 + .../ysql/gen/YSQLCommentGenerator.java | 68 ++ .../yugabyte/ysql/gen/YSQLCommon.java | 441 +++++++++++++ .../ysql/gen/YSQLDeleteGenerator.java | 46 ++ .../ysql/gen/YSQLDiscardGenerator.java | 39 ++ .../ysql/gen/YSQLDropIndexGenerator.java | 41 ++ .../ysql/gen/YSQLExpressionGenerator.java | 563 ++++++++++++++++ .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 153 +++++ .../ysql/gen/YSQLInsertGenerator.java | 127 ++++ .../ysql/gen/YSQLNotifyGenerator.java | 45 ++ .../ysql/gen/YSQLRandomQueryGenerator.java | 62 ++ .../ysql/gen/YSQLReindexGenerator.java | 58 ++ .../ysql/gen/YSQLSequenceGenerator.java | 90 +++ .../yugabyte/ysql/gen/YSQLSetGenerator.java | 196 ++++++ .../ysql/gen/YSQLStatisticsGenerator.java | 74 +++ .../yugabyte/ysql/gen/YSQLTableGenerator.java | 247 +++++++ .../ysql/gen/YSQLTableGroupGenerator.java | 24 + .../ysql/gen/YSQLTransactionGenerator.java | 27 + .../ysql/gen/YSQLTruncateGenerator.java | 41 ++ .../ysql/gen/YSQLUpdateGenerator.java | 76 +++ .../ysql/gen/YSQLVacuumGenerator.java | 19 + .../yugabyte/ysql/gen/YSQLViewGenerator.java | 73 +++ .../yugabyte/ysql/oracle/YSQLCatalog.java | 94 +++ .../yugabyte/ysql/oracle/YSQLFuzzer.java | 108 ++++ .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 165 +++++ .../YSQLPivotedQuerySynthesisOracle.java | 147 +++++ .../oracle/tlp/YSQLTLPAggregateOracle.java | 194 ++++++ .../yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 117 ++++ .../ysql/oracle/tlp/YSQLTLPHavingOracle.java | 66 ++ .../ysql/oracle/tlp/YSQLTLPWhereOracle.java | 45 ++ 84 files changed, 9296 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/yugabyte/ycql/YCQLErrors.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLOptions.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLProvider.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLSchema.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLGlobalState.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLOptions.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLProvider.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLSchema.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLVisitor.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java diff --git a/configs/spotbugs-exclude.xml b/configs/spotbugs-exclude.xml index 366e8ac55..5cc1baf73 100644 --- a/configs/spotbugs-exclude.xml +++ b/configs/spotbugs-exclude.xml @@ -11,5 +11,10 @@ - + + + + + + diff --git a/pom.xml b/pom.xml index 234d805f1..fdb287a10 100644 --- a/pom.xml +++ b/pom.xml @@ -286,6 +286,16 @@ postgresql 42.5.0 + + com.ing.data + cassandra-jdbc-wrapper + 4.6.0 + + + com.yugabyte + jdbc-yugabytedb + 42.3.0 + org.xerial sqlite-jdbc diff --git a/src/sqlancer/yugabyte/ycql/YCQLErrors.java b/src/sqlancer/yugabyte/ycql/YCQLErrors.java new file mode 100644 index 000000000..2bcddb71c --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLErrors.java @@ -0,0 +1,21 @@ +package sqlancer.yugabyte.ycql; + +import sqlancer.common.query.ExpectedErrors; + +public final class YCQLErrors { + + private YCQLErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.add("Signature mismatch in call to builtin function"); + errors.add("Qualified name not allowed for column reference"); + errors.add("Datatype Mismatch"); + errors.add("Invalid Datatype"); + errors.add("Invalid CQL Statement"); + errors.add("Invalid SQL Statement"); + errors.add("Order by clause contains invalid expression"); + errors.add("Invalid Function Call"); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLOptions.java b/src/sqlancer/yugabyte/ycql/YCQLOptions.java new file mode 100644 index 000000000..5955d28dd --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLOptions.java @@ -0,0 +1,47 @@ +package sqlancer.yugabyte.ycql; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ycql.YCQLOptions.YCQLOracleFactory; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.test.YCQLFuzzer; + +@Parameters(commandDescription = "YCQL") +public class YCQLOptions implements DBMSSpecificOptions { + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates = 5; + + @Parameter(names = "--datacenter", description = "YCQL datacenter, can be found in system.local table", arity = 1) + public String datacenter = "datacenter1"; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(YCQLOracleFactory.FUZZER); + + public enum YCQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YCQLGlobalState globalState) throws SQLException { + return new YCQLFuzzer(globalState); + } + + } + } + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLProvider.java b/src/sqlancer/yugabyte/ycql/YCQLProvider.java new file mode 100644 index 000000000..3d408f8c1 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLProvider.java @@ -0,0 +1,156 @@ +package sqlancer.yugabyte.ycql; + +import static sqlancer.yugabyte.ycql.YCQLSchema.getTableNames; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.gen.YCQLAlterTableGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLDeleteGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLIndexGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLInsertGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLRandomQuerySynthesizer; +import sqlancer.yugabyte.ycql.gen.YCQLTableGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLUpdateGenerator; + +@AutoService(DatabaseProvider.class) +public class YCQLProvider extends SQLProviderAdapter { + + public YCQLProvider() { + super(YCQLGlobalState.class, YCQLOptions.class); + } + + public enum Action implements AbstractAction { + + ALTER(YCQLAlterTableGenerator::getQuery), // + INSERT(YCQLInsertGenerator::getQuery), // + CREATE_INDEX(YCQLIndexGenerator::getQuery), // + DELETE(YCQLDeleteGenerator::generate), // + UPDATE(YCQLUpdateGenerator::getQuery), // + EXPLAIN((g) -> { + ExpectedErrors errors = new ExpectedErrors(); + YCQLErrors.addExpressionErrors(errors); + return new SQLQueryAdapter( + "EXPLAIN " + YCQLToStringVisitor + .asString(YCQLRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), + errors); + }); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(YCQLGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(YCQLGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case ALTER: + return r.getInteger(0, 10); + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case CREATE_INDEX: + case UPDATE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + case EXPLAIN: + return r.getInteger(0, 2); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + default: + throw new AssertionError(a); + } + } + + public static class YCQLGlobalState extends SQLGlobalState { + + @Override + protected YCQLSchema readSchema() throws SQLException { + return YCQLSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(YCQLGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new YCQLTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + YCQLProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLException { + try { + Class.forName("com.ing.data.cassandra.jdbc.CassandraDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + final String host = globalState.getOptions().getHost(); + final String url = "jdbc:cassandra://%s:9042/%s?localdatacenter=%s"; + final Connection connection = DriverManager.getConnection( + String.format(url, host, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); + + try (Statement stmt = connection.createStatement()) { + try { + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); + } catch (Exception se) { + // try again + List tableNames = getTableNames( + new SQLConnection(DriverManager.getConnection(String.format(url, host, + globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))), + globalState.getDatabaseName()); + for (String tableName : tableNames) { + stmt.execute("DROP TABLE " + globalState.getDatabaseName() + "." + tableName); + } + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); + } + + stmt.execute("CREATE KEYSPACE IF NOT EXISTS " + globalState.getDatabaseName()); + } + + return new SQLConnection(DriverManager.getConnection(String.format(url, host, globalState.getDatabaseName(), + globalState.getDbmsSpecificOptions().datacenter))); + } + + @Override + public String getDBMSName() { + return "ycql"; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java new file mode 100644 index 000000000..4845520d2 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -0,0 +1,267 @@ +package sqlancer.yugabyte.ycql; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; + +public class YCQLSchema extends AbstractSchema { + + public enum YCQLDataType { + + INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP; + + public static YCQLDataType getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public static class YCQLCompositeDataType { + + private final YCQLDataType dataType; + + private final int size; + + public YCQLCompositeDataType(YCQLDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public YCQLDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static YCQLCompositeDataType getRandom() { + YCQLDataType type = YCQLDataType.getRandom(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case BOOLEAN: + case VARCHAR: + case DATE: + case TIMESTAMP: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new YCQLCompositeDataType(type, size); + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 8: + return Randomly.fromOptions("BIGINT"); + case 4: + return Randomly.fromOptions("INTEGER", "INT"); + case 2: + return Randomly.fromOptions("SMALLINT"); + case 1: + return Randomly.fromOptions("TINYINT"); + default: + throw new AssertionError(size); + } + case VARCHAR: + return "VARCHAR"; + case FLOAT: + switch (size) { + case 8: + return Randomly.fromOptions("DOUBLE"); + case 4: + return Randomly.fromOptions("FLOAT"); + default: + throw new AssertionError(size); + } + case BOOLEAN: + return Randomly.fromOptions("BOOLEAN"); + case TIMESTAMP: + return Randomly.fromOptions("TIMESTAMP"); + case DATE: + return Randomly.fromOptions("DATE"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + } + + public static class YCQLColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + + public YCQLColumn(String name, YCQLCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class YCQLTables extends AbstractTables { + + public YCQLTables(List tables) { + super(tables); + } + + } + + public YCQLSchema(List databaseTables) { + super(databaseTables); + } + + public YCQLTables getRandomTableNonEmptyTables() { + return new YCQLTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + private static YCQLCompositeDataType getColumnType(String typeString) { + YCQLDataType primitiveType; + int size = -1; + switch (typeString.toUpperCase()) { + case "INT": + case "INTEGER": + primitiveType = YCQLDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = YCQLDataType.INT; + size = 2; + break; + case "BIGINT": + primitiveType = YCQLDataType.INT; + size = 8; + break; + case "TINYINT": + primitiveType = YCQLDataType.INT; + size = 1; + break; + case "VARCHAR": + case "TEXT": + primitiveType = YCQLDataType.VARCHAR; + break; + case "FLOAT": + primitiveType = YCQLDataType.FLOAT; + size = 4; + break; + case "DOUBLE": + primitiveType = YCQLDataType.FLOAT; + size = 8; + break; + case "BOOLEAN": + primitiveType = YCQLDataType.BOOLEAN; + break; + case "DATE": + primitiveType = YCQLDataType.DATE; + break; + case "TIMESTAMP": + primitiveType = YCQLDataType.TIMESTAMP; + break; + case "INTERVAL": + throw new IgnoreMeException(); + // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP + // '1969-12-29 06:50:27')) + default: + throw new IgnoreMeException(); + } + return new YCQLCompositeDataType(primitiveType, size); + } + + public static class YCQLTable extends AbstractRelationalTable { + + public YCQLTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static YCQLSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con, databaseName); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; + } + List databaseColumns = getTableColumns(con, databaseName, tableName); + boolean isView = tableName.startsWith("v"); + YCQLTable t = new YCQLTable(tableName, databaseColumns, isView); + for (YCQLColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new YCQLSchema(databaseTables); + } + + public static List getTableNames(SQLConnection con, String databaseName) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + String.format("select * from system_schema.tables where keyspace_name = '%s'", databaseName))) { + while (rs.next()) { + tableNames.add(rs.getString("table_name")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "select * from system_schema.columns where keyspace_name = '%s' and table_name = '%s'", + databaseName, tableName))) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("type"); + boolean isPrimaryKey = rs.getString("kind").contentEquals("partition_key"); + YCQLColumn c = new YCQLColumn(columnName, getColumnType(dataType), isPrimaryKey, true); + columns.add(c); + } + } + } + return columns; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java new file mode 100644 index 000000000..ddc7e6e8a --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java @@ -0,0 +1,64 @@ +package sqlancer.yugabyte.ycql; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.yugabyte.ycql.ast.YCQLConstant; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import sqlancer.yugabyte.ycql.ast.YCQLSelect; + +public class YCQLToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof YCQLConstant) { + visit((YCQLConstant) expr); + } else if (expr instanceof YCQLSelect) { + visit((YCQLSelect) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(YCQLConstant constant) { + sb.append(constant.toString()); + } + + private void visit(YCQLSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(Node expr) { + YCQLToStringVisitor visitor = new YCQLToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java new file mode 100644 index 000000000..dd85b4b66 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java @@ -0,0 +1,173 @@ +package sqlancer.yugabyte.ycql.ast; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +import sqlancer.common.ast.newast.Node; + +public class YCQLConstant implements Node { + + private YCQLConstant() { + } + + public static class YCQLNullConstant extends YCQLConstant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class YCQLIntConstant extends YCQLConstant { + + private final long value; + + public YCQLIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class YCQLDoubleConstant extends YCQLConstant { + + private final double value; + + public YCQLDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + } + + public static class YCQLTextConstant extends YCQLConstant { + + private final String value; + + public YCQLTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static class YCQLDateConstant extends YCQLConstant { + + public String textRepr; + + public YCQLDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("'%s'", textRepr); + } + + } + + public static class YCQLTimestampConstant extends YCQLConstant { + + public String textRepr; + + public YCQLTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("'%s'", textRepr); + } + + } + + public static class YCQLBooleanConstant extends YCQLConstant { + + private final boolean value; + + public YCQLBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static Node createStringConstant(String text) { + return new YCQLTextConstant(text); + } + + public static Node createFloatConstant(double val) { + return new YCQLDoubleConstant(val); + } + + public static Node createIntConstant(long val) { + return new YCQLIntConstant(val); + } + + public static Node createNullConstant() { + return new YCQLNullConstant(); + } + + public static Node createBooleanConstant(boolean val) { + return new YCQLBooleanConstant(val); + } + + public static Node createDateConstant(long integer) { + return new YCQLDateConstant(integer); + } + + public static Node createTimestampConstant(long integer) { + return new YCQLTimestampConstant(integer); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java b/src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java new file mode 100644 index 000000000..4177344eb --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java @@ -0,0 +1,8 @@ +package sqlancer.yugabyte.ycql.ast; + +public interface YCQLExpression { + + default YCQLConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java new file mode 100644 index 000000000..646182875 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java @@ -0,0 +1,18 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class YCQLSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java new file mode 100644 index 000000000..d15d7fe1c --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java @@ -0,0 +1,48 @@ +package sqlancer.yugabyte.ycql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLCompositeDataType; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; + +public final class YCQLAlterTableGenerator { + + private YCQLAlterTableGenerator() { + } + + enum Action { + ADD_COLUMN, DROP_COLUMN + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(YCQLCompositeDataType.getRandom().toString()); + break; + case DROP_COLUMN: + sb.append("DROP "); + sb.append(table.getRandomColumn().getName()); + break; + default: + throw new AssertionError(action); + } + + errors.add("Alter key column. Can't alter key column"); + errors.add("cannot remove a key column"); + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java new file mode 100644 index 000000000..108cd1be9 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java @@ -0,0 +1,31 @@ +package sqlancer.yugabyte.ycql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLErrors; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; + +public final class YCQLDeleteGenerator { + + private YCQLDeleteGenerator() { + } + + public static SQLQueryAdapter generate(YCQLGlobalState globalState) { + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(YCQLToStringVisitor.asString( + new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); + } + + YCQLErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java new file mode 100644 index 000000000..296be911d --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java @@ -0,0 +1,297 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLDataType; +import sqlancer.yugabyte.ycql.ast.YCQLConstant; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; + +public final class YCQLExpressionGenerator extends UntypedExpressionGenerator, YCQLColumn> { + + private final YCQLGlobalState globalState; + + public YCQLExpressionGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, FUNC, BETWEEN, IN + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBoolean()) { + YCQLAggregateFunction aggregate = YCQLAggregateFunction.getRandom(); + allowAggregates = false; + return new NewFunctionNode<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); + } + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case BINARY_COMPARISON: + Operator op = YCQLBinaryComparisonOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = YCQLBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), YCQLBinaryArithmeticOperator.getRandom()); + case FUNC: + DBFunction func = DBFunction.getRandom(); + return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new NewBetweenOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new NewInOperatorNode(generateExpression(depth + 1), + generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); + default: + throw new AssertionError(expr); + } + } + + @Override + protected Node generateColumn() { + YCQLColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode(column); + } + + @Override + public Node generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return YCQLConstant.createNullConstant(); + } + YCQLDataType type = YCQLDataType.getRandom(); + switch (type) { + case INT: + return YCQLConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DATE: + return YCQLConstant.createDateConstant(globalState.getRandomly().getInteger()); + case TIMESTAMP: + return YCQLConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case VARCHAR: + return YCQLConstant.createStringConstant(globalState.getRandomly().getString()); + case BOOLEAN: + return YCQLConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + return YCQLConstant.createFloatConstant(globalState.getRandomly().getDouble()); + default: + throw new AssertionError(); + } + } + + @Override + public List> generateOrderBys() { + List> expr = super.generateOrderBys(); + List> newExpr = new ArrayList<>(expr.size()); + for (Node curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + }; + + public enum YCQLAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1), SUM(1); + + private final int nrArgs; + + YCQLAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static YCQLAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } + + public enum DBFunction { + // YCQL functions + BLOB(1), // + TIMEUUID(1), // + DATE(0), // + TIME(0), // + TIMESTAMP(0), // + BIGINT(1), // + UUID(0); // + // // extras + // PARTITION_HASH(2), // + // WRITETIME(1), // + // TTL(1); // + + private final int nrArgs; + private final boolean isVariadic; + + DBFunction(int nrArgs) { + this(nrArgs, false); + } + + DBFunction(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static DBFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + + } + + public enum YCQLUnaryPostfixOperator implements Operator { + + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private final String textRepr; + + YCQLUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static YCQLUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum YCQLUnaryPrefixOperator implements Operator { + + NOT("NOT"), PLUS("+"), MINUS("-"); + + private final String textRepr; + + YCQLUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static YCQLUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum YCQLBinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum YCQLBinaryArithmeticOperator implements Operator { + ADD("+"), SUB("-"), MULT("*"), DIV("/"); + + private String textRepr; + + YCQLBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum YCQLBinaryComparisonOperator implements Operator { + + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); + + private final String textRepr; + + YCQLBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public NewFunctionNode generateArgsForAggregate( + YCQLAggregateFunction aggregateFunction) { + return new NewFunctionNode( + generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + } + + public Node generateAggregate() { + YCQLAggregateFunction aggrFunc = YCQLAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, YCQLUnaryPrefixOperator.NOT); + } + + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, YCQLUnaryPostfixOperator.IS_NULL); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java new file mode 100644 index 000000000..2dc565196 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -0,0 +1,56 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; + +public final class YCQLIndexGenerator { + + private YCQLIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + if (Randomly.getBoolean()) { + errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + sb.append("UNIQUE "); + } + sb.append("INDEX "); + sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); + sb.append(" ON "); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append("("); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + } + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + Node expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(); + sb.append(YCQLToStringVisitor.asString(expr)); + } + errors.add("Query timed out after PT2S"); + errors.add("Invalid SQL Statement"); + errors.add("Invalid CQL Statement"); + errors.add( + "Invalid Table Definition. Transactions cannot be enabled in an index of a table without transactions enabled."); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java new file mode 100644 index 000000000..a27ec9c92 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -0,0 +1,65 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ycql.YCQLErrors; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; + +public class YCQLInsertGenerator extends AbstractInsertGenerator { + + private final YCQLGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public YCQLInsertGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + return new YCQLInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getColumns(); + sb.append(globalState.getDatabaseName()).append(".").append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + + errors.add("Invalid Arguments"); + errors.add("Null Argument for Primary Key"); + + YCQLErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertColumns(List columns) { + sb.append("("); + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + insertValue(columns.get(nrColumn)); + } + sb.append(")"); + } + + @Override + protected void insertValue(YCQLColumn tiDBColumn) { + // TODO: select a more meaningful value + sb.append(YCQLToStringVisitor.asString(new YCQLExpressionGenerator(globalState).generateConstant())); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java new file mode 100644 index 000000000..7a10009ad --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java @@ -0,0 +1,55 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTables; +import sqlancer.yugabyte.ycql.ast.YCQLConstant; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import sqlancer.yugabyte.ycql.ast.YCQLSelect; + +public final class YCQLRandomQuerySynthesizer { + + private YCQLRandomQuerySynthesizer() { + } + + public static YCQLSelect generateSelect(YCQLGlobalState globalState, int nrColumns) { + YCQLTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YCQLExpressionGenerator gen = new YCQLExpressionGenerator(globalState).setColumns(targetTables.getColumns()); + YCQLSelect select = new YCQLSelect(); + List> columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + Node expression = gen.generateExpression(); + columns.add(expression); + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + Optional> table = tables.stream() + .map(t -> new TableReferenceNode(t)).findFirst(); + select.setFromList(table.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(Randomly.nonEmptySubset(select.getFetchColumns())); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YCQLConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(YCQLConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + return select; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java new file mode 100644 index 000000000..148a1cdbc --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java @@ -0,0 +1,57 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLCompositeDataType; + +public class YCQLTableGenerator { + + public SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + sb.append(columns.get(i).getType()); + // todo PK, STATIC + } + errors.add("Query timed out after PT2S"); + errors.add("Invalid type for index"); + List primaryKeyColumns = Randomly.nonEmptySubset(columns); + sb.append(", PRIMARY KEY("); + sb.append(primaryKeyColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(")"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + YCQLCompositeDataType columnType = YCQLCompositeDataType.getRandom(); + columns.add(new YCQLColumn(columnName, columnType, false, false)); + } + return columns; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java new file mode 100644 index 000000000..d417ca96a --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -0,0 +1,56 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLErrors; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; + +public final class YCQLUpdateGenerator { + + private YCQLUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + StringBuilder sb = new StringBuilder("UPDATE "); + ExpectedErrors errors = new ExpectedErrors(); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + YCQLExpressionGenerator gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(" SET "); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append("="); + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + YCQLErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(YCQLToStringVisitor.asString(expr)); + } + + errors.add("Invalid Arguments"); + errors.add("Invalid CQL Statement"); + errors.add("Invalid SQL Statement"); + errors.add("Datatype Mismatch"); + errors.add("Null Argument for Primary Key"); + errors.add("Missing Argument for Primary Key"); + + YCQLErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java new file mode 100644 index 000000000..8dee32c3e --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java @@ -0,0 +1,73 @@ +package sqlancer.yugabyte.ycql.test; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLProvider; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; +import sqlancer.yugabyte.ycql.gen.YCQLRandomQuerySynthesizer; + +public class YCQLFuzzer implements TestOracle { + private final YCQLProvider.YCQLGlobalState globalState; + private final List testQueries; + private final ExpectedErrors errors = new ExpectedErrors(); + + public YCQLFuzzer(YCQLProvider.YCQLGlobalState globalState) { + this.globalState = globalState; + + errors.add("Query timed out after PT2S"); + errors.add("Datatype Mismatch"); + errors.add("Invalid CQL Statement"); + errors.add("Invalid SQL Statement"); + errors.add("Invalid Arguments"); + errors.add("Invalid Function Call"); + + testQueries = new ArrayList<>(); + + testQueries.add(new SelectQuery()); + testQueries.add(new ActionQuery(YCQLProvider.Action.UPDATE)); + testQueries.add(new ActionQuery(YCQLProvider.Action.DELETE)); + testQueries.add(new ActionQuery(YCQLProvider.Action.INSERT)); + } + + @Override + public void check() throws Exception { + Query s = testQueries.get(globalState.getRandomly().getInteger(0, testQueries.size())); + globalState.executeStatement(s.getQuery(globalState, errors)); + globalState.getManager().incrementSelectQueryCount(); + } + + private static class Query { + public SQLQueryAdapter getQuery(YCQLProvider.YCQLGlobalState state, ExpectedErrors errors) throws Exception { + throw new IllegalAccessException("Should be implemented"); + }; + } + + private static class ActionQuery extends Query { + private final YCQLProvider.Action action; + + ActionQuery(YCQLProvider.Action action) { + this.action = action; + } + + @Override + public SQLQueryAdapter getQuery(YCQLProvider.YCQLGlobalState state, ExpectedErrors errors) throws Exception { + return action.getQuery(state); + } + } + + private static class SelectQuery extends Query { + + @Override + public SQLQueryAdapter getQuery(YCQLProvider.YCQLGlobalState state, ExpectedErrors errors) throws Exception { + return new SQLQueryAdapter( + YCQLToStringVisitor.asString( + YCQLRandomQuerySynthesizer.generateSelect(state, Randomly.smallNumber() + 1)) + ";", + errors); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java b/src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java new file mode 100644 index 000000000..3a8b8946d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql; + +import java.util.Optional; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public final class YSQLCompoundDataType { + + private final YSQLDataType dataType; + private final YSQLCompoundDataType elemType; + private final Integer size; + + private YSQLCompoundDataType(YSQLDataType dataType, YSQLCompoundDataType elemType, Integer size) { + this.dataType = dataType; + this.elemType = elemType; + this.size = size; + } + + public static YSQLCompoundDataType create(YSQLDataType type, int size) { + return new YSQLCompoundDataType(type, null, size); + } + + public static YSQLCompoundDataType create(YSQLDataType type) { + return new YSQLCompoundDataType(type, null, null); + } + + public YSQLDataType getDataType() { + return dataType; + } + + public YSQLCompoundDataType getElemType() { + if (elemType == null) { + throw new AssertionError(); + } + return elemType; + } + + public Optional getSize() { + if (size == null) { + return Optional.empty(); + } else { + return Optional.of(size); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java new file mode 100644 index 000000000..cf175ca39 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java @@ -0,0 +1,152 @@ +package sqlancer.yugabyte.ysql; + +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLSubquery; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; + +public final class YSQLExpectedValueVisitor implements YSQLVisitor { + + private static final int NR_TABS = 0; + private final StringBuilder sb = new StringBuilder(); + + private void print(YSQLExpression expr) { + YSQLToStringVisitor v = new YSQLToStringVisitor(); + v.visit(expr); + sb.append("\t".repeat(NR_TABS)); + sb.append(v.get()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + @Override + public void visit(YSQLConstant constant) { + print(constant); + } + + @Override + public void visit(YSQLPostfixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(YSQLColumnValue c) { + print(c); + } + + @Override + public void visit(YSQLPrefixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(YSQLSelect op) { + visit(op.getWhereClause()); + } + + @Override + public void visit(YSQLOrderByTerm op) { + + } + + @Override + public void visit(YSQLFunction f) { + print(f); + for (int i = 0; i < f.getArguments().length; i++) { + visit(f.getArguments()[i]); + } + } + + @Override + public void visit(YSQLCastOperation cast) { + print(cast); + visit(cast.getExpression()); + } + + @Override + public void visit(YSQLBetweenOperation op) { + print(op); + visit(op.getExpr()); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(YSQLInOperation op) { + print(op); + visit(op.getExpr()); + for (YSQLExpression right : op.getListElements()) { + visit(right); + } + } + + @Override + public void visit(YSQLPostfixText op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(YSQLAggregate op) { + print(op); + for (YSQLExpression expr : op.getArgs()) { + visit(expr); + } + } + + @Override + public void visit(YSQLSimilarTo op) { + print(op); + visit(op.getString()); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + } + + @Override + public void visit(YSQLPOSIXRegularExpression op) { + print(op); + visit(op.getString()); + visit(op.getRegex()); + } + + @Override + public void visit(YSQLFromTable from) { + print(from); + } + + @Override + public void visit(YSQLSubquery subquery) { + print(subquery); + } + + @Override + public void visit(YSQLBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public String get() { + return sb.toString(); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLGlobalState.java b/src/sqlancer/yugabyte/ysql/YSQLGlobalState.java new file mode 100644 index 000000000..093f06aed --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLGlobalState.java @@ -0,0 +1,127 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; + +public class YSQLGlobalState extends SQLGlobalState { + + public static final char IMMUTABLE = 'i'; + public static final char STABLE = 's'; + public static final char VOLATILE = 'v'; + // store and allow filtering by function volatility classifications + private final Map functionsAndTypes = new HashMap<>(); + private List operators = Collections.emptyList(); + private List collates = Collections.emptyList(); + private List opClasses = Collections.emptyList(); + private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + + @Override + public void setConnection(SQLConnection con) { + super.setConnection(con); + try { + this.opClasses = getOpclasses(getConnection()); + this.operators = getOperators(getConnection()); + this.collates = getCollnames(getConnection()); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + @Override + public YSQLSchema readSchema() throws SQLException { + return YSQLSchema.fromConnection(getConnection(), getDatabaseName()); + } + + private List getCollnames(SQLConnection con) throws SQLException { + List opClasses = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("SELECT collname FROM pg_collation WHERE collname LIKE '%utf8' or collname = 'C';")) { + while (rs.next()) { + opClasses.add(rs.getString(1)); + } + } + } + return opClasses; + } + + private List getOpclasses(SQLConnection con) throws SQLException { + List opClasses = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("select opcname FROM pg_opclass;")) { + while (rs.next()) { + opClasses.add(rs.getString(1)); + } + } + } + return opClasses; + } + + private List getOperators(SQLConnection con) throws SQLException { + List opClasses = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { + while (rs.next()) { + opClasses.add(rs.getString(1)); + } + } + } + return opClasses; + } + + public List getOperators() { + return operators; + } + + public String getRandomOperator() { + return Randomly.fromList(operators); + } + + public List getCollates() { + return collates; + } + + public String getRandomCollate() { + return Randomly.fromList(collates); + } + + public List getOpClasses() { + return opClasses; + } + + public String getRandomOpclass() { + return Randomly.fromList(opClasses); + } + + public void addFunctionAndType(String functionName, Character functionType) { + this.functionsAndTypes.put(functionName, functionType); + } + + public Map getFunctionsAndTypes() { + return this.functionsAndTypes; + } + + public void setDefaultAllowedFunctionTypes() { + this.allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + } + + public List getAllowedFunctionTypes() { + return this.allowedFunctionTypes; + } + + public void setAllowedFunctionTypes(List types) { + this.allowedFunctionTypes = types; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLOptions.java b/src/sqlancer/yugabyte/ysql/YSQLOptions.java new file mode 100644 index 000000000..4cd299660 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLOptions.java @@ -0,0 +1,98 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ysql.YSQLOptions.YSQLOracleFactory; +import sqlancer.yugabyte.ysql.oracle.YSQLCatalog; +import sqlancer.yugabyte.ysql.oracle.YSQLFuzzer; +import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; +import sqlancer.yugabyte.ysql.oracle.YSQLPivotedQuerySynthesisOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPAggregateOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPHavingOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPWhereOracle; + +@Parameters(separators = "=", commandDescription = "YSQL (default port: " + YSQLOptions.DEFAULT_PORT + + ", default host: " + YSQLOptions.DEFAULT_HOST) +public class YSQLOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 5433; + + @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) + public boolean allowBulkInsert; + + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for YSQL") + public List oracle = Arrays.asList(YSQLOracleFactory.QUERY_PARTITIONING); + + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) + public boolean testCollations = true; + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the YSQL server", arity = 1) + public String connectionURL = String.format("jdbc:yugabytedb://%s:%d/yugabyte", YSQLOptions.DEFAULT_HOST, + YSQLOptions.DEFAULT_PORT); + + @Override + public List getTestOracleFactory() { + return oracle; + } + + public enum YSQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLFuzzer(globalState); + } + }, + CATALOG { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLCatalog(globalState); + } + }, + NOREC { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLNoRECOracle(globalState); + } + }, + PQS { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + HAVING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + List oracles = new ArrayList<>(); + oracles.add(new YSQLTLPWhereOracle(globalState)); + oracles.add(new YSQLTLPHavingOracle(globalState)); + oracles.add(new YSQLTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java new file mode 100644 index 000000000..278e5316e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -0,0 +1,366 @@ +package sqlancer.yugabyte.ysql; + +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.YSQLOptions.YSQLOracleFactory; +import sqlancer.yugabyte.ysql.gen.YSQLAlterTableGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLAnalyzeGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLCommentGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLDeleteGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLDiscardGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLDropIndexGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLIndexGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLInsertGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLNotifyGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLSequenceGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLSetGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTableGroupGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTransactionGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTruncateGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLUpdateGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLVacuumGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLViewGenerator; + +@AutoService(DatabaseProvider.class) +public class YSQLProvider extends SQLProviderAdapter { + + public static final Object CREATION_LOCK = new Object(); + /** + * Generate only data types and expressions that are understood by PQS. + */ + public static boolean generateOnlyKnown; + protected String entryURL; + protected String username; + protected String password; + protected String entryPath; + protected String host; + protected int port; + protected String testURL; + protected String databaseName; + protected String createDatabaseCommand; + + public YSQLProvider() { + super(YSQLGlobalState.class, YSQLOptions.class); + } + + protected YSQLProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + public static int mapActions(YSQLGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + switch (a) { + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 3); + break; + case DISCARD: + case DROP_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case COMMIT: + nrPerformed = r.getInteger(0, 0); + break; + case ALTER_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case RESET: + nrPerformed = r.getInteger(0, 3); + break; + case ANALYZE: + nrPerformed = r.getInteger(0, 3); + break; + case TABLEGROUP: + nrPerformed = r.getInteger(0, 3); + break; + case DELETE: + case RESET_ROLE: + case VACUUM: + case SET_CONSTRAINTS: + case SET: + case COMMENT_ON: + case NOTIFY: + case LISTEN: + case UNLISTEN: + case CREATE_SEQUENCE: + case TRUNCATE: + nrPerformed = r.getInteger(0, 2); + break; + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + + } + + @Override + public void generateDatabase(YSQLGlobalState globalState) throws Exception { + readFunctions(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + prepareTables(globalState); + } + + @Override + public SQLConnection createDatabase(YSQLGlobalState globalState) throws SQLException { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == YSQLOracleFactory.PQS)) { + generateOnlyKnown = true; + } + + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + entryPath = "/yugabyte"; + entryURL = globalState.getDbmsSpecificOptions().connectionURL; + String entryDatabaseName = entryPath.substring(1); + databaseName = globalState.getDatabaseName(); + + try { + URI uri = new URI(entryURL); + String userInfoURI = uri.getUserInfo(); + String pathURI = uri.getPath(); + if (userInfoURI != null) { + // username and password specified in URL take precedence + if (userInfoURI.contains(":")) { + String[] userInfo = userInfoURI.split(":", 2); + username = userInfo[0]; + password = userInfo[1]; + } else { + username = userInfoURI; + password = null; + } + int userInfoIndex = entryURL.indexOf(userInfoURI); + String preUserInfo = entryURL.substring(0, userInfoIndex); + String postUserInfo = entryURL.substring(userInfoIndex + userInfoURI.length() + 1); + entryURL = preUserInfo + postUserInfo; + } + if (pathURI != null) { + entryPath = pathURI; + } + if (host == null) { + host = uri.getHost(); + } + if (port == MainOptions.NO_SET_PORT) { + port = uri.getPort(); + } + entryURL = String.format("jdbc:yugabytedb://%s:%d/%s", host, port, entryDatabaseName); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + + createDatabaseSync(globalState, entryDatabaseName); + + int databaseIndex = entryURL.indexOf("/" + entryDatabaseName) + 1; + String preDatabaseName = entryURL.substring(0, databaseIndex); + String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); + testURL = preDatabaseName + databaseName + postDatabaseName; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + + return new SQLConnection(createConnectionSafely(testURL, username, password)); + } + + @Override + public String getDBMSName() { + return "ysql"; + } + + // for some reason yugabyte unable to create few databases simultaneously + private void createDatabaseSync(YSQLGlobalState globalState, String entryDatabaseName) throws SQLException { + synchronized (CREATION_LOCK) { + exceptionLessSleep(5000); + + Connection con = createConnectionSafely(entryURL, username, password); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + createDatabaseCommand = getCreateDatabaseCommand(globalState); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + } + } + + private Connection createConnectionSafely(String entryURL, String user, String password) { + Connection con = null; + IllegalStateException lastException = new IllegalStateException("Empty exception"); + long endTime = System.currentTimeMillis() + 30000; + while (System.currentTimeMillis() < endTime) { + try { + con = DriverManager.getConnection(entryURL, user, password); + break; + } catch (SQLException throwables) { + lastException = new IllegalStateException(throwables); + } + } + + if (con == null) { + throw lastException; + } + + return con; + } + + protected void readFunctions(YSQLGlobalState globalState) throws SQLException { + SQLQueryAdapter query = new SQLQueryAdapter("SELECT proname, provolatile FROM pg_proc;"); + SQLancerResultSet rs = query.executeAndGet(globalState); + while (rs.next()) { + String functionName = rs.getString(1); + Character functionType = rs.getString(2).charAt(0); + globalState.addFunctionAndType(functionName, functionType); + } + } + + protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { + synchronized (CREATION_LOCK) { + boolean prevCreationFailed = false; // small optimization - wait only after failed requests + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + if (!prevCreationFailed) { + exceptionLessSleep(5000); + } + + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = YSQLTableGenerator.generate(tableName, generateOnlyKnown, + globalState); + globalState.executeStatement(createTable); + prevCreationFailed = false; + } catch (IgnoreMeException e) { + prevCreationFailed = true; + } + } + } + } + + private void exceptionLessSleep(long timeout) { + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + protected void prepareTables(YSQLGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + YSQLProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 15000;\n")); + } + + private String getCreateDatabaseCommand(YSQLGlobalState state) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE DATABASE ").append(databaseName).append(" "); + if (Randomly.getBoolean() && state.getDbmsSpecificOptions().testCollations) { + sb.append("WITH "); + if (Randomly.getBoolean()) { + sb.append("ENCODING '"); + sb.append(Randomly.fromOptions("utf8")); + sb.append("' "); + } + // disabled due to https://github.com/yugabyte/yugabyte-db/issues/11357 + // if (Randomly.getBoolean()) { + // sb.append("COLOCATED = true "); + // } + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } + } + sb.append(" TEMPLATE template0"); + + } + return sb.toString(); + } + + public enum Action implements AbstractAction { + ANALYZE(YSQLAnalyzeGenerator::create), // + ALTER_TABLE(g -> YSQLAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g)), // + COMMIT(g -> { + SQLQueryAdapter query; + if (Randomly.getBoolean()) { + query = new SQLQueryAdapter("COMMIT", true); + } else if (Randomly.getBoolean()) { + query = YSQLTransactionGenerator.executeBegin(); + } else { + query = new SQLQueryAdapter("ROLLBACK", true); + } + return query; + }), // + DELETE(YSQLDeleteGenerator::create), // + DISCARD(YSQLDiscardGenerator::create), // + DROP_INDEX(YSQLDropIndexGenerator::create), // + CREATE_INDEX(YSQLIndexGenerator::generate), // + INSERT(YSQLInsertGenerator::insert), // + UPDATE(YSQLUpdateGenerator::create), // + TRUNCATE(YSQLTruncateGenerator::create), // + TABLEGROUP(YSQLTableGroupGenerator::create), // + VACUUM(YSQLVacuumGenerator::create), // + SET(YSQLSetGenerator::create), // TODO insert yugabyte sets + SET_CONSTRAINTS((g) -> { + String sb = "SET CONSTRAINTS ALL " + Randomly.fromOptions("DEFERRED", "IMMEDIATE"); + return new SQLQueryAdapter(sb); + }), // + RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // + COMMENT_ON(YSQLCommentGenerator::generate), // + RESET((g) -> new SQLQueryAdapter("RESET ALL") /* + * https://www.postgres.org/docs/devel/sql-reset.html TODO: also + * configuration parameter + */), // + NOTIFY(YSQLNotifyGenerator::createNotify), // + LISTEN((g) -> YSQLNotifyGenerator.createListen()), // + UNLISTEN((g) -> YSQLNotifyGenerator.createUnlisten()), // + CREATE_SEQUENCE(YSQLSequenceGenerator::createSequence), // + CREATE_VIEW(YSQLViewGenerator::create); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(YSQLGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLSchema.java b/src/sqlancer/yugabyte/ysql/YSQLSchema.java new file mode 100644 index 000000000..6aba5ac7d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLSchema.java @@ -0,0 +1,329 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.postgresql.util.PSQLException; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; + +public class YSQLSchema extends AbstractSchema { + + private final String databaseName; + + public YSQLSchema(List databaseTables, String databaseName) { + super(databaseTables); + this.databaseName = databaseName; + } + + public static YSQLDataType getColumnType(String typeString) { + switch (typeString) { + case "smallint": + case "integer": + case "bigint": + return YSQLDataType.INT; + case "boolean": + return YSQLDataType.BOOLEAN; + case "text": + case "character": + case "character varying": + case "name": + return YSQLDataType.TEXT; + case "numeric": + return YSQLDataType.DECIMAL; + case "double precision": + return YSQLDataType.FLOAT; + case "real": + return YSQLDataType.REAL; + case "int4range": + return YSQLDataType.RANGE; + case "money": + return YSQLDataType.MONEY; + case "bytea": + return YSQLDataType.BYTEA; + case "bit": + case "bit varying": + return YSQLDataType.BIT; + case "inet": + return YSQLDataType.INET; + default: + throw new AssertionError(typeString); + } + } + + public static YSQLSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + try { + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + "SELECT table_name, table_schema, table_type, is_insertable_into FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + String tableTypeSchema = rs.getString("table_schema"); + boolean isInsertable = rs.getBoolean("is_insertable_into"); + // TODO: also check insertable + // TODO: insert into view? + boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || + // tableTypeStr.contains("LOCAL TEMPORARY") && + // !isInsertable; + YSQLTable.TableType tableType = getTableType(tableTypeSchema); + List databaseColumns = getTableColumns(con, tableName); + List indexes = getIndexes(con, tableName); + List statistics = getStatistics(con); + YSQLTable t = new YSQLTable(tableName, databaseColumns, indexes, tableType, statistics, isView, + isInsertable); + for (YSQLColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + return new YSQLSchema(databaseTables, databaseName); + } catch (SQLIntegrityConstraintViolationException e) { + throw new AssertionError(e); + } + } + + protected static List getStatistics(SQLConnection con) throws SQLException { + List statistics = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT stxname FROM pg_statistic_ext ORDER BY stxname;")) { + while (rs.next()) { + statistics.add(new YSQLStatisticsObject(rs.getString("stxname"))); + } + } + } + return statistics; + } + + protected static YSQLTable.TableType getTableType(String tableTypeStr) throws AssertionError { + YSQLTable.TableType tableType; + if (tableTypeStr.contentEquals("public")) { + tableType = YSQLTable.TableType.STANDARD; + } else if (tableTypeStr.startsWith("pg_temp")) { + tableType = YSQLTable.TableType.TEMPORARY; + } else { + throw new AssertionError(tableTypeStr); + } + return tableType; + } + + protected static List getIndexes(SQLConnection con, String tableName) throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String + .format("SELECT indexname FROM pg_indexes WHERE tablename='%s' ORDER BY indexname;", tableName))) { + while (rs.next()) { + String indexName = rs.getString("indexname"); + if (DBMSCommon.matchesIndexName(indexName)) { + indexes.add(YSQLIndex.create(indexName)); + } + } + } + } + return indexes; + } + + protected static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("select column_name, data_type from INFORMATION_SCHEMA.COLUMNS where table_name = '" + + tableName + "' ORDER BY column_name")) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + YSQLColumn c = new YSQLColumn(columnName, getColumnType(dataType)); + columns.add(c); + } + } + } + return columns; + } + + public YSQLTables getRandomTableNonEmptyTables() { + return new YSQLTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public String getDatabaseName() { + return databaseName; + } + + public enum YSQLDataType { + // TODO: 23.02.2022 Planned types + // SMALLINT, INT, BIGINT, NUMERIC, DECIMAL, REAL, DOUBLE_PRECISION, VARCHAR, CHAR, TEXT, DATE, TIME, + // TIMESTAMP, TIMESTAMPZ, INTERVAL, INTEGER_ARR + INT, BOOLEAN, BYTEA, TEXT, DECIMAL, FLOAT, REAL, RANGE, MONEY, BIT, INET; + + public static YSQLDataType getRandomType() { + List dataTypes = new ArrayList<>(Arrays.asList(values())); + if (YSQLProvider.generateOnlyKnown) { + dataTypes.remove(YSQLDataType.DECIMAL); + dataTypes.remove(YSQLDataType.FLOAT); + dataTypes.remove(YSQLDataType.REAL); + dataTypes.remove(YSQLDataType.INET); + dataTypes.remove(YSQLDataType.RANGE); + dataTypes.remove(YSQLDataType.MONEY); + dataTypes.remove(YSQLDataType.BIT); + } + return Randomly.fromList(dataTypes); + } + } + + public static class YSQLColumn extends AbstractTableColumn { + + public YSQLColumn(String name, YSQLDataType columnType) { + super(name, null, columnType); + } + + public static YSQLColumn createDummy(String name) { + return new YSQLColumn(name, YSQLDataType.INT); + } + + } + + public static class YSQLTables extends AbstractTables { + + public YSQLTables(List tables) { + super(tables); + } + + public YSQLRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String randomRow = String.format("SELECT %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + // columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + + // c.getName() + ")") + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet randomRowValues = s.executeQuery(randomRow); + if (!randomRowValues.next()) { + throw new AssertionError("could not find random row! " + randomRow + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + YSQLColumn column = getColumns().get(i); + int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + YSQLConstant constant; + if (randomRowValues.getString(columnIndex) == null) { + constant = YSQLConstant.createNullConstant(); + } else { + switch (column.getType()) { + case INT: + constant = YSQLConstant.createIntConstant(randomRowValues.getLong(columnIndex)); + break; + case BOOLEAN: + constant = YSQLConstant.createBooleanConstant(randomRowValues.getBoolean(columnIndex)); + break; + case TEXT: + constant = YSQLConstant.createTextConstant(randomRowValues.getString(columnIndex)); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !randomRowValues.next(); + return new YSQLRowValue(this, values); + } catch (PSQLException e) { + throw new IgnoreMeException(); + } + + } + + } + + public static class YSQLRowValue extends AbstractRowValue { + + protected YSQLRowValue(YSQLTables tables, Map values) { + super(tables, values); + } + + } + + public static class YSQLTable extends AbstractRelationalTable { + + private final TableType tableType; + private final List statistics; + private final boolean isInsertable; + + public YSQLTable(String tableName, List columns, List indexes, TableType tableType, + List statistics, boolean isView, boolean isInsertable) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + } + + public List getStatistics() { + return statistics; + } + + public TableType getTableType() { + return tableType; + } + + public boolean isInsertable() { + return isInsertable; + } + + public enum TableType { + STANDARD, TEMPORARY + } + + } + + public static final class YSQLStatisticsObject { + private final String name; + + public YSQLStatisticsObject(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + public static final class YSQLIndex extends TableIndex { + + private YSQLIndex(String indexName) { + super(indexName); + } + + public static YSQLIndex create(String indexName) { + return new YSQLIndex(indexName); + } + + @Override + public String getIndexName() { + if (super.getIndexName().contentEquals("PRIMARY")) { + return "`PRIMARY`"; + } else { + return super.getIndexName(); + } + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java new file mode 100644 index 000000000..39732b728 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java @@ -0,0 +1,329 @@ +package sqlancer.yugabyte.ysql; + +import java.util.Optional; + +import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLJoin.YSQLJoinType; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLSubquery; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; + +public final class YSQLToStringVisitor extends ToStringVisitor implements YSQLVisitor { + + @Override + public void visitSpecific(YSQLExpression expr) { + YSQLVisitor.super.visit(expr); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(YSQLConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public void visit(YSQLPostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" "); + sb.append(op.getOperatorTextRepresentation()); + } + + @Override + public void visit(YSQLColumnValue c) { + sb.append(c.getColumn().getFullQualifiedName()); + } + + @Override + public void visit(YSQLPrefixOperation op) { + sb.append(op.getTextRepresentation()); + sb.append(" ("); + visit(op.getExpression()); + sb.append(")"); + } + + @Override + public void visit(YSQLSelect s) { + sb.append("SELECT "); + switch (s.getSelectOption()) { + case DISTINCT: + sb.append("DISTINCT "); + if (s.getDistinctOnClause() != null) { + sb.append("ON ("); + visit(s.getDistinctOnClause()); + sb.append(") "); + } + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + if (s.getFetchColumns() == null) { + sb.append("*"); + } else { + visit(s.getFetchColumns()); + } + sb.append(" FROM "); + visit(s.getFromList()); + + for (YSQLJoin j : s.getJoinClauses()) { + sb.append(" "); + switch (j.getType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } + sb.append("JOIN"); + break; + case LEFT: + sb.append("LEFT OUTER JOIN"); + break; + case RIGHT: + sb.append("RIGHT OUTER JOIN"); + break; + case FULL: + sb.append("FULL OUTER JOIN"); + break; + case CROSS: + sb.append("CROSS JOIN"); + break; + default: + throw new AssertionError(j.getType()); + } + sb.append(" "); + visit(j.getTableReference()); + if (j.getType() != YSQLJoinType.CROSS) { + sb.append(" ON "); + visit(j.getOnClause()); + } + } + + if (s.getWhereClause() != null) { + sb.append(" WHERE "); + visit(s.getWhereClause()); + } + if (s.getGroupByExpressions().size() > 0) { + sb.append(" GROUP BY "); + visit(s.getGroupByExpressions()); + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + + } + if (!s.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(s.getOrderByExpressions()); + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(YSQLOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder()); + } + + @Override + public void visit(YSQLFunction f) { + sb.append(f.getFunctionName()); + sb.append("("); + int i = 0; + for (YSQLExpression arg : f.getArguments()) { + if (i++ != 0) { + sb.append(", "); + } + visit(arg); + } + sb.append(")"); + } + + @Override + public void visit(YSQLCastOperation cast) { + if (Randomly.getBoolean()) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + appendType(cast); + sb.append(")"); + } else { + sb.append("("); + visit(cast.getExpression()); + sb.append(")::"); + appendType(cast); + } + } + + @Override + public void visit(YSQLBetweenOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(") BETWEEN "); + if (op.isSymmetric()) { + sb.append("SYMMETRIC "); + } + sb.append("("); + visit(op.getLeft()); + sb.append(") AND ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(YSQLInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN ("); + visit(op.getListElements()); + sb.append(")"); + } + + @Override + public void visit(YSQLPostfixText op) { + visit(op.getExpr()); + sb.append(op.getText()); + } + + @Override + public void visit(YSQLAggregate op) { + sb.append(op.getFunction()); + sb.append("("); + visit(op.getArgs()); + sb.append(")"); + } + + @Override + public void visit(YSQLSimilarTo op) { + sb.append("("); + visit(op.getString()); + sb.append(" SIMILAR TO "); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + sb.append(")"); + } + + @Override + public void visit(YSQLPOSIXRegularExpression op) { + visit(op.getString()); + sb.append(op.getOp().getStringRepresentation()); + visit(op.getRegex()); + } + + @Override + public void visit(YSQLFromTable from) { + if (from.isOnly()) { + sb.append("ONLY "); + } + sb.append(from.getTable().getName()); + if (!from.isOnly() && Randomly.getBoolean()) { + sb.append("*"); + } + } + + @Override + public void visit(YSQLSubquery subquery) { + sb.append("("); + visit(subquery.getSelect()); + sb.append(") AS "); + sb.append(subquery.getName()); + } + + @Override + public void visit(YSQLBinaryLogicalOperation op) { + super.visit((BinaryOperation) op); + } + + private void appendType(YSQLCastOperation cast) { + YSQLCompoundDataType compoundType = cast.getCompoundType(); + switch (compoundType.getDataType()) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: // TODO support also other int types + sb.append("INT"); + break; + case TEXT: + // TODO: append TEXT, CHAR + sb.append(Randomly.fromOptions("VARCHAR")); + break; + case REAL: + sb.append("REAL"); + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + sb.append("FLOAT"); + break; + case RANGE: + sb.append("int4range"); + break; + case MONEY: + sb.append("MONEY"); + break; + case INET: + sb.append("INET"); + break; + case BIT: + sb.append("BIT"); + break; + case BYTEA: + sb.append("BYTEA"); + break; + // if (Randomly.getBoolean()) { + // sb.append("("); + // sb.append(Randomly.getNotCachedInteger(1, 100)); + // sb.append(")"); + // } + default: + throw new AssertionError(cast.getType()); + } + Optional size = compoundType.getSize(); + if (size.isPresent()) { + sb.append("("); + sb.append(size.get()); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLVisitor.java new file mode 100644 index 000000000..a73aade4c --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLVisitor.java @@ -0,0 +1,120 @@ +package sqlancer.yugabyte.ysql; + +import java.util.List; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLSubquery; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public interface YSQLVisitor { + + static String asString(YSQLExpression expr) { + YSQLToStringVisitor visitor = new YSQLToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + static String asExpectedValues(YSQLExpression expr) { + YSQLExpectedValueVisitor v = new YSQLExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + + static String getExpressionAsString(YSQLGlobalState globalState, YSQLDataType type, List columns) { + YSQLExpression expression = YSQLExpressionGenerator.generateExpression(globalState, columns, type); + YSQLToStringVisitor visitor = new YSQLToStringVisitor(); + visitor.visit(expression); + return visitor.get(); + } + + void visit(YSQLConstant constant); + + void visit(YSQLPostfixOperation op); + + void visit(YSQLColumnValue c); + + void visit(YSQLPrefixOperation op); + + void visit(YSQLSelect op); + + void visit(YSQLOrderByTerm op); + + void visit(YSQLFunction f); + + void visit(YSQLCastOperation cast); + + void visit(YSQLBetweenOperation op); + + void visit(YSQLInOperation op); + + void visit(YSQLPostfixText op); + + void visit(YSQLAggregate op); + + void visit(YSQLSimilarTo op); + + void visit(YSQLPOSIXRegularExpression op); + + void visit(YSQLFromTable from); + + void visit(YSQLSubquery subquery); + + void visit(YSQLBinaryLogicalOperation op); + + default void visit(YSQLExpression expression) { + if (expression instanceof YSQLConstant) { + visit((YSQLConstant) expression); + } else if (expression instanceof YSQLPostfixOperation) { + visit((YSQLPostfixOperation) expression); + } else if (expression instanceof YSQLColumnValue) { + visit((YSQLColumnValue) expression); + } else if (expression instanceof YSQLPrefixOperation) { + visit((YSQLPrefixOperation) expression); + } else if (expression instanceof YSQLSelect) { + visit((YSQLSelect) expression); + } else if (expression instanceof YSQLOrderByTerm) { + visit((YSQLOrderByTerm) expression); + } else if (expression instanceof YSQLFunction) { + visit((YSQLFunction) expression); + } else if (expression instanceof YSQLCastOperation) { + visit((YSQLCastOperation) expression); + } else if (expression instanceof YSQLBetweenOperation) { + visit((YSQLBetweenOperation) expression); + } else if (expression instanceof YSQLInOperation) { + visit((YSQLInOperation) expression); + } else if (expression instanceof YSQLAggregate) { + visit((YSQLAggregate) expression); + } else if (expression instanceof YSQLPostfixText) { + visit((YSQLPostfixText) expression); + } else if (expression instanceof YSQLSimilarTo) { + visit((YSQLSimilarTo) expression); + } else if (expression instanceof YSQLPOSIXRegularExpression) { + visit((YSQLPOSIXRegularExpression) expression); + } else if (expression instanceof YSQLFromTable) { + visit((YSQLFromTable) expression); + } else if (expression instanceof YSQLSubquery) { + visit((YSQLSubquery) expression); + } else { + throw new AssertionError(expression); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java b/src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java new file mode 100644 index 000000000..27daefcf9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java @@ -0,0 +1,58 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate.YSQLAggregateFunction; + +/** + * @see Built-in Aggregate Functions + */ +public class YSQLAggregate extends FunctionNode implements YSQLExpression { + + public YSQLAggregate(List args, YSQLAggregateFunction func) { + super(func, args); + } + + public enum YSQLAggregateFunction { + AVG(YSQLDataType.INT, YSQLDataType.FLOAT, YSQLDataType.REAL, YSQLDataType.DECIMAL), BIT_AND(YSQLDataType.INT), + BIT_OR(YSQLDataType.INT), BOOL_AND(YSQLDataType.BOOLEAN), BOOL_OR(YSQLDataType.BOOLEAN), + COUNT(YSQLDataType.INT), EVERY(YSQLDataType.BOOLEAN), MAX, MIN, + // STRING_AGG + SUM(YSQLDataType.INT, YSQLDataType.FLOAT, YSQLDataType.REAL, YSQLDataType.DECIMAL); + + private final YSQLDataType[] supportedReturnTypes; + + YSQLAggregateFunction(YSQLDataType... supportedReturnTypes) { + this.supportedReturnTypes = supportedReturnTypes.clone(); + } + + public static List getAggregates(YSQLDataType type) { + return Arrays.stream(values()).filter(p -> p.supportsReturnType(type)).collect(Collectors.toList()); + } + + public List getTypes(YSQLDataType returnType) { + return Collections.singletonList(returnType); + } + + public boolean supportsReturnType(YSQLDataType returnType) { + return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType) + || supportedReturnTypes.length == 0; + } + + public YSQLDataType getRandomReturnType() { + if (supportedReturnTypes.length == 0) { + return Randomly.fromOptions(YSQLDataType.getRandomType()); + } else { + return Randomly.fromOptions(supportedReturnTypes); + } + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java b/src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java new file mode 100644 index 000000000..96e432370 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java @@ -0,0 +1,35 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.common.visitor.UnaryOperation; + +public class YSQLAlias implements UnaryOperation, YSQLExpression { + + private final YSQLExpression expr; + private final String alias; + + public YSQLAlias(YSQLExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + @Override + public YSQLExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return " as " + alias; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java new file mode 100644 index 000000000..4a6c123d0 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java @@ -0,0 +1,63 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public final class YSQLBetweenOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final YSQLExpression left; + private final YSQLExpression right; + private final boolean isSymmetric; + + public YSQLBetweenOperation(YSQLExpression expr, YSQLExpression left, YSQLExpression right, boolean symmetric) { + this.expr = expr; + this.left = left; + this.right = right; + isSymmetric = symmetric; + } + + public YSQLExpression getExpr() { + return expr; + } + + public YSQLExpression getLeft() { + return left; + } + + public YSQLExpression getRight() { + return right; + } + + public boolean isSymmetric() { + return isSymmetric; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLBinaryComparisonOperation leftComparison = new YSQLBinaryComparisonOperation(left, expr, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryComparisonOperation rightComparison = new YSQLBinaryComparisonOperation(expr, right, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryLogicalOperation andOperation = new YSQLBinaryLogicalOperation(leftComparison, rightComparison, + YSQLBinaryLogicalOperation.BinaryLogicalOperator.AND); + if (isSymmetric) { + YSQLBinaryComparisonOperation leftComparison2 = new YSQLBinaryComparisonOperation(right, expr, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryComparisonOperation rightComparison2 = new YSQLBinaryComparisonOperation(expr, left, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryLogicalOperation andOperation2 = new YSQLBinaryLogicalOperation(leftComparison2, rightComparison2, + YSQLBinaryLogicalOperation.BinaryLogicalOperator.AND); + YSQLBinaryLogicalOperation orOp = new YSQLBinaryLogicalOperation(andOperation, andOperation2, + YSQLBinaryLogicalOperation.BinaryLogicalOperator.OR); + return orOp.getExpectedValue(); + } else { + return andOperation.getExpectedValue(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java new file mode 100644 index 000000000..a4385c86c --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java @@ -0,0 +1,106 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryArithmeticOperation.YSQLBinaryOperator; + +public class YSQLBinaryArithmeticOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryArithmeticOperation(YSQLExpression left, YSQLExpression right, YSQLBinaryOperator op) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpected = getLeft().getExpectedValue(); + YSQLConstant rightExpected = getRight().getExpectedValue(); + if (leftExpected == null || rightExpected == null) { + return null; + } + return getOp().apply(leftExpected, rightExpected); + } + + public enum YSQLBinaryOperator implements Operator { + + ADDITION("+") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, Long::sum); + } + + }, + SUBTRACTION("-") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + + } + + }, + MODULO("%") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + + } + }, + EXPONENTIATION("^") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return null; + } + }; + + private final String textRepresentation; + + YSQLBinaryOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + private static YSQLConstant applyBitOperation(YSQLConstant left, YSQLConstant right, BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + long leftVal = left.cast(YSQLDataType.INT).asInt(); + long rightVal = right.cast(YSQLDataType.INT).asInt(); + long value = op.apply(leftVal, rightVal); + return YSQLConstant.createIntConstant(value); + } + } + + public static YSQLBinaryOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public abstract YSQLConstant apply(YSQLConstant left, YSQLConstant right); + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java new file mode 100644 index 000000000..0d1cb8f2e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java @@ -0,0 +1,46 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryBitOperation.YSQLBinaryBitOperator; + +public class YSQLBinaryBitOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryBitOperation(YSQLBinaryBitOperator op, YSQLExpression left, YSQLExpression right) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BIT; + } + + public enum YSQLBinaryBitOperator implements Operator { + CONCATENATION("||"), // + BITWISE_AND("&"), // + BITWISE_OR("|"), // + BITWISE_XOR("#"), // + BITWISE_SHIFT_LEFT("<<"), // + BITWISE_SHIFT_RIGHT(">>"); + + private final String text; + + YSQLBinaryBitOperator(String text) { + this.text = text; + } + + public static YSQLBinaryBitOperator getRandom() { + return Randomly.fromOptions(YSQLBinaryBitOperator.values()); + } + + @Override + public String getTextRepresentation() { + return text; + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java new file mode 100644 index 000000000..8ced603ac --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java @@ -0,0 +1,135 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator; + +public class YSQLBinaryComparisonOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryComparisonOperation(YSQLExpression left, YSQLExpression right, YSQLBinaryComparisonOperator op) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpectedValue = getLeft().getExpectedValue(); + YSQLConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); + } + + public enum YSQLBinaryComparisonOperator implements Operator { + EQUALS("=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + return leftVal.isEquals(rightVal); + } + }, + IS_DISTINCT("IS DISTINCT FROM") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + return YSQLConstant + .createBooleanConstant(!IS_NOT_DISTINCT.getExpectedValue(leftVal, rightVal).asBoolean()); + } + }, + IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + if (leftVal.isNull()) { + return YSQLConstant.createBooleanConstant(rightVal.isNull()); + } else if (rightVal.isNull()) { + return YSQLConstant.createFalse(); + } else { + return leftVal.isEquals(rightVal); + } + } + }, + NOT_EQUALS("!=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant isEquals = leftVal.isEquals(rightVal); + if (isEquals.isBoolean()) { + return YSQLConstant.createBooleanConstant(!isEquals.asBoolean()); + } + return isEquals; + } + }, + LESS("<") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + return leftVal.isLessThan(rightVal); + } + }, + LESS_EQUALS("<=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant lessThan = leftVal.isLessThan(rightVal); + if (lessThan.isBoolean() && !lessThan.asBoolean()) { + return leftVal.isEquals(rightVal); + } else { + return lessThan; + } + } + }, + GREATER(">") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return YSQLConstant.createFalse(); + } else { + YSQLConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return YSQLConstant.createNullConstant(); + } + return YSQLPrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + }, + GREATER_EQUALS(">=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return YSQLConstant.createTrue(); + } else { + YSQLConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return YSQLConstant.createNullConstant(); + } + return YSQLPrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + + }; + + private final String textRepresentation; + + YSQLBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static YSQLBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(YSQLBinaryComparisonOperator.values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public abstract YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal); + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java new file mode 100644 index 000000000..89cce762f --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java @@ -0,0 +1,88 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation.BinaryLogicalOperator; + +public class YSQLBinaryLogicalOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryLogicalOperation(YSQLExpression left, YSQLExpression right, BinaryLogicalOperator op) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpectedValue = getLeft().getExpectedValue(); + YSQLConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + + public enum BinaryLogicalOperator implements Operator { + AND { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + YSQLConstant leftBool = left.cast(YSQLDataType.BOOLEAN); + YSQLConstant rightBool = right.cast(YSQLDataType.BOOLEAN); + if (leftBool.isNull()) { + if (rightBool.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + if (rightBool.asBoolean()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createFalse(); + } + } + } else if (!leftBool.asBoolean()) { + return YSQLConstant.createFalse(); + } + assert leftBool.asBoolean(); + if (rightBool.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createBooleanConstant(rightBool.isBoolean() && rightBool.asBoolean()); + } + } + }, + OR { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + YSQLConstant leftBool = left.cast(YSQLDataType.BOOLEAN); + YSQLConstant rightBool = right.cast(YSQLDataType.BOOLEAN); + if (leftBool.isBoolean() && leftBool.asBoolean()) { + return YSQLConstant.createTrue(); + } + if (rightBool.isBoolean() && rightBool.asBoolean()) { + return YSQLConstant.createTrue(); + } + if (leftBool.isNull() || rightBool.isNull()) { + return YSQLConstant.createNullConstant(); + } + return YSQLConstant.createFalse(); + } + }; + + public static BinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract YSQLConstant apply(YSQLConstant left, YSQLConstant right); + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java new file mode 100644 index 000000000..4bf4a8ea6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java @@ -0,0 +1,71 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLBinaryRangeOperation extends BinaryNode implements YSQLExpression { + + private final String op; + + public YSQLBinaryRangeOperation(YSQLBinaryRangeComparisonOperator op, YSQLExpression left, YSQLExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + public YSQLBinaryRangeOperation(YSQLBinaryRangeOperator op, YSQLExpression left, YSQLExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public String getOperatorRepresentation() { + return op; + } + + public enum YSQLBinaryRangeOperator implements Operator { + UNION("+"), INTERSECTION("*"), DIFFERENCE("-"); + + private final String textRepresentation; + + YSQLBinaryRangeOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static YSQLBinaryRangeOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum YSQLBinaryRangeComparisonOperator { + CONTAINS_RANGE_OR_ELEMENT("@>"), RANGE_OR_ELEMENT_IS_CONTAINED("<@"), OVERLAP("&&"), STRICT_LEFT_OF("<<"), + STRICT_RIGHT_OF(">>"), NOT_RIGHT_OF("&<"), NOT_LEFT_OF(">&"), ADJACENT("-|-"); + + private final String textRepresentation; + + YSQLBinaryRangeComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static YSQLBinaryRangeComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java new file mode 100644 index 000000000..cacd8ad76 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLCastOperation implements YSQLExpression { + + private final YSQLExpression expression; + private final YSQLCompoundDataType type; + + public YSQLCastOperation(YSQLExpression expression, YSQLCompoundDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + @Override + public YSQLDataType getExpressionType() { + return type.getDataType(); + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant expectedValue = expression.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type.getDataType()); + } + + public YSQLExpression getExpression() { + return expression; + } + + public YSQLDataType getType() { + return type.getDataType(); + } + + public YSQLCompoundDataType getCompoundType() { + return type; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java b/src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java new file mode 100644 index 000000000..82b164c54 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java @@ -0,0 +1,33 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLCollate implements YSQLExpression { + + private final YSQLExpression expr; + private final String collate; + + public YSQLCollate(YSQLExpression expr, String collate) { + this.expr = expr; + this.collate = collate; + } + + public String getCollate() { + return collate; + } + + public YSQLExpression getExpr() { + return expr; + } + + @Override + public YSQLDataType getExpressionType() { + return expr.getExpressionType(); + } + + @Override + public YSQLConstant getExpectedValue() { + return null; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java b/src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java new file mode 100644 index 000000000..243bdaf57 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java @@ -0,0 +1,34 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLColumnValue implements YSQLExpression { + + private final YSQLColumn c; + private final YSQLConstant expectedValue; + + public YSQLColumnValue(YSQLColumn c, YSQLConstant expectedValue) { + this.c = c; + this.expectedValue = expectedValue; + } + + public static YSQLColumnValue create(YSQLColumn c, YSQLConstant expected) { + return new YSQLColumnValue(c, expected); + } + + @Override + public YSQLDataType getExpressionType() { + return c.getType(); + } + + @Override + public YSQLConstant getExpectedValue() { + return expectedValue; + } + + public YSQLColumn getColumn() { + return c; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java new file mode 100644 index 000000000..a74263b22 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java @@ -0,0 +1,37 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.common.ast.BinaryNode; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLConcatOperation extends BinaryNode implements YSQLExpression { + + public YSQLConcatOperation(YSQLExpression left, YSQLExpression right) { + super(left, right); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.TEXT; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpectedValue = getLeft().getExpectedValue(); + YSQLConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + if (leftExpectedValue.isNull() || rightExpectedValue.isNull()) { + return YSQLConstant.createNullConstant(); + } + String leftStr = leftExpectedValue.cast(YSQLDataType.TEXT).getUnquotedTextRepresentation(); + String rightStr = rightExpectedValue.cast(YSQLDataType.TEXT).getUnquotedTextRepresentation(); + return YSQLConstant.createTextConstant(leftStr + rightStr); + } + + @Override + public String getOperatorRepresentation() { + return "||"; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java b/src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java new file mode 100644 index 000000000..9fd2eaad6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java @@ -0,0 +1,611 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.math.BigDecimal; + +import sqlancer.IgnoreMeException; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public abstract class YSQLConstant implements YSQLExpression { + + public static YSQLConstant createNullConstant() { + return new YSQLNullConstant(); + } + + public static YSQLConstant createIntConstant(long val) { + return new IntConstant(val); + } + + public static YSQLConstant createBooleanConstant(boolean val) { + return new BooleanConstant(val); + } + + public static YSQLConstant createFalse() { + return createBooleanConstant(false); + } + + public static YSQLConstant createTrue() { + return createBooleanConstant(true); + } + + public static YSQLConstant createTextConstant(String string) { + return new StringConstant(string); + } + + public static YSQLConstant createByteConstant(String string) { + return new ByteConstant(string); + } + + public static YSQLConstant createDecimalConstant(BigDecimal bigDecimal) { + return new DecimalConstant(bigDecimal); + } + + public static YSQLConstant createFloatConstant(float val) { + return new FloatConstant(val); + } + + public static YSQLConstant createDoubleConstant(double val) { + return new DoubleConstant(val); + } + + public static YSQLConstant createRange(long left, boolean leftIsInclusive, long right, boolean rightIsInclusive) { + long realLeft; + long realRight; + if (left > right) { + realRight = left; + realLeft = right; + } else { + realLeft = left; + realRight = right; + } + return new RangeConstant(realLeft, leftIsInclusive, realRight, rightIsInclusive); + } + + public static YSQLExpression createBitConstant(long integer) { + return new BitConstant(integer); + } + + public static YSQLExpression createInetConstant(String val) { + return new InetConstant(val); + } + + public abstract String getTextRepresentation(); + + public abstract String getUnquotedTextRepresentation(); + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isString() { + return false; + } + + @Override + public YSQLConstant getExpectedValue() { + return this; + } + + public boolean isNull() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isBoolean() { + return false; + } + + public abstract YSQLConstant isEquals(YSQLConstant rightVal); + + public boolean isInt() { + return false; + } + + protected abstract YSQLConstant isLessThan(YSQLConstant rightVal); + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract YSQLConstant cast(YSQLDataType type); + + public static class BooleanConstant extends YSQLConstant { + + private final boolean value; + + public BooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return value ? "TRUE" : "FALSE"; + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return YSQLConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(value == rightVal.cast(YSQLDataType.BOOLEAN).asBoolean()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isString()) { + return isLessThan(rightVal.cast(YSQLDataType.BOOLEAN)); + } else { + assert rightVal.isBoolean(); + return YSQLConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + switch (type) { + case BOOLEAN: + return this; + case INT: + return YSQLConstant.createIntConstant(value ? 1 : 0); + case TEXT: + return YSQLConstant.createTextConstant(value ? "true" : "false"); + default: + return null; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + } + + public static class YSQLNullConstant extends YSQLConstant { + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + return YSQLConstant.createNullConstant(); + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + return YSQLConstant.createNullConstant(); + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + return YSQLConstant.createNullConstant(); + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + + } + + public static class StringConstant extends YSQLConstant { + + protected final String value; + + public StringConstant(String value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'", value.replace("'", "''")); + } + + @Override + public String getUnquotedTextRepresentation() { + return value; + } + + @Override + public String asString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(YSQLDataType.INT).isEquals(rightVal.cast(YSQLDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(YSQLDataType.BOOLEAN).isEquals(rightVal.cast(YSQLDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(YSQLDataType.INT).isLessThan(rightVal.cast(YSQLDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(YSQLDataType.BOOLEAN).isLessThan(rightVal.cast(YSQLDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + if (type == YSQLDataType.TEXT) { + return this; + } + String s = value.trim(); + switch (type) { + case BOOLEAN: + try { + return YSQLConstant.createBooleanConstant(Long.parseLong(s) != 0); + } catch (NumberFormatException e) { + } + switch (s.toUpperCase()) { + case "T": + case "TR": + case "TRU": + case "TRUE": + case "1": + case "YES": + case "YE": + case "Y": + case "ON": + return YSQLConstant.createTrue(); + case "F": + case "FA": + case "FAL": + case "FALS": + case "FALSE": + case "N": + case "NO": + case "OF": + case "OFF": + default: + return YSQLConstant.createFalse(); + } + case INT: + try { + return YSQLConstant.createIntConstant(Long.parseLong(s)); + } catch (NumberFormatException e) { + return YSQLConstant.createIntConstant(-1); + } + case TEXT: + return this; + default: + return null; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.TEXT; + } + + } + + public static class IntConstant extends YSQLConstant { + + private final long val; + + public IntConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + @Override + public long asInt() { + return val; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(YSQLDataType.BOOLEAN).isEquals(rightVal); + } else if (rightVal.isInt()) { + return YSQLConstant.createBooleanConstant(val == rightVal.asInt()); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(val == rightVal.cast(YSQLDataType.INT).asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public boolean isInt() { + return true; + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return YSQLConstant.createBooleanConstant(val < rightVal.asInt()); + } else if (rightVal.isBoolean()) { + throw new AssertionError(rightVal); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(val < rightVal.cast(YSQLDataType.INT).asInt()); + } else { + throw new IgnoreMeException(); + } + + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + switch (type) { + case BOOLEAN: + return YSQLConstant.createBooleanConstant(val != 0); + case INT: + return this; + case TEXT: + return YSQLConstant.createTextConstant(String.valueOf(val)); + default: + return null; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + } + + public static class ByteConstant extends StringConstant { + + public ByteConstant(String value) { + super(value); + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'::bytea", value.replace("'", "''")); + } + } + + public abstract static class YSQLConstantBase extends YSQLConstant { + + @Override + public String getUnquotedTextRepresentation() { + return null; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + return null; + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + return null; + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + return null; + } + } + + public static class DecimalConstant extends YSQLConstantBase { + + private final BigDecimal val; + + public DecimalConstant(BigDecimal val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.DECIMAL; + } + + } + + public static class InetConstant extends YSQLConstantBase { + + private final String val; + + public InetConstant(String val) { + this.val = "'" + val + "'"; + } + + @Override + public String getTextRepresentation() { + return val; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INET; + } + + } + + public static class FloatConstant extends YSQLConstantBase { + + private final float val; + + public FloatConstant(float val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.FLOAT; + } + + } + + public static class DoubleConstant extends YSQLConstantBase { + + private final double val; + + public DoubleConstant(double val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.FLOAT; + } + + } + + public static class BitConstant extends YSQLConstantBase { + + private final long val; + + public BitConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.format("B'%s'", Long.toBinaryString(val)); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BIT; + } + + } + + public static class RangeConstant extends YSQLConstantBase { + + private final long left; + private final boolean leftIsInclusive; + private final long right; + private final boolean rightIsInclusive; + + public RangeConstant(long left, boolean leftIsInclusive, long right, boolean rightIsInclusive) { + this.left = left; + this.leftIsInclusive = leftIsInclusive; + this.right = right; + this.rightIsInclusive = rightIsInclusive; + } + + @Override + public String getTextRepresentation() { + StringBuilder sb = new StringBuilder(); + sb.append("'"); + if (leftIsInclusive) { + sb.append("["); + } else { + sb.append("("); + } + sb.append(left); + sb.append(","); + sb.append(right); + if (rightIsInclusive) { + sb.append("]"); + } else { + sb.append(")"); + } + sb.append("'"); + sb.append("::int4range"); + return sb.toString(); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.RANGE; + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java new file mode 100644 index 000000000..9544c8838 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java @@ -0,0 +1,14 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public interface YSQLExpression { + + default YSQLDataType getExpressionType() { + return null; + } + + default YSQLConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java b/src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java new file mode 100644 index 000000000..54349ebe9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java @@ -0,0 +1,283 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLFunction implements YSQLExpression { + + private final String func; + private final YSQLExpression[] args; + private final YSQLDataType returnType; + private YSQLFunctionWithResult functionWithKnownResult; + + public YSQLFunction(YSQLFunctionWithResult func, YSQLDataType returnType, YSQLExpression... args) { + functionWithKnownResult = func; + this.func = func.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public YSQLFunction(YSQLFunctionWithUnknownResult f, YSQLDataType returnType, YSQLExpression... args) { + this.func = f.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public String getFunctionName() { + return func; + } + + public YSQLExpression[] getArguments() { + return args.clone(); + } + + @Override + public YSQLDataType getExpressionType() { + return returnType; + } + + @Override + public YSQLConstant getExpectedValue() { + if (functionWithKnownResult == null) { + return null; + } + YSQLConstant[] constants = new YSQLConstant[args.length]; + for (int i = 0; i < constants.length; i++) { + constants[i] = args[i].getExpectedValue(); + if (constants[i] == null) { + return null; + } + } + return functionWithKnownResult.apply(constants, args); + } + + public enum YSQLFunctionWithResult { + ABS(1, "abs") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createIntConstant(Math.abs(evaluatedArgs[0].cast(YSQLDataType.INT).asInt())); + } + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { returnType }; + } + + }, + LOWER(1, "lower") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return YSQLConstant.createTextConstant(text.toLowerCase()); + } + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.TEXT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { YSQLDataType.TEXT }; + } + + }, + LENGTH(1, "length") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } + String text = evaluatedArgs[0].asString(); + return YSQLConstant.createIntConstant(text.length()); + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { YSQLDataType.TEXT }; + } + }, + UPPER(1, "upper") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return YSQLConstant.createTextConstant(text.toUpperCase()); + } + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.TEXT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { YSQLDataType.TEXT }; + } + + }, + // NULL_IF(2, "nullif") { + // + // @Override + // public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression[] args) { + // YSQLConstant equals = evaluatedArgs[0].isEquals(evaluatedArgs[1]); + // if (equals.isBoolean() && equals.asBoolean()) { + // return YSQLConstant.createNullConstant(); + // } else { + // // TODO: SELECT (nullif('1', FALSE)); yields '1', but should yield TRUE + // return evaluatedArgs[0]; + // } + // } + // + // @Override + // public boolean supportsReturnType(YSQLDataType type) { + // return true; + // } + // + // @Override + // public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + // return getType(nrArguments, returnType); + // } + // + // @Override + // public boolean checkArguments(YSQLExpression[] constants) { + // for (YSQLExpression e : constants) { + // if (!(e instanceof YSQLNullConstant)) { + // return true; + // } + // } + // return false; + // } + // + // }, + NUM_NONNULLS(1, "num_nonnulls") { + @Override + public YSQLConstant apply(YSQLConstant[] args, YSQLExpression... origArgs) { + int nr = 0; + for (YSQLConstant c : args) { + if (!c.isNull()) { + nr++; + } + } + return YSQLConstant.createIntConstant(nr); + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }, + NUM_NULLS(1, "num_nulls") { + @Override + public YSQLConstant apply(YSQLConstant[] args, YSQLExpression... origArgs) { + int nr = 0; + for (YSQLConstant c : args) { + if (c.isNull()) { + nr++; + } + } + return YSQLConstant.createIntConstant(nr); + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }; + + final int nrArgs; + private final String functionName; + private final boolean variadic; + + YSQLFunctionWithResult(int nrArgs, String functionName) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = false; + } + + public YSQLDataType[] getRandomTypes(int nr) { + YSQLDataType[] types = new YSQLDataType[nr]; + for (int i = 0; i < types.length; i++) { + types[i] = YSQLDataType.getRandomType(); + } + return types; + } + + /** + * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number + * of arguments is returned. + * + * @return the number of arguments + */ + public int getNrArgs() { + return nrArgs; + } + + public abstract YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args); + + @Override + public String toString() { + return functionName; + } + + public boolean isVariadic() { + return variadic; + } + + public String getName() { + return functionName; + } + + public abstract boolean supportsReturnType(YSQLDataType type); + + public abstract YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments); + + public boolean checkArguments(YSQLExpression... constants) { + return true; + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java b/src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java new file mode 100644 index 000000000..8fbd1aa5d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java @@ -0,0 +1,174 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public enum YSQLFunctionWithUnknownResult { + + ABBREV("abbrev", YSQLDataType.TEXT, YSQLDataType.INET), + BROADCAST("broadcast", YSQLDataType.INET, YSQLDataType.INET), FAMILY("family", YSQLDataType.INT, YSQLDataType.INET), + HOSTMASK("hostmask", YSQLDataType.INET, YSQLDataType.INET), MASKLEN("masklen", YSQLDataType.INT, YSQLDataType.INET), + NETMASK("netmask", YSQLDataType.INET, YSQLDataType.INET), + SET_MASKLEN("set_masklen", YSQLDataType.INET, YSQLDataType.INET, YSQLDataType.INT), + TEXT("text", YSQLDataType.TEXT, YSQLDataType.INET), + INET_SAME_FAMILY("inet_same_family", YSQLDataType.BOOLEAN, YSQLDataType.INET, YSQLDataType.INET), + + // https://www.postgres.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-SIGNAL-TABLE + // PG_RELOAD_CONF("pg_reload_conf", YSQLDataType.BOOLEAN), // too much output + // PG_ROTATE_LOGFILE("pg_rotate_logfile", YSQLDataType.BOOLEAN), prints warning + + // https://www.postgresql.org/docs/devel/functions-info.html#FUNCTIONS-INFO-SESSION-TABLE + CURRENT_DATABASE("current_database", YSQLDataType.TEXT), // name + // CURRENT_QUERY("current_query", YSQLDataType.TEXT), // can generate false positives + CURRENT_SCHEMA("current_schema", YSQLDataType.TEXT), // name + // CURRENT_SCHEMAS("current_schemas", YSQLDataType.TEXT, YSQLDataType.BOOLEAN), + INET_CLIENT_PORT("inet_client_port", YSQLDataType.INT), INET_SERVER_PORT("inet_server_port", YSQLDataType.INT), + PG_BACKEND_PID("pg_backend_pid", YSQLDataType.INT), PG_CURRENT_LOGFILE("pg_current_logfile", YSQLDataType.TEXT), + // PG_IS_OTHER_TEMP_SCHEMA("pg_is_other_temp_schema", YSQLDataType.BOOLEAN), + // PG_JIT_AVAILABLE("pg_is_other_temp_schema", YSQLDataType.BOOLEAN), + PG_NOTIFICATION_QUEUE_USAGE("pg_notification_queue_usage", YSQLDataType.REAL), + PG_TRIGGER_DEPTH("pg_trigger_depth", YSQLDataType.INT), VERSION("version", YSQLDataType.TEXT), + + // + TO_CHAR("to_char", YSQLDataType.TEXT, YSQLDataType.BYTEA, YSQLDataType.TEXT) { + @Override + public YSQLExpression[] getArguments(YSQLDataType returnType, YSQLExpressionGenerator gen, int depth) { + YSQLExpression[] args = super.getArguments(returnType, gen, depth); + args[0] = gen.generateExpression(YSQLDataType.getRandomType()); + return args; + } + }, + + // String functions + ASCII("ascii", YSQLDataType.INT, YSQLDataType.TEXT), + BTRIM("btrim", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + CHR("chr", YSQLDataType.TEXT, YSQLDataType.INT), + CONVERT_FROM("convert_from", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT) { + @Override + public YSQLExpression[] getArguments(YSQLDataType returnType, YSQLExpressionGenerator gen, int depth) { + YSQLExpression[] args = super.getArguments(returnType, gen, depth); + args[1] = YSQLConstant.createTextConstant("UTF8"); + return args; + } + }, + // concat + // segfault + BIT_LENGTH("bit_length", YSQLDataType.INT, YSQLDataType.BYTEA), + INITCAP("initcap", YSQLDataType.TEXT, YSQLDataType.TEXT), + LEFT("left", YSQLDataType.TEXT, YSQLDataType.INT, YSQLDataType.TEXT), + LOWER("lower", YSQLDataType.TEXT, YSQLDataType.TEXT), MD5("md5", YSQLDataType.TEXT, YSQLDataType.TEXT), + UPPER("upper", YSQLDataType.TEXT, YSQLDataType.TEXT), + // PG_CLIENT_ENCODING("pg_client_encoding", YSQLDataType.TEXT), + QUOTE_LITERAL("quote_literal", YSQLDataType.TEXT, YSQLDataType.TEXT), + QUOTE_IDENT("quote_ident", YSQLDataType.TEXT, YSQLDataType.TEXT), + REGEX_REPLACE("regexp_replace", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + // todo mute repeat function because it may provide OOMs + // REPEAT("repeat", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT), + REPLACE("replace", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + REVERSE("reverse", YSQLDataType.TEXT, YSQLDataType.TEXT), + RIGHT("right", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT), + RPAD("rpad", YSQLDataType.TEXT, YSQLDataType.INT, YSQLDataType.TEXT), + RTRIM("rtrim", YSQLDataType.TEXT, YSQLDataType.TEXT), + SPLIT_PART("split_part", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT), + STRPOS("strpos", YSQLDataType.INT, YSQLDataType.TEXT, YSQLDataType.TEXT), + SUBSTR("substr", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT, YSQLDataType.INT), + TO_ASCII("to_ascii", YSQLDataType.TEXT, YSQLDataType.TEXT), TO_HEX("to_hex", YSQLDataType.INT, YSQLDataType.TEXT), + TRANSLATE("translate", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + // mathematical functions + // https://www.postgresql.org/docs/9.5/functions-math.html + ABS("abs", YSQLDataType.REAL, YSQLDataType.REAL), CBRT("cbrt", YSQLDataType.REAL, YSQLDataType.REAL), + CEILING("ceiling", YSQLDataType.REAL), // + DEGREES("degrees", YSQLDataType.REAL), EXP("exp", YSQLDataType.REAL), LN("ln", YSQLDataType.REAL), + LOG("log", YSQLDataType.REAL), LOG2("log", YSQLDataType.REAL, YSQLDataType.REAL), PI("pi", YSQLDataType.REAL), + POWER("power", YSQLDataType.REAL, YSQLDataType.REAL), TRUNC("trunc", YSQLDataType.REAL, YSQLDataType.INT), + TRUNC2("trunc", YSQLDataType.REAL, YSQLDataType.INT, YSQLDataType.REAL), FLOOR("floor", YSQLDataType.REAL), + + // trigonometric functions - complete + // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-TRIG-TABLE + ACOS("acos", YSQLDataType.REAL), // + ACOSD("acosd", YSQLDataType.REAL), // + ASIN("asin", YSQLDataType.REAL), // + ASIND("asind", YSQLDataType.REAL), // + ATAN("atan", YSQLDataType.REAL), // + ATAND("atand", YSQLDataType.REAL), // + ATAN2("atan2", YSQLDataType.REAL, YSQLDataType.REAL), // + ATAN2D("atan2d", YSQLDataType.REAL, YSQLDataType.REAL), // + COS("cos", YSQLDataType.REAL), // + COSD("cosd", YSQLDataType.REAL), // + COT("cot", YSQLDataType.REAL), // + COTD("cotd", YSQLDataType.REAL), // + SIN("sin", YSQLDataType.REAL), // + SIND("sind", YSQLDataType.REAL), // + TAN("tan", YSQLDataType.REAL), // + TAND("tand", YSQLDataType.REAL), // + + // hyperbolic functions - complete + // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-HYP-TABLE + SINH("sinh", YSQLDataType.REAL), // + COSH("cosh", YSQLDataType.REAL), // + TANH("tanh", YSQLDataType.REAL), // + ASINH("asinh", YSQLDataType.REAL), // + ACOSH("acosh", YSQLDataType.REAL), // + ATANH("atanh", YSQLDataType.REAL), // + + // https://www.postgresql.org/docs/devel/functions-binarystring.html + GET_BIT("get_bit", YSQLDataType.INT, YSQLDataType.TEXT, YSQLDataType.INT), + GET_BYTE("get_byte", YSQLDataType.INT, YSQLDataType.TEXT, YSQLDataType.INT), + + // range functions + // https://www.postgresql.org/docs/devel/functions-range.html#RANGE-FUNCTIONS-TABLE + RANGE_LOWER("lower", YSQLDataType.INT, YSQLDataType.RANGE), // + RANGE_UPPER("upper", YSQLDataType.INT, YSQLDataType.RANGE), // + RANGE_ISEMPTY("isempty", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_LOWER_INC("lower_inc", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_UPPER_INC("upper_inc", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_LOWER_INF("lower_inf", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_UPPER_INF("upper_inf", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_MERGE("range_merge", YSQLDataType.RANGE, YSQLDataType.RANGE, YSQLDataType.RANGE), // + + // https://www.postgresql.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE + GET_COLUMN_SIZE("get_column_size", YSQLDataType.INT, YSQLDataType.TEXT); + // PG_DATABASE_SIZE("pg_database_size", YSQLDataType.INT, YSQLDataType.INT); + // PG_SIZE_BYTES("pg_size_bytes", YSQLDataType.INT, YSQLDataType.TEXT); + + private final String functionName; + private final YSQLDataType returnType; + private final YSQLDataType[] argTypes; + + YSQLFunctionWithUnknownResult(String functionName, YSQLDataType returnType, YSQLDataType... indexType) { + this.functionName = functionName; + this.returnType = returnType; + this.argTypes = indexType.clone(); + } + + public static List getSupportedFunctions(YSQLDataType type) { + List functions = new ArrayList<>(); + for (YSQLFunctionWithUnknownResult func : values()) { + if (func.isCompatibleWithReturnType(type)) { + functions.add(func); + } + } + return functions; + } + + public boolean isCompatibleWithReturnType(YSQLDataType t) { + return t == returnType; + } + + public YSQLExpression[] getArguments(YSQLDataType returnType, YSQLExpressionGenerator gen, int depth) { + YSQLExpression[] args = new YSQLExpression[argTypes.length]; + for (int i = 0; i < args.length; i++) { + args[i] = gen.generateExpression(depth, argTypes[i]); + } + return args; + + } + + public String getName() { + return functionName; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java new file mode 100644 index 000000000..a8c5b9490 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java @@ -0,0 +1,65 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.List; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLInOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final List listElements; + private final boolean isTrue; + + public YSQLInOperation(YSQLExpression expr, List listElements, boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public YSQLExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + public boolean isTrue() { + return isTrue; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftValue = expr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return YSQLConstant.createNullConstant(); + } + boolean isNull = false; + for (YSQLExpression expr : getListElements()) { + YSQLConstant rightExpectedValue = expr.getExpectedValue(); + if (rightExpectedValue == null) { + return null; + } + if (rightExpectedValue.isNull()) { + isNull = true; + } else if (rightExpectedValue.isEquals(this.expr.getExpectedValue()).isBoolean() + && rightExpectedValue.isEquals(this.expr.getExpectedValue()).asBoolean()) { + return YSQLConstant.createBooleanConstant(isTrue); + } + } + + if (isNull) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createBooleanConstant(!isTrue); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java new file mode 100644 index 000000000..746d46cfd --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java @@ -0,0 +1,49 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLJoin implements YSQLExpression { + + private final YSQLExpression tableReference; + private final YSQLExpression onClause; + private final YSQLJoinType type; + + public YSQLJoin(YSQLExpression tableReference, YSQLExpression onClause, YSQLJoinType type) { + this.tableReference = tableReference; + this.onClause = onClause; + this.type = type; + } + + public YSQLExpression getTableReference() { + return tableReference; + } + + public YSQLExpression getOnClause() { + return onClause; + } + + public YSQLJoinType getType() { + return type; + } + + @Override + public YSQLDataType getExpressionType() { + throw new AssertionError(); + } + + @Override + public YSQLConstant getExpectedValue() { + throw new AssertionError(); + } + + public enum YSQLJoinType { + INNER, LEFT, RIGHT, FULL, CROSS; + + public static YSQLJoinType getRandom() { + return Randomly.fromOptions(values()); + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java b/src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java new file mode 100644 index 000000000..e57a347f9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java @@ -0,0 +1,42 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLOrderByTerm implements YSQLExpression { + + private final YSQLOrder order; + private final YSQLExpression expr; + + public YSQLOrderByTerm(YSQLExpression expr, YSQLOrder order) { + this.expr = expr; + this.order = order; + } + + public YSQLOrder getOrder() { + return order; + } + + public YSQLExpression getExpr() { + return expr; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + + @Override + public YSQLConstant getExpectedValue() { + throw new AssertionError(this); + } + + public enum YSQLOrder { + ASC, DESC; + + public static YSQLOrder getRandomOrder() { + return Randomly.fromOptions(YSQLOrder.values()); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java new file mode 100644 index 000000000..975ac6aaf --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java @@ -0,0 +1,65 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPOSIXRegularExpression implements YSQLExpression { + + private final YSQLExpression string; + private final YSQLExpression regex; + private final POSIXRegex op; + + public YSQLPOSIXRegularExpression(YSQLExpression string, YSQLExpression regex, POSIXRegex op) { + this.string = string; + this.regex = regex; + this.op = op; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + return null; + } + + public YSQLExpression getRegex() { + return regex; + } + + public YSQLExpression getString() { + return string; + } + + public POSIXRegex getOp() { + return op; + } + + public enum POSIXRegex implements Operator { + MATCH_CASE_SENSITIVE("~"), MATCH_CASE_INSENSITIVE("~*"), NOT_MATCH_CASE_SENSITIVE("!~"), + NOT_MATCH_CASE_INSENSITIVE("!~*"); + + private final String repr; + + POSIXRegex(String repr) { + this.repr = repr; + } + + public static POSIXRegex getRandom() { + return Randomly.fromOptions(values()); + } + + public String getStringRepresentation() { + return repr; + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java new file mode 100644 index 000000000..65ab056c5 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java @@ -0,0 +1,146 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPostfixOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final PostfixOperator op; + private final String operatorTextRepresentation; + + public YSQLPostfixOperation(YSQLExpression expr, PostfixOperator op) { + this.expr = expr; + this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); + this.op = op; + } + + public static YSQLExpression create(YSQLExpression expr, PostfixOperator op) { + return new YSQLPostfixOperation(expr, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.apply(expectedValue); + } + + public String getOperatorTextRepresentation() { + return operatorTextRepresentation; + } + + public YSQLExpression getExpression() { + return expr; + } + + public enum PostfixOperator implements Operator { + IS_NULL("IS NULL", "ISNULL") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return YSQLDataType.values(); + } + + }, + IS_UNKNOWN("IS UNKNOWN") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + }, + + IS_NOT_NULL("IS NOT NULL", "NOTNULL") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return YSQLDataType.values(); + } + + }, + IS_NOT_UNKNOWN("IS NOT UNKNOWN") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + }, + IS_TRUE("IS TRUE") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + return YSQLConstant.createFalse(); + } else { + return YSQLConstant.createBooleanConstant(expectedValue.cast(YSQLDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + + }, + IS_FALSE("IS FALSE") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + return YSQLConstant.createFalse(); + } else { + return YSQLConstant.createBooleanConstant(!expectedValue.cast(YSQLDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + + }; + + private final String[] textRepresentations; + + PostfixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public static PostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract YSQLConstant apply(YSQLConstant expectedValue); + + public abstract YSQLDataType[] getInputDataTypes(); + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java new file mode 100644 index 000000000..af9e64498 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java @@ -0,0 +1,36 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPostfixText implements YSQLExpression { + + private final YSQLExpression expr; + private final String text; + private final YSQLConstant expectedValue; + private final YSQLDataType type; + + public YSQLPostfixText(YSQLExpression expr, String text, YSQLConstant expectedValue, YSQLDataType type) { + this.expr = expr; + this.text = text; + this.expectedValue = expectedValue; + this.type = type; + } + + public YSQLExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + @Override + public YSQLDataType getExpressionType() { + return type; + } + + @Override + public YSQLConstant getExpectedValue() { + return expectedValue; + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java new file mode 100644 index 000000000..d34f07567 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java @@ -0,0 +1,115 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.IgnoreMeException; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPrefixOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final PrefixOperator op; + + public YSQLPrefixOperation(YSQLExpression expr, PrefixOperator op) { + this.expr = expr; + this.op = op; + } + + @Override + public YSQLDataType getExpressionType() { + return op.getExpressionType(); + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.getExpectedValue(expectedValue); + } + + public YSQLDataType[] getInputDataTypes() { + return op.dataTypes; + } + + public String getTextRepresentation() { + return op.textRepresentation; + } + + public YSQLExpression getExpression() { + return expr; + } + + public enum PrefixOperator implements Operator { + NOT("NOT", YSQLDataType.BOOLEAN) { + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + protected YSQLConstant getExpectedValue(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createBooleanConstant(!expectedValue.cast(YSQLDataType.BOOLEAN).asBoolean()); + } + } + }, + UNARY_PLUS("+", YSQLDataType.INT) { + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + @Override + protected YSQLConstant getExpectedValue(YSQLConstant expectedValue) { + // TODO: actual converts to double precision + return expectedValue; + } + + }, + UNARY_MINUS("-", YSQLDataType.INT) { + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + @Override + protected YSQLConstant getExpectedValue(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + // TODO + throw new IgnoreMeException(); + } + if (expectedValue.isInt() && expectedValue.asInt() == Long.MIN_VALUE) { + throw new IgnoreMeException(); + } + try { + return YSQLConstant.createIntConstant(-expectedValue.asInt()); + } catch (UnsupportedOperationException e) { + return null; + } + } + + }; + + private final String textRepresentation; + private final YSQLDataType[] dataTypes; + + PrefixOperator(String textRepresentation, YSQLDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract YSQLDataType getExpressionType(); + + protected abstract YSQLConstant getExpectedValue(YSQLConstant expectedValue); + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java new file mode 100644 index 000000000..ce477c238 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java @@ -0,0 +1,135 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public class YSQLSelect extends SelectBase implements YSQLExpression { + + private SelectType selectOption = SelectType.ALL; + private List joinClauses = Collections.emptyList(); + private YSQLExpression distinctOnClause; + private ForClause forClause; + + public void setSelectType(SelectType fromOptions) { + this.setSelectOption(fromOptions); + } + + public SelectType getSelectOption() { + return selectOption; + } + + public void setSelectOption(SelectType fromOptions) { + this.selectOption = fromOptions; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + + public List getJoinClauses() { + return joinClauses; + } + + public void setJoinClauses(List joinStatements) { + this.joinClauses = joinStatements; + + } + + public YSQLExpression getDistinctOnClause() { + return distinctOnClause; + } + + public void setDistinctOnClause(YSQLExpression distinctOnClause) { + if (selectOption != SelectType.DISTINCT) { + throw new IllegalArgumentException(); + } + this.distinctOnClause = distinctOnClause; + } + + public ForClause getForClause() { + return forClause; + } + + public void setForClause(ForClause forClause) { + this.forClause = forClause; + } + + public enum ForClause { + UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); + + private final String textRepresentation; + + ForClause(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static ForClause getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + + public enum SelectType { + DISTINCT, ALL; + + public static SelectType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public static class YSQLFromTable implements YSQLExpression { + private final YSQLTable t; + private final boolean only; + + public YSQLFromTable(YSQLTable t, boolean only) { + this.t = t; + this.only = only; + } + + public YSQLTable getTable() { + return t; + } + + public boolean isOnly() { + return only; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + } + + public static class YSQLSubquery implements YSQLExpression { + private final YSQLSelect s; + private final String name; + + public YSQLSubquery(YSQLSelect s, String name) { + this.s = s; + this.name = name; + } + + public YSQLSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java b/src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java new file mode 100644 index 000000000..794525bc2 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java @@ -0,0 +1,39 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLSimilarTo implements YSQLExpression { + + private final YSQLExpression string; + private final YSQLExpression similarTo; + private final YSQLExpression escapeCharacter; + + public YSQLSimilarTo(YSQLExpression string, YSQLExpression similarTo, YSQLExpression escapeCharacter) { + this.string = string; + this.similarTo = similarTo; + this.escapeCharacter = escapeCharacter; + } + + public YSQLExpression getString() { + return string; + } + + public YSQLExpression getSimilarTo() { + return similarTo; + } + + public YSQLExpression getEscapeCharacter() { + return escapeCharacter; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + return null; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java new file mode 100644 index 000000000..035b2fbe4 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java @@ -0,0 +1,182 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public class YSQLAlterTableGenerator { + + private final YSQLTable randomTable; + private final Randomly r; + private final YSQLGlobalState globalState; + + public YSQLAlterTableGenerator(YSQLTable randomTable, YSQLGlobalState globalState) { + this.randomTable = randomTable; + this.globalState = globalState; + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter create(YSQLTable randomTable, YSQLGlobalState globalState) { + return new YSQLAlterTableGenerator(randomTable, globalState).generate(); + } + + public List getActions(ExpectedErrors errors) { + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonInsertUpdateErrors(errors); + YSQLCommon.addCommonTableErrors(errors); + errors.add("duplicate key value violates unique constraint"); + errors.add("cannot drop key column"); + errors.add("cannot drop desired object(s) because other objects depend on them"); + errors.add("invalid input syntax for"); + errors.add("cannot remove a key column"); + errors.add("it has pending trigger events"); + errors.add("could not open relation"); + errors.add("functions in index expression must be marked IMMUTABLE"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("has no default operator class for access method"); + errors.add("does not accept data type"); + errors.add("does not exist for access method"); + errors.add("could not find cast from"); + errors.add("does not exist"); // TODO: investigate + errors.add("constraints on permanent tables may reference only permanent tables"); + List action; + if (Randomly.getBoolean()) { + action = Randomly.nonEmptySubset(Action.values()); + } else { + // make it more likely that the ALTER TABLE succeeds + action = Randomly.subset(Randomly.smallNumber(), Action.values()); + } + if (randomTable.getColumns().size() == 1) { + action.remove(Action.ALTER_TABLE_DROP_COLUMN); + } + if (!randomTable.hasIndexes()) { + action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); + } + if (action.isEmpty()) { + throw new IgnoreMeException(); + } + return action; + } + + public SQLQueryAdapter generate() { + ExpectedErrors errors = new ExpectedErrors(); + int i = 0; + List action = getActions(errors); + StringBuilder sb = new StringBuilder(); + sb.append("ALTER TABLE "); + if (Randomly.getBoolean()) { + sb.append(" ONLY"); + errors.add("cannot use ONLY for foreign key on partitioned table"); + } + sb.append(" "); + sb.append(randomTable.getName()); + sb.append(" "); + for (Action a : action) { + if (i++ != 0) { + sb.append(", "); + } + switch (a) { + case ALTER_TABLE_DROP_COLUMN: + sb.append("DROP "); + if (Randomly.getBoolean()) { + sb.append(" IF EXISTS "); + } + sb.append(randomTable.getRandomColumn().getName()); + errors.add("because other objects depend on it"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("RESTRICT", "CASCADE")); + } + errors.add("does not exist"); + errors.add("cannot drop column"); + errors.add("cannot drop key column"); + errors.add("cannot drop inherited column"); + break; + case ADD_TABLE_CONSTRAINT: + sb.append("ADD "); + sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); + YSQLCommon.addTableConstraint(sb, randomTable, globalState, errors); + errors.add("already exists"); + errors.add("multiple primary keys for table"); + errors.add("could not create unique index"); + errors.add("contains null values"); + errors.add("cannot cast type"); + errors.add("unsupported PRIMARY KEY constraint with partition key definition"); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("insufficient columns in UNIQUE constraint definition"); + errors.add("which is part of the partition key"); + errors.add("out of range"); + errors.add("there is no unique constraint matching given keys for referenced table"); + errors.add("constraints on temporary tables may reference only temporary tables"); + errors.add("constraints on unlogged tables may reference only permanent or unlogged tables"); + errors.add("constraints on permanent tables may reference only permanent tables"); + errors.add("cannot reference partitioned table"); + errors.add("cannot be implemented"); + errors.add("violates foreign key constraint"); + errors.add("unsupported ON COMMIT and foreign key combination"); + errors.add("USING INDEX is not supported on partitioned tables"); + if (Randomly.getBoolean()) { + sb.append(" NOT VALID"); + errors.add("cannot be marked NOT VALID"); + errors.add("cannot add NOT VALID foreign key on partitioned table"); + } else { + errors.add("is violated by some row"); + } + break; + case ADD_TABLE_CONSTRAINT_USING_INDEX: + sb.append("ADD "); + sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); + sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); + sb.append(" USING INDEX "); + sb.append(randomTable.getRandomIndex().getIndexName()); + errors.add("already exists"); + errors.add("PRIMARY KEY containing column of type"); + errors.add("not valid"); + errors.add("is not a unique index"); + errors.add("is already associated with a constraint"); + errors.add("Cannot create a primary key or unique constraint using such an index"); + errors.add("multiple primary keys for table"); + errors.add("appears twice in unique constraint"); + errors.add("appears twice in primary key constraint"); + errors.add("contains null values"); + errors.add("insufficient columns in PRIMARY KEY constraint definition"); + errors.add("which is part of the partition key"); + break; + case DISABLE_ROW_LEVEL_SECURITY: + sb.append("DISABLE ROW LEVEL SECURITY"); + break; + case ENABLE_ROW_LEVEL_SECURITY: + sb.append("ENABLE ROW LEVEL SECURITY"); + break; + case FORCE_ROW_LEVEL_SECURITY: + sb.append("FORCE ROW LEVEL SECURITY"); + break; + case NO_FORCE_ROW_LEVEL_SECURITY: + sb.append("NO FORCE ROW LEVEL SECURITY"); + break; + default: + throw new AssertionError(a); + } + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + protected enum Action { + // ALTER_TABLE_ADD_COLUMN, // [ COLUMN ] column data_type [ COLLATE collation ] [ + // column_constraint [ ... ] ] + ALTER_TABLE_DROP_COLUMN, // DROP [ COLUMN ] [ IF EXISTS ] column [ RESTRICT | CASCADE ] + ADD_TABLE_CONSTRAINT, // ADD table_constraint [ NOT VALID ] + ADD_TABLE_CONSTRAINT_USING_INDEX, // ADD table_constraint_using_index + DISABLE_ROW_LEVEL_SECURITY, // DISABLE ROW LEVEL SECURITY + ENABLE_ROW_LEVEL_SECURITY, // ENABLE ROW LEVEL SECURITY + FORCE_ROW_LEVEL_SECURITY, // FORCE ROW LEVEL SECURITY + NO_FORCE_ROW_LEVEL_SECURITY, // NO FORCE ROW LEVEL SECURITY + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java new file mode 100644 index 000000000..bee2ec5ef --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java @@ -0,0 +1,37 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public final class YSQLAnalyzeGenerator { + + private YSQLAnalyzeGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + YSQLTable table = globalState.getSchema().getRandomTable(); + StringBuilder sb = new StringBuilder("ANALYZE"); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(" VERBOSE"); + sb.append(")"); + } + sb.append(" "); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(table.getRandomNonEmptyColumnSubset().stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + } + + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("deadlock")); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java new file mode 100644 index 000000000..6e852d167 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java @@ -0,0 +1,32 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public final class YSQLClusterGenerator { + + private YSQLClusterGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("there is no previously clustered index for table"); + errors.add("cannot cluster a partitioned table"); + errors.add("access method does not support clustering"); + StringBuilder sb = new StringBuilder("CLUSTER "); + if (Randomly.getBoolean()) { + YSQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean() && !table.getIndexes().isEmpty()) { + sb.append(" USING "); + sb.append(table.getRandomIndex().getIndexName()); + errors.add("cannot cluster on partial index"); + } + } + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java new file mode 100644 index 000000000..f020a8994 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java @@ -0,0 +1,68 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +/** + * @see COMMENT + */ +public final class YSQLCommentGenerator { + + private YSQLCommentGenerator() { + } + + public static SQLQueryAdapter generate(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("COMMENT ON "); + Action type = Randomly.fromOptions(Action.values()); + YSQLTable randomTable = globalState.getSchema().getRandomTable(); + switch (type) { + case INDEX: + sb.append("INDEX "); + if (randomTable.getIndexes().isEmpty()) { + throw new IgnoreMeException(); + } else { + sb.append(randomTable.getRandomIndex().getIndexName()); + } + break; + case COLUMN: + sb.append("COLUMN "); + sb.append(randomTable.getRandomColumn().getFullQualifiedName()); + break; + case STATISTICS: + sb.append("STATISTICS "); + if (randomTable.getStatistics().isEmpty()) { + throw new IgnoreMeException(); + } else { + sb.append(randomTable.getStatistics().get(0).getName()); + } + break; + case TABLE: + sb.append("TABLE "); + if (randomTable.isView()) { + throw new IgnoreMeException(); + } + sb.append(randomTable.getName()); + break; + default: + throw new AssertionError(type); + } + sb.append(" IS "); + if (Randomly.getBoolean()) { + sb.append("NULL"); + } else { + sb.append("'"); + sb.append(globalState.getRandomly().getString().replace("'", "''")); + sb.append("'"); + } + return new SQLQueryAdapter(sb.toString()); + } + + private enum Action { + INDEX, COLUMN, STATISTICS, TABLE + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java new file mode 100644 index 000000000..e92af3e0f --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java @@ -0,0 +1,441 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; + +public final class YSQLCommon { + + private YSQLCommon() { + } + + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.add("An I/O error occurred while sending to the backend"); + errors.add("Conflicts with committed transaction"); + errors.add("cannot be changed"); + errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); + + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); + errors.add("but it cannot be referenced from this part of the query"); + errors.add("missing FROM-clause entry for table"); + + errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); + } + + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.add("PRIMARY KEY containing column of type 'INET' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'VARBIT' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'INT4RANGE' not yet supported"); + errors.add("INDEX on column of type 'INET' not yet supported"); + errors.add("INDEX on column of type 'VARBIT' not yet supported"); + errors.add("INDEX on column of type 'INT4RANGE' not yet supported"); + errors.add("is not commutative"); // exclude + errors.add("cannot be changed"); + errors.add("operator requires run-time type coercion"); // exclude + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.add("syntax error at or near \"(\""); + errors.add("does not exist"); + errors.add("is not unique"); + errors.add("cannot be changed"); + errors.add("invalid reference to FROM-clause entry for table"); + + errors.add("Invalid column number"); + errors.add("specified more than once"); + errors.add("You might need to add explicit type casts"); + errors.add("invalid regular expression"); + errors.add("could not determine which collation to use"); + errors.add("invalid input syntax for integer"); + errors.add("invalid regular expression"); + errors.add("operator does not exist"); + errors.add("quantifier operand invalid"); + errors.add("collation mismatch"); + errors.add("collations are not supported"); + errors.add("operator is not unique"); + errors.add("is not a valid binary digit"); + errors.add("invalid hexadecimal digit"); + errors.add("invalid hexadecimal data: odd number of digits"); + errors.add("zero raised to a negative power is undefined"); + errors.add("cannot convert infinity to numeric"); + errors.add("division by zero"); + errors.add("invalid input syntax for type money"); + errors.add("invalid input syntax for type"); + errors.add("cannot cast type"); + errors.add("value overflows numeric format"); + errors.add("is of type boolean but expression is of type text"); + errors.add("a negative number raised to a non-integer power yields a complex result"); + errors.add("could not determine polymorphic type because input has type unknown"); + addToCharFunctionErrors(errors); + addBitStringOperationErrors(errors); + addFunctionErrors(errors); + addCommonRangeExpressionErrors(errors); + addCommonRegexExpressionErrors(errors); + } + + private static void addToCharFunctionErrors(ExpectedErrors errors) { + errors.add("multiple decimal points"); + errors.add("and decimal point together"); + errors.add("multiple decimal points"); + errors.add("cannot use \"S\" twice"); + errors.add("must be ahead of \"PR\""); + errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); + errors.add("cannot use \"S\" and \"SG\" together"); + errors.add("cannot use \"S\" and \"MI\" together"); + errors.add("cannot use \"S\" and \"PL\" together"); + errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); + errors.add("is not a number"); + } + + private static void addBitStringOperationErrors(ExpectedErrors errors) { + errors.add("cannot XOR bit strings of different sizes"); + errors.add("cannot AND bit strings of different sizes"); + errors.add("cannot OR bit strings of different sizes"); + errors.add("must be type boolean, not type text"); + } + + private static void addFunctionErrors(ExpectedErrors errors) { + errors.add("out of valid range"); // get_bit/get_byte + errors.add("cannot take logarithm of a negative number"); + errors.add("cannot take logarithm of zero"); + errors.add("requested character too large for encoding"); // chr + errors.add("null character not permitted"); // chr + errors.add("requested character not valid for encoding"); // chr + errors.add("requested length too large"); // repeat + errors.add("invalid memory alloc request size"); // repeat + errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + } + + private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + errors.add("is not a valid hexadecimal digit"); + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.add("range lower bound must be less than or equal to range upper bound"); + errors.add("result of range difference would not be contiguous"); + errors.add("out of range"); + errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.add("value too long for type character"); + errors.add("not found in view targetlist"); + } + + public static boolean appendDataType(YSQLDataType type, StringBuilder sb, boolean allowSerial, + boolean generateOnlyKnown, List opClasses) throws AssertionError { + boolean serial = false; + switch (type) { + case BOOLEAN: + sb.append("boolean"); + break; + case INT: + if (Randomly.getBoolean() && allowSerial) { + serial = true; + sb.append(Randomly.fromOptions("serial", "bigserial")); + } else { + sb.append(Randomly.fromOptions("smallint", "integer", "bigint")); + } + break; + case TEXT: + if (Randomly.getBoolean()) { + sb.append("TEXT"); + } else if (Randomly.getBoolean()) { + // TODO: support CHAR (without VAR) + if (YSQLProvider.generateOnlyKnown || Randomly.getBoolean()) { + sb.append("VAR"); + } + sb.append("CHAR"); + sb.append("("); + sb.append(ThreadLocalRandom.current().nextInt(1, 500)); + sb.append(")"); + } else { + sb.append("name"); + } + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + case REAL: + if (Randomly.getBoolean()) { + sb.append("REAL"); + } else { + sb.append("FLOAT"); + } + break; + case RANGE: + sb.append(Randomly.fromOptions("int4range", "int4range")); // , "int8range", "numrange" + break; + case MONEY: + sb.append("money"); + break; + case BYTEA: + sb.append("bytea"); + break; + case BIT: + sb.append("BIT"); + // if (Randomly.getBoolean()) { + sb.append(" VARYING"); + // } + sb.append("("); + sb.append(Randomly.getNotCachedInteger(1, 500)); + sb.append(")"); + break; + case INET: + sb.append("inet"); + break; + default: + throw new AssertionError(type); + } + return serial; + } + + public static void generateWith(StringBuilder sb, YSQLGlobalState globalState, ExpectedErrors errors, + List columnsToBeAdded, boolean isTemporaryTable) { + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" "); + // disabled https://github.com/yugabyte/yugabyte-db/issues/11357 + // sb.append(" WITH ("); + // ArrayList values = new ArrayList<>(Arrays.asList(StorageParameters.values())); + // errors.add("unrecognized parameter"); + // errors.add("ALTER TABLE / ADD CONSTRAINT USING INDEX is not supported on partitioned tables"); + // List subset = Randomly.nonEmptySubset(values); + // int i = 0; + // for (StorageParameters parameter : subset) { + // if (i++ != 0) { + // sb.append(", "); + // } + // sb.append(parameter.parameter); + // sb.append("="); + // sb.append(parameter.op.apply(globalState.getRandomly())); + // } + // sb.append(")"); + } else { + sb.append(" WITHOUT OIDS "); + } + } else if (Randomly.getBoolean() && !isTemporaryTable) { + if (Randomly.getBoolean()) { + sb.append(" SPLIT INTO "); + sb.append(Randomly.smallNumber() + 1); + sb.append(" TABLETS "); + + errors.add("cannot create colocated table with split option"); + errors.add("columns must be present to split by number of tablets"); + errors.add("option is not yet supported for hash partitioned tables"); + } else { + sb.append(" SPLIT AT VALUES ("); + + errors.add("cannot create colocated table with split option"); + errors.add("SPLIT AT option is not yet supported for hash partitioned tables"); + errors.add("Cannot have duplicate split rows"); // just in case + + boolean hasBoolean = false; + for (YSQLColumn column : columnsToBeAdded) { + if (column.getType().equals(YSQLDataType.BOOLEAN)) { + hasBoolean = true; + break; + } + } + + int splits = hasBoolean ? 2 : Randomly.smallNumber() + 2; + long start = Randomly.smallNumber(); + boolean[] bools = { false, true }; + for (int i = 1; i <= splits; i++) { + int size = columnsToBeAdded.size(); + int counter = 1; + for (YSQLColumn c : columnsToBeAdded) { + sb.append("("); + switch (c.getType()) { + case INT: + case REAL: + sb.append(YSQLConstant.createDoubleConstant(start)); + case FLOAT: + sb.append(YSQLConstant.createIntConstant(start)); + break; + case BOOLEAN: + sb.append(YSQLConstant.createBooleanConstant(bools[i - 1])); + break; + case TEXT: + sb.append(YSQLConstant.createTextConstant(String.valueOf(start))); + break; + default: + throw new IgnoreMeException(); + } + sb.append(")"); + counter++; + start += Randomly.smallNumber() + 1; + if (counter <= size) { + sb.append(","); + } + } + + if (i < splits) { + sb.append(","); + } + } + sb.append(")"); + } + } else if (Randomly.getBoolean()) { + errors.add("Cannot use TABLEGROUP with TEMP table"); + sb.append(" TABLEGROUP tg").append( + Randomly.getNotCachedInteger(1, (int) YSQLTableGroupGenerator.UNIQUE_TABLEGROUP_COUNTER.get())); + } + } + + public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder sb, YSQLTable table, + YSQLGlobalState globalState, ExpectedErrors errors) { + // TODO constraint name + List tableConstraints = Randomly.nonEmptySubset(TableConstraints.values()); + if (excludePrimaryKey) { + tableConstraints.remove(TableConstraints.PRIMARY_KEY); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + tableConstraints.remove(TableConstraints.FOREIGN_KEY); + } + for (TableConstraints t : tableConstraints) { + sb.append(", "); + // TODO add index parameters + addTableConstraint(sb, table, globalState, t, errors); + } + } + + public static void addTableConstraint(StringBuilder sb, YSQLTable table, YSQLGlobalState globalState, + ExpectedErrors errors) { + addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); + } + + private static void addTableConstraint(StringBuilder sb, YSQLTable table, YSQLGlobalState globalState, + TableConstraints t, ExpectedErrors errors) { + List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); + List otherColumns; + YSQLCommon.addCommonExpressionErrors(errors); + switch (t) { + case CHECK: + sb.append("CHECK("); + sb.append(YSQLVisitor.getExpressionAsString(globalState, YSQLDataType.BOOLEAN, table.getColumns())); + sb.append(")"); + errors.add("constraint must be added to child tables too"); + errors.add("missing FROM-clause entry for table"); + break; + case UNIQUE: + sb.append("UNIQUE("); + sb.append(randomNonEmptyColumnSubset.stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + break; + case PRIMARY_KEY: + sb.append("PRIMARY KEY("); + sb.append(randomNonEmptyColumnSubset.stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + break; + case FOREIGN_KEY: + sb.append("FOREIGN KEY ("); + sb.append(randomNonEmptyColumnSubset.stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(") REFERENCES "); + YSQLTable randomOtherTable = globalState.getSchema().getRandomTable(tab -> !tab.isView()); + sb.append(randomOtherTable.getName()); + if (randomOtherTable.getColumns().size() < randomNonEmptyColumnSubset.size()) { + throw new IgnoreMeException(); + } + otherColumns = randomOtherTable.getRandomNonEmptyColumnSubset(randomNonEmptyColumnSubset.size()); + sb.append("("); + sb.append(otherColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("MATCH FULL", "MATCH SIMPLE")); + } + if (Randomly.getBoolean()) { + sb.append(" ON DELETE "); + errors.add("ERROR: invalid ON DELETE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" ON UPDATE "); + errors.add("invalid ON UPDATE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" "); + if (Randomly.getBoolean()) { + sb.append("DEFERRABLE"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("INITIALLY DEFERRED", "INITIALLY IMMEDIATE")); + } + } else { + sb.append("NOT DEFERRABLE"); + } + } + break; + default: + throw new AssertionError(t); + } + } + + private static void deleteOrUpdateAction(StringBuilder sb) { + sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in GROUP BY"); + } + + public static void addViewErrors(ExpectedErrors errors) { + errors.add("already exists"); + errors.add("cannot drop columns from view"); + errors.add("non-integer constant in ORDER BY"); // TODO + errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO + errors.add("cannot change data type of view column"); + errors.add("specified more than once"); // TODO + errors.add("materialized views must not use temporary tables or views"); + errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); + errors.add("is not a view"); + errors.add("non-integer constant in DISTINCT ON"); + errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + } + + public enum TableConstraints { + CHECK, UNIQUE, PRIMARY_KEY, FOREIGN_KEY + } + + // private enum StorageParameters { + // COLOCATED("COLOCATED", (r) -> Randomly.getBoolean()); + // // TODO + // + // private final String parameter; + // private final Function op; + // + // StorageParameters(String parameter, Function op) { + // this.parameter = parameter; + // this.op = op; + // } + // } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java new file mode 100644 index 000000000..e2eeb1ceb --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -0,0 +1,46 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; + +public final class YSQLDeleteGenerator { + + private YSQLDeleteGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + YSQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("violates foreign key constraint"); + errors.add("violates not-null constraint"); + errors.add("could not determine which collation to use for string comparison"); + StringBuilder sb = new StringBuilder("DELETE FROM"); + if (Randomly.getBoolean()) { + sb.append(" ONLY"); + } + sb.append(" "); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(YSQLVisitor.asString( + YSQLExpressionGenerator.generateExpression(globalState, table.getColumns(), YSQLDataType.BOOLEAN))); + } + if (Randomly.getBoolean()) { + sb.append(" RETURNING "); + sb.append( + YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, table.getColumns()))); + } + YSQLCommon.addCommonExpressionErrors(errors); + errors.add("out of range"); + errors.add("cannot cast"); + errors.add("invalid input syntax for"); + errors.add("division by zero"); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java new file mode 100644 index 000000000..40ccdfa0d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java @@ -0,0 +1,39 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable.TableType; + +public final class YSQLDiscardGenerator { + + private YSQLDiscardGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("DISCARD "); + // prevent that DISCARD discards all tables (if they are TEMP tables) + boolean hasNonTempTables = globalState.getSchema().getDatabaseTables().stream() + .anyMatch(t -> t.getTableType() == TableType.STANDARD); + String what; + if (hasNonTempTables) { + what = Randomly.fromOptions("ALL", "PLANS", "SEQUENCES", "TEMPORARY", "TEMP"); + } else { + what = Randomly.fromOptions("PLANS", "SEQUENCES"); + } + sb.append(what); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { + + @Override + public boolean couldAffectSchema() { + return canDiscardTemporaryTables(what); + } + }; + } + + private static boolean canDiscardTemporaryTables(String what) { + return what.contentEquals("TEMPORARY") || what.contentEquals("TEMP") || what.contentEquals("ALL"); + } +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java new file mode 100644 index 000000000..da9b5ae61 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java @@ -0,0 +1,41 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLIndex; + +public final class YSQLDropIndexGenerator { + + private YSQLDropIndexGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + List indexes = globalState.getSchema().getRandomTable().getIndexes(); + StringBuilder sb = new StringBuilder(); + sb.append("DROP INDEX "); + if (Randomly.getBoolean() || indexes.isEmpty()) { + sb.append("IF EXISTS "); + if (indexes.isEmpty() || Randomly.getBoolean()) { + sb.append(DBMSCommon.createIndexName(Randomly.smallNumber())); + } else { + sb.append(Randomly.fromList(indexes).getIndexName()); + } + } else { + sb.append(Randomly.fromList(indexes).getIndexName()); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot drop desired object(s) because other objects depend on them", + "cannot drop index", "does not exist"), + true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java new file mode 100644 index 000000000..c6a554d67 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java @@ -0,0 +1,563 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLRowValue; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryArithmeticOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryBitOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryComparisonOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryRangeOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConcatOperation; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLFunctionWithUnknownResult; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; + +public class YSQLExpressionGenerator implements ExpressionGenerator { + + private final int maxDepth; + + private final Randomly r; + private final Map functionsAndTypes; + private final List allowedFunctionTypes; + private List columns; + private YSQLRowValue rw; + private boolean expectedResult; + private YSQLGlobalState globalState; + private boolean allowAggregateFunctions; + + public YSQLExpressionGenerator(YSQLGlobalState globalState) { + this.r = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + this.globalState = globalState; + this.functionsAndTypes = globalState.getFunctionsAndTypes(); + this.allowedFunctionTypes = globalState.getAllowedFunctionTypes(); + } + + public static YSQLExpression generateExpression(YSQLGlobalState globalState, YSQLDataType type) { + return new YSQLExpressionGenerator(globalState).generateExpression(0, type); + } + + private static YSQLCompoundDataType getCompoundDataType(YSQLDataType type) { + switch (type) { + case BOOLEAN: + case DECIMAL: // TODO + case FLOAT: + case INT: + case MONEY: + case RANGE: + case REAL: + case INET: + case BYTEA: + return YSQLCompoundDataType.create(type); + case TEXT: // TODO + case BIT: + if (Randomly.getBoolean() + || YSQLProvider.generateOnlyKnown /* + * The PQS implementation does not check for size specifications + */) { + return YSQLCompoundDataType.create(type); + } else { + return YSQLCompoundDataType.create(type, (int) Randomly.getNotCachedInteger(1, 1000)); + } + default: + throw new AssertionError(type); + } + + } + + public static YSQLExpression generateConstant(Randomly r, YSQLDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return YSQLConstant.createNullConstant(); + } + // if (Randomly.getBooleanWithSmallProbability()) { + // return YSQLConstant.createTextConstant(r.getString()); + // } + switch (type) { + case INT: + if (Randomly.getBooleanWithSmallProbability()) { + return YSQLConstant.createTextConstant(String.valueOf(r.getInteger())); + } else { + return YSQLConstant.createIntConstant(r.getInteger()); + } + case BOOLEAN: + if (Randomly.getBooleanWithSmallProbability() && !YSQLProvider.generateOnlyKnown) { + return YSQLConstant + .createTextConstant(Randomly.fromOptions("TR", "TRUE", "FA", "FALSE", "0", "1", "ON", "off")); + } else { + return YSQLConstant.createBooleanConstant(Randomly.getBoolean()); + } + case TEXT: + return YSQLConstant.createTextConstant(r.getString()); + case DECIMAL: + return YSQLConstant.createDecimalConstant(r.getRandomBigDecimal()); + case FLOAT: + return YSQLConstant.createFloatConstant((float) r.getDouble()); + case REAL: + return YSQLConstant.createDoubleConstant(r.getDouble()); + case RANGE: + return YSQLConstant.createRange(r.getInteger(), Randomly.getBoolean(), r.getInteger(), + Randomly.getBoolean()); + case MONEY: + return new YSQLCastOperation(generateConstant(r, YSQLDataType.FLOAT), + getCompoundDataType(YSQLDataType.MONEY)); + case INET: + return YSQLConstant.createInetConstant(getRandomInet(r)); + case BIT: + return YSQLConstant.createBitConstant(r.getInteger()); + case BYTEA: + return YSQLConstant.createByteConstant(String.valueOf(r.getInteger())); + default: + throw new AssertionError(type); + } + } + + private static String getRandomInet(Randomly r) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 4; i++) { + if (i != 0) { + sb.append('.'); + } + sb.append(r.getInteger() & 255); + } + return sb.toString(); + } + + public static YSQLExpression generateExpression(YSQLGlobalState globalState, List columns, + YSQLDataType type) { + return new YSQLExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); + } + + public static YSQLExpression generateExpression(YSQLGlobalState globalState, List columns) { + return new YSQLExpressionGenerator(globalState).setColumns(columns).generateExpression(0); + + } + + public YSQLExpressionGenerator setColumns(List columns) { + this.columns = columns; + return this; + } + + public YSQLExpressionGenerator setRowValue(YSQLRowValue rw) { + this.rw = rw; + return this; + } + + public YSQLExpression generateExpression(int depth) { + return generateExpression(depth, YSQLDataType.getRandomType()); + } + + public List generateOrderBy() { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new YSQLOrderByTerm(YSQLColumnValue.create(Randomly.fromList(columns), null), + YSQLOrderByTerm.YSQLOrder.getRandomOrder())); + } + return orderBys; + } + + private YSQLExpression generateFunctionWithUnknownResult(int depth, YSQLDataType type) { + List supportedFunctions = YSQLFunctionWithUnknownResult + .getSupportedFunctions(type); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + supportedFunctions = supportedFunctions.stream() + .filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (supportedFunctions.isEmpty()) { + throw new IgnoreMeException(); + } + YSQLFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); + return new YSQLFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); + } + + private YSQLExpression generateFunctionWithKnownResult(int depth, YSQLDataType type) { + List functions = Stream.of(YSQLFunction.YSQLFunctionWithResult.values()) + .filter(f -> f.supportsReturnType(type)).collect(Collectors.toList()); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (functions.isEmpty()) { + throw new IgnoreMeException(); + } + YSQLFunction.YSQLFunctionWithResult randomFunction = Randomly.fromList(functions); + int nrArgs = randomFunction.getNrArgs(); + if (randomFunction.isVariadic()) { + nrArgs += Randomly.smallNumber(); + } + YSQLDataType[] argTypes = randomFunction.getInputTypesForReturnType(type, nrArgs); + YSQLExpression[] args = new YSQLExpression[nrArgs]; + do { + for (int i = 0; i < args.length; i++) { + args[i] = generateExpression(depth + 1, argTypes[i]); + } + } while (!randomFunction.checkArguments(args)); + return new YSQLFunction(randomFunction, type, args); + } + + private YSQLExpression generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (YSQLProvider.generateOnlyKnown) { + validOptions.remove(BooleanExpression.SIMILAR_TO); + validOptions.remove(BooleanExpression.POSIX_REGEX); + validOptions.remove(BooleanExpression.BINARY_RANGE_COMPARISON); + } + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + YSQLPostfixOperation.PostfixOperator random = YSQLPostfixOperation.PostfixOperator.getRandom(); + return YSQLPostfixOperation + .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); + case IN_OPERATION: + return inOperation(depth + 1); + case NOT: + return new YSQLPrefixOperation(generateExpression(depth + 1, YSQLDataType.BOOLEAN), + YSQLPrefixOperation.PrefixOperator.NOT); + case BINARY_LOGICAL_OPERATOR: + YSQLExpression first = generateExpression(depth + 1, YSQLDataType.BOOLEAN); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + first = new YSQLBinaryLogicalOperation(first, generateExpression(depth + 1, YSQLDataType.BOOLEAN), + YSQLBinaryLogicalOperation.BinaryLogicalOperator.getRandom()); + } + return first; + case BINARY_COMPARISON: + YSQLDataType dataType = getMeaningfulType(); + return generateComparison(depth, dataType); + case CAST: + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(YSQLDataType.BOOLEAN)); + case FUNCTION: + return generateFunction(depth + 1, YSQLDataType.BOOLEAN); + case BETWEEN: + YSQLDataType type = getMeaningfulType(); + return new YSQLBetweenOperation(generateExpression(depth + 1, type), generateExpression(depth + 1, type), + generateExpression(depth + 1, type), Randomly.getBoolean()); + case SIMILAR_TO: + assert !expectedResult; + // TODO also generate the escape character + return new YSQLSimilarTo(generateExpression(depth + 1, YSQLDataType.TEXT), + generateExpression(depth + 1, YSQLDataType.TEXT), null); + case POSIX_REGEX: + assert !expectedResult; + return new YSQLPOSIXRegularExpression(generateExpression(depth + 1, YSQLDataType.TEXT), + generateExpression(depth + 1, YSQLDataType.TEXT), + YSQLPOSIXRegularExpression.POSIXRegex.getRandom()); + case BINARY_RANGE_COMPARISON: + // TODO element check + return new YSQLBinaryRangeOperation(YSQLBinaryRangeOperation.YSQLBinaryRangeComparisonOperator.getRandom(), + generateExpression(depth + 1, YSQLDataType.RANGE), + generateExpression(depth + 1, YSQLDataType.RANGE)); + default: + throw new AssertionError(); + } + } + + private YSQLDataType getMeaningfulType() { + // make it more likely that the expression does not only consist of constant + // expressions + if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { + return YSQLDataType.getRandomType(); + } else { + return Randomly.fromList(columns).getType(); + } + } + + private YSQLExpression generateFunction(int depth, YSQLDataType type) { + if (YSQLProvider.generateOnlyKnown || Randomly.getBoolean()) { + return generateFunctionWithKnownResult(depth, type); + } else { + return generateFunctionWithUnknownResult(depth, type); + } + } + + private YSQLExpression generateComparison(int depth, YSQLDataType dataType) { + YSQLExpression leftExpr = generateExpression(depth + 1, dataType); + YSQLExpression rightExpr = generateExpression(depth + 1, dataType); + return getComparison(leftExpr, rightExpr); + } + + private YSQLExpression getComparison(YSQLExpression leftExpr, YSQLExpression rightExpr) { + return new YSQLBinaryComparisonOperation(leftExpr, rightExpr, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.getRandom()); + } + + private YSQLExpression inOperation(int depth) { + YSQLDataType type = YSQLDataType.getRandomType(); + YSQLExpression leftExpr = generateExpression(depth + 1, type); + List rightExpr = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + rightExpr.add(generateExpression(depth + 1, type)); + } + return new YSQLInOperation(leftExpr, rightExpr, Randomly.getBoolean()); + } + + public YSQLExpression generateExpression(int depth, YSQLDataType originalType) { + YSQLDataType dataType = originalType; + if (dataType == YSQLDataType.REAL && Randomly.getBoolean()) { + dataType = Randomly.fromOptions(YSQLDataType.INT, YSQLDataType.FLOAT); + } + if (dataType == YSQLDataType.FLOAT && Randomly.getBoolean()) { + dataType = YSQLDataType.INT; + } + return generateExpressionInternal(depth, dataType); + } + + private YSQLExpression generateExpressionInternal(int depth, YSQLDataType dataType) throws AssertionError { + if (allowAggregateFunctions && Randomly.getBoolean()) { + allowAggregateFunctions = false; // aggregate function calls cannot be nested + return getAggregate(dataType); + } + if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { + // generic expression + if (Randomly.getBoolean() || depth > maxDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(r, dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(r, dataType); + } else { + return createColumnOfType(dataType); + } + } + } else { + if (Randomly.getBoolean()) { + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(dataType)); + } else { + return generateFunctionWithUnknownResult(depth, dataType); + } + } + } else { + switch (dataType) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case TEXT: + return generateTextExpression(depth); + case DECIMAL: + case REAL: + case FLOAT: + case MONEY: + case INET: + return generateConstant(r, dataType); + case BYTEA: + return generateByteExpression(); + case BIT: + return generateBitExpression(depth); + case RANGE: + return generateRangeExpression(depth); + default: + throw new AssertionError(dataType); + } + } + } + + private YSQLExpression generateRangeExpression(int depth) { + RangeExpression option; + List validOptions = new ArrayList<>(Arrays.asList(RangeExpression.values())); + option = Randomly.fromList(validOptions); + switch (option) { + case BINARY_OP: + return new YSQLBinaryRangeOperation(YSQLBinaryRangeOperation.YSQLBinaryRangeOperator.getRandom(), + generateExpression(depth + 1, YSQLDataType.RANGE), + generateExpression(depth + 1, YSQLDataType.RANGE)); + default: + throw new AssertionError(option); + } + } + + private YSQLExpression generateTextExpression(int depth) { + TextExpression option; + List validOptions = new ArrayList<>(Arrays.asList(TextExpression.values())); + option = Randomly.fromList(validOptions); + + switch (option) { + case CAST: + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(YSQLDataType.TEXT)); + case FUNCTION: + return generateFunction(depth + 1, YSQLDataType.TEXT); + case CONCAT: + return generateConcat(depth); + default: + throw new AssertionError(); + } + } + + private YSQLExpression generateConcat(int depth) { + YSQLExpression left = generateExpression(depth + 1, YSQLDataType.TEXT); + YSQLExpression right = generateExpression(depth + 1); + return new YSQLConcatOperation(left, right); + } + + private YSQLExpression generateByteExpression() { + return YSQLConstant.createByteConstant("Th\\000omas"); + } + + private YSQLExpression generateBitExpression(int depth) { + BitExpression option; + option = Randomly.fromOptions(BitExpression.values()); + switch (option) { + case BINARY_OPERATION: + return new YSQLBinaryBitOperation(YSQLBinaryBitOperation.YSQLBinaryBitOperator.getRandom(), + generateExpression(depth + 1, YSQLDataType.BIT), generateExpression(depth + 1, YSQLDataType.BIT)); + default: + throw new AssertionError(); + } + } + + private YSQLExpression generateIntExpression(int depth) { + IntExpression option; + option = Randomly.fromOptions(IntExpression.values()); + switch (option) { + case CAST: + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(YSQLDataType.INT)); + case UNARY_OPERATION: + YSQLExpression intExpression = generateExpression(depth + 1, YSQLDataType.INT); + return new YSQLPrefixOperation(intExpression, Randomly.getBoolean() + ? YSQLPrefixOperation.PrefixOperator.UNARY_PLUS : YSQLPrefixOperation.PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, YSQLDataType.INT); + case BINARY_ARITHMETIC_EXPRESSION: + return new YSQLBinaryArithmeticOperation(generateExpression(depth + 1, YSQLDataType.INT), + generateExpression(depth + 1, YSQLDataType.INT), + YSQLBinaryArithmeticOperation.YSQLBinaryOperator.getRandom()); + default: + throw new AssertionError(); + } + } + + private YSQLExpression createColumnOfType(YSQLDataType type) { + List columns = filterColumns(type); + YSQLColumn fromList = Randomly.fromList(columns); + YSQLConstant value = rw == null ? null : rw.getValues().get(fromList); + return YSQLColumnValue.create(fromList, value); + } + + final List filterColumns(YSQLDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + public YSQLExpression generateExpressionWithExpectedResult(YSQLDataType type) { + this.expectedResult = true; + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns).setRowValue(rw); + YSQLExpression expr; + do { + expr = gen.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + public List generateExpressions(int nr) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(0)); + } + return expressions; + } + + public YSQLExpression generateExpression(YSQLDataType dataType) { + return generateExpression(0, dataType); + } + + public YSQLExpressionGenerator setGlobalState(YSQLGlobalState globalState) { + this.globalState = globalState; + return this; + } + + public YSQLExpression generateHavingClause() { + this.allowAggregateFunctions = true; + YSQLExpression expression = generateExpression(YSQLDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + + public YSQLExpression generateAggregate() { + return getAggregate(YSQLDataType.getRandomType()); + } + + private YSQLExpression getAggregate(YSQLDataType dataType) { + List aggregates = YSQLAggregate.YSQLAggregateFunction + .getAggregates(dataType); + YSQLAggregate.YSQLAggregateFunction agg = Randomly.fromList(aggregates); + return generateArgsForAggregate(dataType, agg); + } + + public YSQLAggregate generateArgsForAggregate(YSQLDataType dataType, YSQLAggregate.YSQLAggregateFunction agg) { + List types = agg.getTypes(dataType); + List args = new ArrayList<>(); + for (YSQLDataType argType : types) { + args.add(generateExpression(argType)); + } + return new YSQLAggregate(args, agg); + } + + public YSQLExpressionGenerator allowAggregates(boolean value) { + allowAggregateFunctions = value; + return this; + } + + @Override + public YSQLExpression generatePredicate() { + return generateExpression(YSQLDataType.BOOLEAN); + } + + @Override + public YSQLExpression negatePredicate(YSQLExpression predicate) { + return new YSQLPrefixOperation(predicate, YSQLPrefixOperation.PrefixOperator.NOT); + } + + @Override + public YSQLExpression isNull(YSQLExpression expr) { + return new YSQLPostfixOperation(expr, YSQLPostfixOperation.PostfixOperator.IS_NULL); + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, CAST, BETWEEN, IN_OPERATION, + SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON + } + + private enum RangeExpression { + BINARY_OP + } + + private enum TextExpression { + CAST, FUNCTION, CONCAT + } + + private enum BitExpression { + BINARY_OPERATION + } + + private enum IntExpression { + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java new file mode 100644 index 000000000..faa7908b0 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -0,0 +1,153 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLIndex; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public final class YSQLIndexGenerator { + + private YSQLIndexGenerator() { + } + + public static SQLQueryAdapter generate(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE"); + if (Randomly.getBoolean()) { + sb.append(" UNIQUE"); + } + sb.append(" INDEX "); + /* + * Commented out as a workaround for https://www.postgresql.org/message-id/CA%2Bu7OA4XYhc- + * qyCgJqwwgMGZDWAyeH821oa5oMzm_HEifZ4BeA%40mail.gmail.com + */ + // if (Randomly.getBoolean()) { + // sb.append("CONCURRENTLY "); + // } + YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized + // views + String indexName = getNewIndexName(randomTable); + sb.append(indexName); + sb.append(" ON "); + if (Randomly.getBoolean()) { + sb.append("ONLY "); + } + sb.append(randomTable.getName()); + IndexType method; + if (Randomly.getBoolean()) { + sb.append(" USING "); + method = Randomly.fromOptions(IndexType.values()); + sb.append(method); + } else { + method = IndexType.BTREE; + } + + sb.append("("); + if (method == IndexType.HASH) { + sb.append(randomTable.getRandomColumn().getName()); + } else { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + if (Randomly.getBoolean()) { + sb.append(randomTable.getRandomColumn().getName()); + } else { + sb.append("("); + YSQLExpression expression = YSQLExpressionGenerator.generateExpression(globalState, + randomTable.getColumns()); + sb.append(YSQLVisitor.asString(expression)); + sb.append(")"); + } + + // if (Randomly.getBoolean()) { + // sb.append(" "); + // sb.append("COLLATE "); + // sb.append(Randomly.fromOptions("C", "POSIX")); + // } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" "); + sb.append(globalState.getRandomOpclass()); + errors.add("does not accept"); + errors.add("does not exist for access method"); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + } + + sb.append(")"); + if (Randomly.getBoolean() && method != IndexType.HASH) { + sb.append(" INCLUDE("); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + } + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + YSQLExpression expr = new YSQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()) + .setGlobalState(globalState).generateExpression(YSQLDataType.BOOLEAN); + sb.append(YSQLVisitor.asString(expr)); + } + errors.add("already contains data"); // CONCURRENT INDEX failed + errors.add("You might need to add explicit type casts"); + errors.add("INDEX on column of type"); + errors.add("collations are not supported"); // TODO check + errors.add("because it has pending trigger events"); + errors.add("duplicate key value violates unique constraint"); + errors.add("could not determine which collation to use for"); + errors.add("index method \"gist\" not supported yet"); + errors.add("is duplicated"); + errors.add("already exists"); + errors.add("could not create unique index"); + errors.add("has no default operator class"); + errors.add("does not support"); + errors.add("cannot cast"); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("insufficient columns in UNIQUE constraint definition"); + errors.add("invalid input syntax for"); + errors.add("must be type "); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("out of range"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("functions in index expression must be marked IMMUTABLE"); + errors.add("result of range difference would not be contiguous"); + errors.add("which is part of the partition key"); + YSQLCommon.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private static String getNewIndexName(YSQLTable randomTable) { + List indexes = randomTable.getIndexes(); + int indexI = 0; + while (true) { + String indexName = DBMSCommon.createIndexName(indexI++); + if (indexes.stream().noneMatch(i -> i.getIndexName().equals(indexName))) { + return indexName; + } + } + } + + public enum IndexType { + BTREE, HASH, GIST, GIN + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java new file mode 100644 index 000000000..1817891b3 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java @@ -0,0 +1,127 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public final class YSQLInsertGenerator { + + private YSQLInsertGenerator() { + } + + public static SQLQueryAdapter insert(YSQLGlobalState globalState) { + YSQLTable table = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("cannot insert into column"); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonInsertUpdateErrors(errors); + YSQLCommon.addCommonExpressionErrors(errors); + errors.add("multiple assignments to same column"); + errors.add("violates foreign key constraint"); + errors.add("value too long for type character varying"); + errors.add("conflicting key value violates exclusion constraint"); + errors.add("violates not-null constraint"); + errors.add("current transaction is aborted"); + errors.add("bit string too long"); + errors.add("new row violates check option for view"); + errors.add("reached maximum value of sequence"); + errors.add("but expression is of type"); + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" OVERRIDING"); + sb.append(" "); + sb.append(Randomly.fromOptions("SYSTEM", "USER")); + sb.append(" VALUE"); + } + sb.append(" VALUES"); + + if (globalState.getDbmsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { + StringBuilder sbRowValue = new StringBuilder(); + sbRowValue.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sbRowValue.append(", "); + } + sbRowValue.append(YSQLVisitor.asString( + YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), columns.get(i).getType()))); + } + sbRowValue.append(")"); + + int n = (int) Randomly.getNotCachedInteger(100, 1000); + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(sbRowValue); + } + } else { + int n = Randomly.smallNumber() + 1; + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + insertRow(globalState, sb, columns, n == 1); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" ON CONFLICT "); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(table.getRandomColumn().getName()); + sb.append(")"); + errors.add("there is no unique or exclusion constraint matching the ON CONFLICT specification"); + } + sb.append(" DO NOTHING"); + } + errors.add("duplicate key value violates unique constraint"); + errors.add("identity column defined as GENERATED ALWAYS"); + errors.add("out of range"); + errors.add("violates check constraint"); + errors.add("no partition of relation"); + errors.add("invalid input syntax"); + errors.add("division by zero"); + errors.add("violates foreign key constraint"); + errors.add("data type unknown"); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private static void insertRow(YSQLGlobalState globalState, StringBuilder sb, List columns, + boolean canBeDefault) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + if (!Randomly.getBooleanWithSmallProbability() || !canBeDefault) { + YSQLExpression generateConstant; + if (Randomly.getBoolean()) { + generateConstant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), + columns.get(i).getType()); + } else { + generateConstant = new YSQLExpressionGenerator(globalState) + .generateExpression(columns.get(i).getType()); + } + sb.append(YSQLVisitor.asString(generateConstant)); + } else { + sb.append("DEFAULT"); + } + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java new file mode 100644 index 000000000..a6fb58d8e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLNotifyGenerator { + + private YSQLNotifyGenerator() { + } + + private static String getChannel() { + return Randomly.fromOptions("asdf", "test"); + } + + public static SQLQueryAdapter createNotify(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("NOTIFY "); + sb.append(getChannel()); + if (Randomly.getBoolean()) { + sb.append(", "); + sb.append("'"); + sb.append(globalState.getRandomly().getString().replace("'", "''")); + sb.append("'"); + } + return new SQLQueryAdapter(sb.toString()); + } + + public static SQLQueryAdapter createListen() { + String sb = "LISTEN " + getChannel(); + return new SQLQueryAdapter(sb); + } + + public static SQLQueryAdapter createUnlisten() { + StringBuilder sb = new StringBuilder(); + sb.append("UNLISTEN "); + if (Randomly.getBoolean()) { + sb.append(getChannel()); + } else { + sb.append("*"); + } + return new SQLQueryAdapter(sb.toString()); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java new file mode 100644 index 000000000..03109227d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java @@ -0,0 +1,62 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.ForClause; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.SelectType; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; + +public final class YSQLRandomQueryGenerator { + + private YSQLRandomQueryGenerator() { + } + + public static YSQLSelect createRandomQuery(int nrColumns, YSQLGlobalState globalState) { + List columns = new ArrayList<>(); + YSQLTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < nrColumns; i++) { + columns.add(gen.generateExpression(0)); + } + YSQLSelect select = new YSQLSelect(); + select.setSelectType(SelectType.getRandom()); + if (select.getSelectOption() == SelectType.DISTINCT && Randomly.getBoolean()) { + select.setDistinctOnClause(gen.generateExpression(0)); + } + select.setFromList(tables.getTables().stream().map(t -> new YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return select; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java new file mode 100644 index 000000000..7526d8dc6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java @@ -0,0 +1,58 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLIndex; + +public final class YSQLReindexGenerator { + + private YSQLReindexGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("could not create unique index"); // CONCURRENT INDEX + StringBuilder sb = new StringBuilder(); + sb.append("REINDEX"); + // if (Randomly.getBoolean()) { + // sb.append(" VERBOSE"); + // } + sb.append(" "); + Scope scope = Randomly.fromOptions(Scope.values()); + switch (scope) { + case INDEX: + sb.append("INDEX "); + List indexes = globalState.getSchema().getRandomTable().getIndexes(); + if (indexes.isEmpty()) { + throw new IgnoreMeException(); + } + sb.append(indexes.stream().map(YSQLIndex::getIndexName).collect(Collectors.joining())); + break; + case TABLE: + sb.append("TABLE "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + break; + case DATABASE: + sb.append("DATABASE "); + sb.append(globalState.getSchema().getDatabaseName()); + break; + default: + throw new AssertionError(scope); + } + errors.add("already contains data"); // FIXME bug report + errors.add("does not exist"); // internal index + errors.add("REINDEX is not yet implemented for partitioned indexes"); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private enum Scope { + INDEX, TABLE, DATABASE + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java new file mode 100644 index 000000000..3d7104b5e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java @@ -0,0 +1,90 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLSequenceGenerator { + + private YSQLSequenceGenerator() { + } + + public static SQLQueryAdapter createSequence(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("TEMPORARY", "TEMP")); + } + sb.append(" SEQUENCE"); + // TODO keep track of sequences + sb.append(" IF NOT EXISTS"); + // TODO generate sequence names + sb.append(" seq"); + if (Randomly.getBoolean()) { + sb.append(" AS "); + sb.append(Randomly.fromOptions("smallint", "integer", "bigint")); + } + if (Randomly.getBoolean()) { + sb.append(" INCREMENT"); + if (Randomly.getBoolean()) { + sb.append(" BY"); + } + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + errors.add("INCREMENT must not be zero"); + } + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" MINVALUE"); + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + } else { + sb.append(" NO MINVALUE"); + } + errors.add("must be less than MAXVALUE"); + } + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" MAXVALUE"); + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + } else { + sb.append(" NO MAXVALUE"); + } + errors.add("must be less than MAXVALUE"); + } + if (Randomly.getBoolean()) { + sb.append(" START"); + if (Randomly.getBoolean()) { + sb.append(" WITH"); + } + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + errors.add("cannot be less than MINVALUE"); + errors.add("cannot be greater than MAXVALUE"); + } + if (Randomly.getBoolean()) { + sb.append(" CACHE "); + sb.append(globalState.getRandomly().getPositiveIntegerNotNull()); + } + errors.add("is out of range"); + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" NO"); + } + sb.append(" CYCLE"); + } + if (Randomly.getBoolean()) { + sb.append(" OWNED BY "); + // if (Randomly.getBoolean()) { + sb.append("NONE"); + // } else { + // sb.append(s.getRandomTable().getRandomColumn().getFullQualifiedName()); + // } + } + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java new file mode 100644 index 000000000..3406ca3de --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java @@ -0,0 +1,196 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.function.Function; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLSetGenerator { + + private YSQLSetGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + ArrayList options = new ArrayList<>(Arrays.asList(ConfigurationOption.values())); + options.remove(ConfigurationOption.DEFAULT_WITH_OIDS); + ConfigurationOption option = Randomly.fromList(options); + sb.append("SET "); + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions("SESSION", "LOCAL")); + sb.append(" "); + } + sb.append(option.getOptionName()); + sb.append("="); + if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append(option.op.apply(globalState.getRandomly())); + } + // todo avoiding props that are not represented in YSQL + ExpectedErrors errors = new ExpectedErrors(); + errors.add("unrecognized configuration parameter"); + errors.add("cannot be changed"); + + return new SQLQueryAdapter(sb.toString(), errors); + } + + private enum ConfigurationOption { + // YUGABYTE + YB_DEBUG_REPORT_ERROR_STACKTRACE("yb_debug_report_error_stacktrace", + (r) -> Randomly.fromOptions("false", "true")), + YB_DEBUG_LOG_CATCACHE_EVENTS("yb_debug_log_catcache_events", (r) -> Randomly.fromOptions("false", "true")), + YB_DEBUG_LOG_INTERNAL_RESTARTS("yb_debug_log_internal_restarts", (r) -> Randomly.fromOptions("false", "true")), + YB_DEBUG_LOG_DOCDB_REQUESTS("yb_debug_log_docdb_requests", (r) -> Randomly.fromOptions("false", "true")), + // YB_READ_FROM_FOLLOWERS("yb_read_from_followers", (r) -> Randomly.fromOptions("false", "true")), + YB_NON_DDL_TXN_FOR_SYS_TABLES_ALLOWED("yb_non_ddl_txn_for_sys_tables_allowed", + (r) -> Randomly.fromOptions("false", "true")), + YB_TRANSACTION_PRIORITY("yb_transaction_priority", + (r) -> Randomly.fromOptions(0, 0.1, 0.2, 0.3, 0.4, 1, 0.9, 0.8, 0.7, 0.6)), + YB_TRANSACTION_PRIORITY_LOWER_BOUND("yb_transaction_priority_lower_bound", + (r) -> Randomly.fromOptions(0, 0.1, 0.2, 0.3, 0.4)), + YB_TRANSACTION_PRIORITY_UPPER_BOUND("yb_transaction_priority_upper_bound", + (r) -> Randomly.fromOptions(1, 0.9, 0.8, 0.7, 0.6)), + YB_FORMAT_FUNCS_INCLUDE_YB_METADATA("yb_format_funcs_include_yb_metadata", + (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_GEOLOCATION_COSTING("yb_enable_geolocation_costing", (r) -> Randomly.fromOptions("false", "true")), + YB_BINARY_RESTORE("yb_binary_restore", (r) -> Randomly.fromOptions("false", "true")), + YB_TEST_SYSTEM_CATALOGS_CREATION("yb_test_system_catalogs_creation", + (r) -> Randomly.fromOptions("false", "true")), + YB_TEST_FAIL_NEXT_DDL("yb_test_fail_next_ddl", (r) -> Randomly.fromOptions("false", "true")), + YB_DISABLE_TRANSACTIONAL_WRITES("yb_disable_transactional_writes", + (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_OPTIMIZER_STATISTICS("yb_enable_optimizer_statistics", (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_EXPRESSION_PUSHDOWN("yb_enable_expression_pushdown", (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_UPSERT_MODE("yb_enable_upsert_mode", (r) -> Randomly.fromOptions("false", "true")), + YB_PLANNER_CUSTOM_PLAN_FOR_PARTITION_PRUNING("yb_planner_custom_plan_for_partition_pruning", + (r) -> Randomly.fromOptions("false", "true")), + YB_INDEX_STATE_FLAGS_UPDATE_DELAY("yb_index_state_flags_update_delay", + (r) -> Randomly.getNotCachedInteger(200, 1000)), + YB_TEST_PLANNER_CUSTOM_PLAN_THRESHOLD("yb_test_planner_custom_plan_threshold", + (r) -> Randomly.getNotCachedInteger(1, Integer.MAX_VALUE)), + // YSQL values + YSQL_UPGRADE_MODE("ysql_upgrade_mode", (r) -> Randomly.fromOptions("false", "true")), + YSQL_SESSION_MAX_BATCH_SIZE("ysql_session_max_batch_size", + (r) -> Randomly.getNotCachedInteger(1, Integer.MAX_VALUE)), + YSQL_MAX_IN_FLIGHT_OPS("ysql_max_in_flight_ops", (r) -> Randomly.getNotCachedInteger(1, Integer.MAX_VALUE)), + // https://www.postgresql.org/docs/11/runtime-config-wal.html + // This parameter can only be set at server start. + // WAL_LEVEL("wal_level", (r) -> Randomly.fromOptions("replica", "minimal", "logical")), + // FSYNC("fsync", (r) -> Randomly.fromOptions(1, 0)), + SYNCHRONOUS_COMMIT("synchronous_commit", + (r) -> Randomly.fromOptions("remote_apply", "remote_write", "local", "off")), + WAL_COMPRESSION("wal_compression", (r) -> Randomly.fromOptions(1, 0)), + // wal_buffer: server start + // wal_writer_delay: server start + // wal_writer_flush_after + COMMIT_DELAY("commit_delay", (r) -> r.getInteger(0, 100000)), + COMMIT_SIBLINGS("commit_siblings", (r) -> r.getInteger(0, 1000)), + // 19.5.2. Checkpoints + // checkpoint_timeout + // checkpoint_completion_target + // checkpoint_flush_after + // checkpoint_warning + // max_wal_size + // min_wal_size + // 19.5.3. Archiving + // archive_mode + // archive_command + // archive_timeout + // https://www.postgresql.org/docs/11/runtime-config-statistics.html + // 19.9.1. Query and Index Statistics Collector + TRACK_ACTIVITIES("track_activities", (r) -> Randomly.fromOptions(1, 0)), + // track_activity_query_size + TRACK_COUNTS("track_counts", (r) -> Randomly.fromOptions(1, 0)), + TRACK_IO_TIMING("track_io_timing", (r) -> Randomly.fromOptions(1, 0)), + TRACK_FUNCTIONS("track_functions", (r) -> Randomly.fromOptions("'none'", "'pl'", "'all'")), + // stats_temp_directory + // TODO 19.9.2. Statistics Monitoring + // https://www.postgresql.org/docs/11/runtime-config-autovacuum.html + // all can only be set at server-conf time + // 19.11. Client Connection Defaults + VACUUM_FREEZE_TABLE_AGE("vacuum_freeze_table_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), + VACUUM_FREEZE_MIN_AGE("vacuum_freeze_min_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 1000000000)), + VACUUM_MULTIXACT_FREEZE_TABLE_AGE("vacuum_multixact_freeze_table_age", + (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), + VACUUM_MULTIXACT_FREEZE_MIN_AGE("vacuum_multixact_freeze_min_age", + (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 1000000000)), + VACUUM_CLEANUP_INDEX_SCALE_FACTOR("vacuum_cleanup_index_scale_factor", + (r) -> Randomly.fromOptions(0.0, 0.0000001, 0.00001, 0.01, 0.1, 1, 10, 100, 100000, 10000000000.0)), + // TODO others + GIN_FUZZY_SEARCH_LIMIT("gin_fuzzy_search_limit", (r) -> r.getInteger(0, 2147483647)), + // 19.13. Version and Platform Compatibility + DEFAULT_WITH_OIDS("default_with_oids", (r) -> Randomly.fromOptions(0, 1)), + SYNCHRONIZED_SEQSCANS("synchronize_seqscans", (r) -> Randomly.fromOptions(0, 1)), + // https://www.postgresql.org/docs/devel/runtime-config-query.html + ENABLE_BITMAPSCAN("enable_bitmapscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_GATHERMERGE("enable_gathermerge", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_HASHJOIN("enable_hashjoin", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_INDEXSCAN("enable_indexscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_INDEXONLYSCAN("enable_indexonlyscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_MATERIAL("enable_material", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_MERGEJOIN("enable_mergejoin", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_NESTLOOP("enable_nestloop", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARALLEL_APPEND("enable_parallel_append", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARALLEL_HASH("enable_parallel_hash", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARTITION_PRUNING("enable_partition_pruning", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARTITIONWISE_JOIN("enable_partitionwise_join", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARTITIONWISE_AGGREGATE("enable_partitionwise_aggregate", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_SEGSCAN("enable_seqscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_SORT("enable_sort", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_TIDSCAN("enable_tidscan", (r) -> Randomly.fromOptions(1, 0)), + // 19.7.2. Planner Cost Constants (complete as of March 2020) + // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-CONSTANTS + SEQ_PAGE_COST("seq_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + RANDOM_PAGE_COST("random_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + CPU_TUPLE_COST("cpu_tuple_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + CPU_INDEX_TUPLE_COST("cpu_index_tuple_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + CPU_OPERATOR_COST("cpu_operator_cost", (r) -> Randomly.fromOptions(0d, 0.000001, 0.0025, 0.1, 1, 10, 10000)), + PARALLEL_SETUP_COST("parallel_setup_cost", (r) -> r.getLong(0, Long.MAX_VALUE)), + PARALLEL_TUPLE_COST("parallel_tuple_cost", (r) -> r.getLong(0, Long.MAX_VALUE)), + MIN_PARALLEL_TABLE_SCAN_SIZE("min_parallel_table_scan_size", (r) -> r.getInteger(0, 715827882)), + MIN_PARALLEL_INDEX_SCAN_SIZE("min_parallel_index_scan_size", (r) -> r.getInteger(0, 715827882)), + EFFECTIVE_CACHE_SIZE("effective_cache_size", (r) -> r.getInteger(1, 2147483647)), + JIT_ABOVE_COST("jit_above_cost", (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE - 1))), + JIT_INLINE_ABOVE_COST("jit_inline_above_cost", (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE))), + JIT_OPTIMIZE_ABOVE_COST("jit_optimize_above_cost", + (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE))), + // 19.7.3. Genetic Query Optimizer (complete as of March 2020) + // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-GEQO + GEQO("geqo", (r) -> Randomly.fromOptions(1, 0)), + GEQO_THRESHOLD("geqo_threshold", (r) -> r.getInteger(2, 2147483647)), + GEQO_EFFORT("geqo_effort", (r) -> r.getInteger(1, 10)), + GEQO_POO_SIZE("geqo_pool_size", (r) -> r.getInteger(0, 2147483647)), + GEQO_GENERATIONS("geqo_generations", (r) -> r.getInteger(0, 2147483647)), + GEQO_SELECTION_BIAS("geqo_selection_bias", (r) -> Randomly.fromOptions(1.5, 1.8, 2.0)), + GEQO_SEED("geqo_seed", (r) -> Randomly.fromOptions(0, 0.5, 1)), + // 19.7.4. Other Planner Options (complete as of March 2020) + // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-OTHER + DEFAULT_STATISTICS_TARGET("default_statistics_target", (r) -> r.getInteger(1, 10000)), + CONSTRAINT_EXCLUSION("constraint_exclusion", (r) -> Randomly.fromOptions("on", "off", "partition")), + CURSOR_TUPLE_FRACTION("cursor_tuple_fraction", + (r) -> Randomly.fromOptions(0.0, 0.1, 0.000001, 1, 0.5, 0.9999999)), + FROM_COLLAPSE_LIMIT("from_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), + JIT("jit", (r) -> Randomly.fromOptions(1, 0)), + JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), + PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), + FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")); + + private final String optionName; + private final Function op; + + ConfigurationOption(String optionName, Function op) { + this.optionName = optionName; + this.op = op; + } + + public String getOptionName() { + return optionName; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java new file mode 100644 index 000000000..3b212b00b --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java @@ -0,0 +1,74 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLStatisticsObject; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public final class YSQLStatisticsGenerator { + + private YSQLStatisticsGenerator() { + } + + public static SQLQueryAdapter insert(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE STATISTICS "); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO materialized view + if (randomTable.getColumns().size() < 2) { + throw new IgnoreMeException(); + } + sb.append(" "); + sb.append(getNewStatisticsName(randomTable)); + if (Randomly.getBoolean()) { + sb.append(" ("); + List statsSubset; + statsSubset = Randomly.nonEmptySubset("ndistinct", "dependencies", "mcv"); + sb.append(String.join(", ", statsSubset)); + sb.append(")"); + } + + List randomColumns = randomTable.getRandomNonEmptyColumnSubset( + globalState.getRandomly().getInteger(2, randomTable.getColumns().size())); + sb.append(" ON "); + sb.append(randomColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(" FROM "); + sb.append(randomTable.getName()); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot have more than 8 columns in statistics"), + true); + } + + public static SQLQueryAdapter remove(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder("DROP STATISTICS "); + YSQLTable randomTable = globalState.getSchema().getRandomTable(); + List statistics = randomTable.getStatistics(); + if (statistics.isEmpty()) { + throw new IgnoreMeException(); + } + sb.append(Randomly.fromList(statistics).getName()); + return new SQLQueryAdapter(sb.toString(), true); + } + + private static String getNewStatisticsName(YSQLTable randomTable) { + List statistics = randomTable.getStatistics(); + int i = 0; + while (true) { + String candidateName = "s" + i; + if (statistics.stream().noneMatch(stat -> stat.getName().contentEquals(candidateName))) { + return candidateName; + } + i++; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java new file mode 100644 index 000000000..5e376ab4c --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java @@ -0,0 +1,247 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public class YSQLTableGenerator { + + protected final ExpectedErrors errors = new ExpectedErrors(); + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + private final List columnsToBeAdded = new ArrayList<>(); + private final YSQLTable table; + private final boolean generateOnlyKnown; + private final YSQLGlobalState globalState; + private boolean columnCanHavePrimaryKey; + private boolean columnHasPrimaryKey; + private boolean isTemporaryTable; + + public YSQLTableGenerator(String tableName, boolean generateOnlyKnown, YSQLGlobalState globalState) { + this.tableName = tableName; + this.generateOnlyKnown = generateOnlyKnown; + this.globalState = globalState; + table = new YSQLTable(tableName, columnsToBeAdded, null, null, null, false, false); + // YB catalog specific messages + errors.add("The catalog snapshot used for this transaction has been invalidated"); + + errors.add("PRIMARY KEY containing column of type"); + errors.add("specified value cannot be cast to type boolean for column"); + errors.add("already exists"); + errors.add("invalid input syntax for"); + errors.add("is not unique"); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("cannot create partitioned table as inheritance child"); + errors.add("cannot cast"); + errors.add("ERROR: functions in index expression must be marked IMMUTABLE"); + errors.add("functions in partition key expression must be marked IMMUTABLE"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("has no default operator class for access method"); + errors.add("does not exist for access method"); + errors.add("does not accept data type"); + errors.add("but default expression is of type text"); + errors.add("has pseudo-type unknown"); + errors.add("no collation was derived for partition key column"); + errors.add("cannot set colocated true on a non-colocated database"); + errors.add("Cannot split table that does not have primary key"); + errors.add("inherits from generated column but specifies identity"); + errors.add("inherits from generated column but specifies default"); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonTableErrors(errors); + } + + public static SQLQueryAdapter generate(String tableName, boolean generateOnlyKnown, YSQLGlobalState globalState) { + return new YSQLTableGenerator(tableName, generateOnlyKnown, globalState).generate(); + } + + private SQLQueryAdapter generate() { + columnCanHavePrimaryKey = true; + sb.append("CREATE"); + if (Randomly.getBooleanWithSmallProbability()) { + sb.append(" "); + isTemporaryTable = true; + sb.append(Randomly.fromOptions("TEMPORARY", "TEMP")); + } + sb.append(" TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + createStandard(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void createStandard() throws AssertionError { + sb.append("("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + String name = DBMSCommon.createColumnName(i); + createColumn(name); + } + if (Randomly.getBoolean()) { + errors.add("constraints on temporary tables may reference only temporary tables"); + errors.add("constraints on unlogged tables may reference only permanent or unlogged tables"); + errors.add("constraints on permanent tables may reference only permanent tables"); + errors.add("cannot be implemented"); + errors.add("there is no unique constraint matching given keys for referenced table"); + errors.add("cannot reference partitioned table"); + errors.add("unsupported ON COMMIT and foreign key combination"); + errors.add("ERROR: invalid ON DELETE action for foreign key constraint containing generated column"); + errors.add("exclusion constraints are not supported on partitioned tables"); + errors.add("option is not yet supported for hash partitioned tables"); + YSQLCommon.addTableConstraints(columnHasPrimaryKey, sb, table, globalState, errors); + } + sb.append(")"); + generatePartitionBy(); + YSQLCommon.generateWith(sb, globalState, errors, columnsToBeAdded, isTemporaryTable); + if (Randomly.getBoolean() && isTemporaryTable) { + sb.append(" ON COMMIT "); + // todo ON COMMIT DROP fails and it's known issue + // sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS", "DROP")); + sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS")); + sb.append(" "); + } + } + + private void createColumn(String name) throws AssertionError { + sb.append(name); + sb.append(" "); + YSQLDataType type = YSQLDataType.getRandomType(); + boolean serial = YSQLCommon.appendDataType(type, sb, true, generateOnlyKnown, globalState.getCollates()); + YSQLColumn c = new YSQLColumn(name, type); + c.setTable(table); + columnsToBeAdded.add(c); + sb.append(" "); + if (Randomly.getBoolean()) { + createColumnConstraint(type, serial); + } + } + + private void generatePartitionBy() { + if (Randomly.getBoolean()) { + return; + } + sb.append(" PARTITION BY "); + // TODO "RANGE", + String partitionOption = Randomly.fromOptions("RANGE", "LIST", "HASH"); + sb.append(partitionOption); + sb.append("("); + errors.add("unrecognized parameter"); + errors.add("cannot use constant expression"); + errors.add("unrecognized parameter"); + errors.add("unsupported PRIMARY KEY constraint with partition key definition"); + errors.add("which is part of the partition key."); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("does not accept data type"); + int n = partitionOption.contentEquals("LIST") ? 1 : Randomly.smallNumber() + 1; + YSQLCommon.addCommonExpressionErrors(errors); + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("("); + YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, columnsToBeAdded); + sb.append(YSQLVisitor.asString(expr)); + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(globalState.getRandomOpclass()); + errors.add("does not exist for access method"); + } + } + sb.append(")"); + } + + private void createColumnConstraint(YSQLDataType type, boolean serial) { + List constraintSubset = Randomly.nonEmptySubset(ColumnConstraint.values()); + if (Randomly.getBoolean()) { + // make checks constraints less likely + constraintSubset.remove(ColumnConstraint.CHECK); + } + if (!columnCanHavePrimaryKey || columnHasPrimaryKey) { + constraintSubset.remove(ColumnConstraint.PRIMARY_KEY); + } + if (constraintSubset.contains(ColumnConstraint.GENERATED) + && constraintSubset.contains(ColumnConstraint.DEFAULT)) { + // otherwise: ERROR: both default and identity specified for column + constraintSubset.remove(Randomly.fromOptions(ColumnConstraint.GENERATED, ColumnConstraint.DEFAULT)); + } + if (constraintSubset.contains(ColumnConstraint.GENERATED) && type != YSQLDataType.INT) { + // otherwise: ERROR: identity column type must be smallint, integer, or bigint + constraintSubset.remove(ColumnConstraint.GENERATED); + } + if (serial) { + constraintSubset.remove(ColumnConstraint.GENERATED); + constraintSubset.remove(ColumnConstraint.DEFAULT); + constraintSubset.remove(ColumnConstraint.NULL_OR_NOT_NULL); + + } + for (ColumnConstraint c : constraintSubset) { + sb.append(" "); + switch (c) { + case NULL_OR_NOT_NULL: + sb.append(Randomly.fromOptions("NOT NULL", "NULL")); + errors.add("conflicting NULL/NOT NULL declarations"); + break; + case UNIQUE: + sb.append("UNIQUE"); + break; + case PRIMARY_KEY: + sb.append("PRIMARY KEY"); + columnHasPrimaryKey = true; + break; + case DEFAULT: + sb.append("DEFAULT"); + sb.append(" ("); + sb.append(YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, type))); + sb.append(")"); + errors.add("out of range"); + errors.add("is a generated column"); + break; + case CHECK: + sb.append("CHECK ("); + sb.append(YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, columnsToBeAdded, + YSQLDataType.BOOLEAN))); + sb.append(")"); + errors.add("out of range"); + break; + case GENERATED: + sb.append("GENERATED "); + if (Randomly.getBoolean()) { + sb.append(" ALWAYS AS ("); + sb.append(YSQLVisitor + .asString(YSQLExpressionGenerator.generateExpression(globalState, columnsToBeAdded, type))); + sb.append(") STORED"); + errors.add("A generated column cannot reference another generated column."); + errors.add("cannot use generated column in partition key"); + errors.add("generation expression is not immutable"); + errors.add("cannot use column reference in DEFAULT expression"); + } else { + sb.append(Randomly.fromOptions("ALWAYS", "BY DEFAULT")); + sb.append(" AS IDENTITY"); + } + break; + default: + throw new AssertionError(sb); + } + } + } + + private enum ColumnConstraint { + NULL_OR_NOT_NULL, UNIQUE, PRIMARY_KEY, DEFAULT, CHECK, GENERATED + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java new file mode 100644 index 000000000..da1b5c573 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java @@ -0,0 +1,24 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.concurrent.atomic.AtomicLong; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLTableGroupGenerator { + + // TODO rework + public static final AtomicLong UNIQUE_TABLEGROUP_COUNTER = new AtomicLong(1); + + private YSQLTableGroupGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE TABLEGROUP "); + String tableGroupName = "tg" + UNIQUE_TABLEGROUP_COUNTER.incrementAndGet(); + sb.append(tableGroupName); + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java new file mode 100644 index 000000000..67780c016 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public final class YSQLTransactionGenerator { + + private YSQLTransactionGenerator() { + } + + public static SQLQueryAdapter executeBegin() { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("BEGIN"); + if (Randomly.getBoolean()) { + errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); + sb.append(" ISOLATION LEVEL "); + sb.append(Randomly.fromOptions("SERIALIZABLE", "REPEATABLE READ", "READ COMMITTED")); + // if (Randomly.getBoolean()) { + // sb.append(" "); + // sb.append(Randomly.fromOptions("READ WRITE", "READ ONLY")); + // } + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java new file mode 100644 index 000000000..d10bccee6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java @@ -0,0 +1,41 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTable; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLTruncateGenerator { + + private YSQLTruncateGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("TRUNCATE"); + if (Randomly.getBoolean()) { + sb.append(" TABLE"); + } + // TODO partitions + // if (Randomly.getBoolean()) { + // sb.append(" ONLY"); + // } + sb.append(" "); + sb.append(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty().stream().map(AbstractTable::getName) + .collect(Collectors.joining(", "))); + // if (Randomly.getBoolean()) { + // sb.append(" "); + // sb.append(Randomly.fromOptions("RESTART IDENTITY", "CONTINUE IDENTITY")); + // } + // if (Randomly.getBoolean()) { + // sb.append(" "); + // sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + // } + return new SQLQueryAdapter(sb.toString(), ExpectedErrors + .from("cannot truncate a table referenced in a foreign key constraint", "is not a table")); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java new file mode 100644 index 000000000..fe8e71ada --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -0,0 +1,76 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public final class YSQLUpdateGenerator { + + private YSQLUpdateGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + YSQLTable randomTable = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); + StringBuilder sb = new StringBuilder(); + sb.append("UPDATE "); + sb.append(randomTable.getName()); + sb.append(" SET "); + ExpectedErrors errors = ExpectedErrors.from("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", + " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable"); + errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced + // table multiple times + errors.add("new row violates check option for view"); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + YSQLCommon.addCommonInsertUpdateErrors(errors); + + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + YSQLColumn column = columns.get(i); + sb.append(column.getName()); + sb.append(" = "); + if (!Randomly.getBoolean()) { + YSQLExpression constant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(YSQLVisitor.asString(constant)); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append("("); + YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), + column.getType()); + // caused by casts + sb.append(YSQLVisitor.asString(expr)); + sb.append(")"); + } + } + errors.add("invalid input syntax for "); + errors.add("operator does not exist: text = boolean"); + errors.add("violates check constraint"); + errors.add("could not determine which collation to use for string comparison"); + errors.add("but expression is of type"); + YSQLCommon.addCommonExpressionErrors(errors); + if (!Randomly.getBooleanWithSmallProbability()) { + sb.append(" WHERE "); + YSQLExpression where = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), + YSQLDataType.BOOLEAN); + sb.append(YSQLVisitor.asString(where)); + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java new file mode 100644 index 000000000..f873f7812 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java @@ -0,0 +1,19 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLVacuumGenerator { + + private YSQLVacuumGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + String sb = "VACUUM"; + ExpectedErrors errors = new ExpectedErrors(); + errors.add("VACUUM cannot run inside a transaction block"); + return new SQLQueryAdapter(sb, errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java new file mode 100644 index 000000000..a5b57fbd8 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -0,0 +1,73 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; + +public final class YSQLViewGenerator { + + private YSQLViewGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE"); + // boolean materialized; + // boolean recursive = false; + if (Randomly.getBoolean()) { + sb.append(" MATERIALIZED"); + // materialized = true; + } else { + if (Randomly.getBoolean()) { + sb.append(" OR REPLACE"); + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" TEMP", " TEMPORARY")); + } + // if (Randomly.getBoolean()) { + // sb.append(" RECURSIVE"); + // recursive = true; + // } + // materialized = false; + } + sb.append(" VIEW "); + int i = 0; + String[] name = new String[1]; + while (true) { + name[0] = "v" + i++; + if (globalState.getSchema().getDatabaseTables().stream() + .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { + break; + } + } + sb.append(name[0]); + sb.append("("); + int nrColumns = Randomly.smallNumber() + 1; + for (i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(DBMSCommon.createColumnName(i)); + } + sb.append(")"); + // if (Randomly.getBoolean() && !materialized && !recursive) { + // sb.append(" WITH "); + // sb.append(Randomly.fromOptions("CASCADED", "LOCAL")); + // sb.append(" CHECK OPTION"); + // errors.add("WITH CHECK OPTION is supported only on automatically updatable views"); + // } + sb.append(" AS ("); + YSQLSelect select = YSQLRandomQueryGenerator.createRandomQuery(nrColumns, globalState); + sb.append(YSQLVisitor.asString(select)); + sb.append(")"); + YSQLCommon.addGroupingErrors(errors); + YSQLCommon.addViewErrors(errors); + YSQLCommon.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java new file mode 100644 index 000000000..a873ce505 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java @@ -0,0 +1,94 @@ +package sqlancer.yugabyte.ysql.oracle; + +import static sqlancer.yugabyte.ysql.YSQLProvider.CREATION_LOCK; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; + +public class YSQLCatalog implements TestOracle { + protected final YSQLGlobalState state; + + protected final ExpectedErrors errors = new ExpectedErrors(); + protected final Main.StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + + private final List dmlActions = Arrays.asList(YSQLProvider.Action.INSERT, + YSQLProvider.Action.UPDATE, YSQLProvider.Action.DELETE); + private final List catalogActions = Arrays.asList(YSQLProvider.Action.CREATE_VIEW, + YSQLProvider.Action.CREATE_SEQUENCE, YSQLProvider.Action.ALTER_TABLE, YSQLProvider.Action.SET_CONSTRAINTS, + YSQLProvider.Action.DISCARD, YSQLProvider.Action.DROP_INDEX, YSQLProvider.Action.COMMENT_ON, + YSQLProvider.Action.RESET_ROLE, YSQLProvider.Action.RESET); + private final List diskActions = Arrays.asList(YSQLProvider.Action.TRUNCATE, + YSQLProvider.Action.VACUUM); + + public YSQLCatalog(YSQLGlobalState globalState) { + this.state = globalState; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + } + + private YSQLProvider.Action getRandomAction(List actions) { + return actions.get(state.getRandomly().getInteger(0, actions.size())); + } + + protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { + synchronized (CREATION_LOCK) { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = YSQLTableGenerator.generate(tableName, true, globalState); + globalState.executeStatement(createTable); + globalState.getManager().incrementSelectQueryCount(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + } catch (IgnoreMeException e) { + // do nothing + } + } + } + } + + @Override + public void check() throws Exception { + // create table or evaluate catalog test + int seed = state.getRandomly().getInteger(1, 100); + if (seed > 95) { + createTables(state, 1); + } else { + YSQLProvider.Action randomAction; + + if (seed > 40) { + randomAction = getRandomAction(dmlActions); + } else if (seed > 10) { + randomAction = getRandomAction(catalogActions); + } else { + randomAction = getRandomAction(diskActions); + } + + state.executeStatement(randomAction.getQuery(state)); + } + state.getManager().incrementSelectQueryCount(); + } +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java new file mode 100644 index 000000000..e8f791b3f --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java @@ -0,0 +1,108 @@ +package sqlancer.yugabyte.ysql.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLRandomQueryGenerator; + +public class YSQLFuzzer implements TestOracle { + private final YSQLGlobalState globalState; + private final List testQueries; + private final ExpectedErrors errors = new ExpectedErrors(); + + public YSQLFuzzer(YSQLGlobalState globalState) { + this.globalState = globalState; + + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + YSQLCommon.addGroupingErrors(errors); + YSQLCommon.addViewErrors(errors); + + // remove timeout error from scope + errors.add("canceling statement due to statement timeout"); + + // exclude nemesis exceptions + errors.add("terminating connection due to administrator command"); + errors.add("Java heap space"); + errors.add("Connection refused"); + errors.add("Connection to"); + + // get config from -Dconfig.file="path/to/fuzzer.conf" + testQueries = new ArrayList<>(); + try { + Config config = ConfigFactory.load(); + ArrayList queriesList = (ArrayList) config.getList("queries").unwrapped(); + for (Object configValue : queriesList) { + String type = ((String) ((Map) configValue).get("type")).toUpperCase(Locale.ROOT); + Integer weight = (Integer) ((Map) configValue).get("weight"); + + Query query = type.equalsIgnoreCase("SELECT") ? new SelectQuery() + : new ActionQuery(YSQLProvider.Action.valueOf(type)); + + for (int i = 0; i < weight; i++) { + testQueries.add(query); + } + } + } catch (Exception e) { + // do nothing + } finally { + if (testQueries.isEmpty()) { + System.out.println("No configuration found. Using just random select statements"); + testQueries.add(new SelectQuery()); + testQueries.add(new ActionQuery(YSQLProvider.Action.UPDATE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.DELETE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.INSERT)); + } + } + } + + @Override + public void check() throws Exception { + Query s = testQueries.get(globalState.getRandomly().getInteger(0, testQueries.size())); + globalState.executeStatement(s.getQuery(globalState, errors)); + globalState.getManager().incrementSelectQueryCount(); + } + + private static class Query { + public SQLQueryAdapter getQuery(YSQLGlobalState state, ExpectedErrors errors) throws Exception { + throw new IllegalAccessException("Should be implemented"); + }; + } + + private static class ActionQuery extends Query { + private final YSQLProvider.Action action; + + ActionQuery(YSQLProvider.Action action) { + this.action = action; + } + + @Override + public SQLQueryAdapter getQuery(YSQLGlobalState state, ExpectedErrors errors) throws Exception { + return action.getQuery(state); + } + } + + private static class SelectQuery extends Query { + + @Override + public SQLQueryAdapter getQuery(YSQLGlobalState state, ExpectedErrors errors) throws Exception { + return new SQLQueryAdapter( + YSQLVisitor.asString(YSQLRandomQueryGenerator.createRandomQuery(Randomly.smallNumber() + 1, state)) + + ";", + errors); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java new file mode 100644 index 000000000..863b63398 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -0,0 +1,165 @@ +package sqlancer.yugabyte.ysql.oracle; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPBase; + +public class YSQLNoRECOracle extends NoRECBase implements TestOracle { + + private final YSQLSchema s; + + public YSQLNoRECOracle(YSQLGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + } + + public static List getJoinStatements(YSQLGlobalState globalState, List columns, + List tables) { + List joinStatements = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLTable table = Randomly.fromList(tables); + tables.remove(table); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLSelect.YSQLSubquery subquery = YSQLTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public void check() throws SQLException { + YSQLTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + YSQLExpression randomWhereCondition = getRandomWhereCondition(columns); + List tables = randomTables.getTables(); + + List joinStatements = getJoinStatements(state, columns, tables); + List fromTables = tables.stream() + .map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); + int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, + firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + private YSQLExpression getRandomWhereCondition(List columns) { + return new YSQLExpressionGenerator(state).setColumns(columns).generateExpression(YSQLDataType.BOOLEAN); + } + + private int getUnoptimizedQueryCount(List fromTables, YSQLExpression randomWhereCondition, + List joinStatements) throws SQLException { + YSQLSelect select = new YSQLSelect(); + YSQLCastOperation isTrue = new YSQLCastOperation(randomWhereCondition, + YSQLCompoundDataType.create(YSQLDataType.INT)); + YSQLPostfixText asText = new YSQLPostfixText(isTrue, " as count", null, YSQLDataType.INT); + select.setFetchColumns(Collections.singletonList(asText)); + select.setFromList(fromTables); + select.setSelectType(YSQLSelect.SelectType.ALL); + select.setJoinClauses(joinStatements); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + YSQLVisitor.asString(select) + ") as res"; + if (options.logEachSelect()) { + logger.writeCurrent(unoptimizedQueryString); + } + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGet(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getOptimizedQueryCount(List randomTables, List columns, + YSQLExpression randomWhereCondition, List joinStatements) throws SQLException { + YSQLSelect select = new YSQLSelect(); + YSQLColumnValue allColumns = new YSQLColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(Arrays.asList(allColumns)); + select.setFromList(randomTables); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new YSQLExpressionGenerator(state).setColumns(columns).generateOrderBy()); + } + select.setSelectType(YSQLSelect.SelectType.ALL); + select.setJoinClauses(joinStatements); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = YSQLVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..95ac268f4 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java @@ -0,0 +1,147 @@ +package sqlancer.yugabyte.ysql.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLRowValue; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public class YSQLPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + + public YSQLPivotedQuerySynthesisOracle(YSQLGlobalState globalState) throws SQLException { + super(globalState); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + } + + /* + * Prevent name collisions by aliasing the column. + */ + private YSQLColumn getFetchValueAliasedColumn(YSQLColumn c) { + YSQLColumn aliasedColumn = new YSQLColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType()); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + private List generateGroupByClause(List columns, YSQLRowValue rw) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> YSQLColumnValue.create(c, rw.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private YSQLConstant generateLimit() { + if (Randomly.getBoolean()) { + return YSQLConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private YSQLExpression generateOffset() { + if (Randomly.getBoolean()) { + return YSQLConstant.createIntConstant(0); + } else { + return null; + } + } + + private YSQLExpression generateRectifiedExpression(List columns, YSQLRowValue rw) { + YSQLExpression expr = new YSQLExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) + .generateExpressionWithExpectedResult(YSQLDataType.BOOLEAN); + YSQLExpression result; + if (expr.getExpectedValue().isNull()) { + result = YSQLPostfixOperation.create(expr, YSQLPostfixOperation.PostfixOperator.IS_NULL); + } else { + result = YSQLPostfixOperation.create(expr, expr.getExpectedValue().cast(YSQLDataType.BOOLEAN).asBoolean() + ? YSQLPostfixOperation.PostfixOperator.IS_TRUE : YSQLPostfixOperation.PostfixOperator.IS_FALSE); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions + sb.append(query.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (YSQLColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + @Override + public SQLQueryAdapter getRectifiedQuery() throws SQLException { + YSQLTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); + + YSQLSelect selectStatement = new YSQLSelect(); + selectStatement.setSelectType(Randomly.fromOptions(YSQLSelect.SelectType.values())); + List columns = randomFromTables.getColumns(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + + fetchColumns = columns; + selectStatement.setFromList(randomFromTables.getTables().stream() + .map(t -> new YSQLSelect.YSQLFromTable(t, false)).collect(Collectors.toList())); + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new YSQLColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + YSQLExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + YSQLExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + YSQLExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + List orderBy = new YSQLExpressionGenerator(globalState).setColumns(columns).generateOrderBy(); + selectStatement.setOrderByExpressions(orderBy); + return new SQLQueryAdapter(YSQLVisitor.asString(selectStatement)); + } + + @Override + protected String getExpectedValues(YSQLExpression expr) { + return YSQLVisitor.asExpectedValues(expr); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java new file mode 100644 index 000000000..aa6dfd530 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -0,0 +1,194 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.postgresql.util.PSQLException; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate.YSQLAggregateFunction; +import sqlancer.yugabyte.ysql.ast.YSQLAlias; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation.PostfixOperator; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation.PrefixOperator; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; + +public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public YSQLTLPAggregateOracle(YSQLGlobalState state) { + super(state); + YSQLCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() throws SQLException { + YSQLAggregateFunction aggregateFunction = Randomly.fromOptions(YSQLAggregateFunction.MAX, + YSQLAggregateFunction.MIN, YSQLAggregateFunction.SUM, YSQLAggregateFunction.BIT_AND, + YSQLAggregateFunction.BIT_OR, YSQLAggregateFunction.BOOL_AND, YSQLAggregateFunction.BOOL_OR, + YSQLAggregateFunction.COUNT); + YSQLAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), + aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + originalQuery = YSQLVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + String queryFormatString = "-- %s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + String assertionMessage = String.format("the results mismatch!\n%s\n%s", firstQueryString, + secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + private String createMetamorphicUnionQuery(YSQLSelect select, YSQLAggregate aggregate, List from) { + String metamorphicQuery; + YSQLExpression whereClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLExpression negatedClause = new YSQLPrefixOperation(whereClause, PrefixOperator.NOT); + YSQLExpression notNullClause = new YSQLPostfixOperation(whereClause, PostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + YSQLSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); + YSQLSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); + YSQLSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += YSQLVisitor.asString(leftSelect) + " UNION ALL " + YSQLVisitor.asString(middleSelect) + + " UNION ALL " + YSQLVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + } catch (PSQLException e) { + throw new AssertionError(queryString, e); + } + return resultString; + } + + private List mapped(YSQLAggregate aggregate) { + switch (aggregate.getFunction()) { + case SUM: + case COUNT: + case BIT_AND: + case BIT_OR: + case BOOL_AND: + case BOOL_OR: + case MAX: + case MIN: + return aliasArgs(Arrays.asList(aggregate)); + // case AVG: + //// List arg = Arrays.asList(new + // YSQLCast(aggregate.getExpr().get(0), + // YSQLDataType.DECIMAL.get())); + // YSQLAggregate sum = new YSQLAggregate(YSQLAggregateFunction.SUM, + // aggregate.getExpr()); + // YSQLCast count = new YSQLCast( + // new YSQLAggregate(YSQLAggregateFunction.COUNT, aggregate.getExpr()), + // YSQLDataType.DECIMAL.get()); + //// YSQLBinaryArithmeticOperation avg = new + // YSQLBinaryArithmeticOperation(sum, count, + // YSQLBinaryArithmeticOperator.DIV); + // return aliasArgs(Arrays.asList(sum, count)); + default: + throw new AssertionError(aggregate.getFunction()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (YSQLExpression expr : originalAggregateArgs) { + args.add(new YSQLAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(YSQLAggregate aggregate) { + switch (aggregate.getFunction()) { + // case AVG: + // return "SUM(agg0::DECIMAL)/SUM(agg1)::DECIMAL"; + case COUNT: + return YSQLAggregateFunction.SUM + "(agg0)"; + default: + return aggregate.getFunction().toString() + "(agg0)"; + } + } + + private YSQLSelect getSelect(List aggregates, List from, YSQLExpression whereClause, + List joinList) { + YSQLSelect leftSelect = new YSQLSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinClauses(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java new file mode 100644 index 000000000..359f12d92 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -0,0 +1,117 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; +import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; + +public class YSQLTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + protected YSQLSchema s; + protected YSQLTables targetTables; + protected YSQLExpressionGenerator gen; + protected YSQLSelect select; + + public YSQLTLPBase(YSQLGlobalState state) { + super(state); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + } + + public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState, String name, YSQLTables tables) { + List columns = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + YSQLSelect select = new YSQLSelect(); + select.setFromList(tables.getTables().stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(YSQLSelect.ForClause.getRandom()); + } + return new YSQLSelect.YSQLSubquery(select, name); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + List tables = targetTables.getTables(); + List joins = getJoinStatements(state, targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + protected List getJoinStatements(YSQLGlobalState globalState, List columns, + List tables) { + return YSQLNoRECOracle.getJoinStatements(state, columns, tables); + // TODO joins + } + + protected void generateSelectBase(List tables, List joins) { + List tableList = tables.stream() + .map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + gen = new YSQLExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new YSQLSelect(); + select.setFetchColumns(generateFetchColumns()); + select.setFromList(tableList); + select.setWhereClause(null); + select.setJoinClauses(joins); + if (Randomly.getBoolean()) { + select.setForClause(YSQLSelect.ForClause.getRandom()); + } + } + + List generateFetchColumns() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new YSQLColumnValue(YSQLColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(targetTables.getColumns()); + for (YSQLColumn c : targetColumns) { + fetchColumns.add(new YSQLColumnValue(c, null)); + } + return fetchColumns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java new file mode 100644 index 000000000..24410aee5 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java @@ -0,0 +1,66 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; + +public class YSQLTLPHavingOracle extends YSQLTLPBase { + + public YSQLTLPHavingOracle(YSQLGlobalState state) { + super(state); + YSQLCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + havingCheck(); + } + + @Override + List generateFetchColumns() { + List expressions = gen.allowAggregates(true).generateExpressions(Randomly.smallNumber() + 1); + gen.allowAggregates(false); + return expressions; + } + + protected void havingCheck() throws SQLException { + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(YSQLDataType.BOOLEAN)); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = YSQLVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + select.setHavingClause(predicate); + String firstQueryString = YSQLVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = YSQLVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = YSQLVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + protected YSQLExpression generatePredicate() { + return gen.generateHavingClause(); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java new file mode 100644 index 000000000..0af8095d0 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLVisitor; + +public class YSQLTLPWhereOracle extends YSQLTLPBase { + + public YSQLTLPWhereOracle(YSQLGlobalState state) { + super(state); + } + + @Override + public void check() throws SQLException { + super.check(); + whereCheck(); + } + + protected void whereCheck() throws SQLException { + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + String originalQueryString = YSQLVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setOrderByExpressions(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = YSQLVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = YSQLVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = YSQLVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } +} From 120bc079d4a141367500e2cfa28a6806657f5ada Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Tue, 4 Oct 2022 22:27:33 +0800 Subject: [PATCH 0616/1284] Support latest versions of CockroachDB and TiDB --- src/sqlancer/ComparatorHelper.java | 11 ++-- src/sqlancer/Randomly.java | 6 +- src/sqlancer/SQLProviderAdapter.java | 10 +++- src/sqlancer/cockroachdb/CockroachDBBugs.java | 45 +++++++++++++- .../cockroachdb/CockroachDBErrors.java | 60 +++++++++++++------ .../cockroachdb/CockroachDBOptions.java | 2 +- .../cockroachdb/CockroachDBSchema.java | 17 ++++++ .../cockroachdb/ast/CockroachDBConstant.java | 12 ++-- .../CockroachDBCreateStatisticsGenerator.java | 7 +-- .../gen/CockroachDBIndexGenerator.java | 19 ++++-- ...CockroachDBSetClusterSettingGenerator.java | 6 +- .../gen/CockroachDBSetSessionGenerator.java | 3 +- .../gen/CockroachDBTableGenerator.java | 30 +++------- .../gen/CockroachDBTruncateGenerator.java | 2 +- .../oracle/tlp/CockroachDBTLPBase.java | 2 +- .../tlp/CockroachDBTLPHavingOracle.java | 5 +- .../common/query/SQLQueryAdapter.java | 12 +--- src/sqlancer/tidb/TiDBBugs.java | 40 ++++++++----- src/sqlancer/tidb/TiDBErrors.java | 42 +++++++++---- .../tidb/TiDBExpressionGenerator.java | 43 +++++++++---- src/sqlancer/tidb/TiDBSchema.java | 34 +++++++++++ .../ast/TiDBBinaryComparisonOperation.java | 4 +- src/sqlancer/tidb/ast/TiDBFunctionCall.java | 5 +- src/sqlancer/tidb/ast/TiDBRegexOperation.java | 5 +- .../tidb/gen/TiDBAlterTableGenerator.java | 11 ++-- .../tidb/gen/TiDBAnalyzeTableGenerator.java | 9 +-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 5 +- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 53 +++++----------- .../tidb/gen/TiDBUpdateGenerator.java | 2 +- 29 files changed, 316 insertions(+), 186 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 145342ac4..016f41f75 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -57,16 +57,17 @@ public static List getResultSetFirstColumnAsString(String queryString, E throw new IgnoreMeException(); } while (result.next()) { - resultSet.add(result.getString(1)); + String resultTemp = result.getString(1); + if (resultTemp != null) { + resultTemp = resultTemp.replaceAll("[\\.]0+$", ""); + } + resultSet.add(resultTemp); // Remove the trailing zeros as many DBMS treat it as non-bugs } } catch (Exception e) { if (e instanceof IgnoreMeException) { throw e; } - if (e instanceof NumberFormatException) { - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/57 - throw new IgnoreMeException(); - } + if (e.getMessage() == null) { throw new AssertionError(queryString, e); } diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index c26e8aa17..fa317f1f0 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -159,13 +159,17 @@ public static List extractNrRandomColumns(List columns, int nr) { public static int smallNumber() { // no need to cache for small numbers - return (int) (Math.abs(getThreadRandom().get().nextGaussian()) * 2); + return (int) (Math.abs(getThreadRandom().get().nextGaussian())) * 2; } public static boolean getBoolean() { return getThreadRandom().get().nextBoolean(); } + public static double getPercentage() { + return getThreadRandom().get().nextDouble(); + } + private static ThreadLocal getThreadRandom() { if (THREAD_RANDOM.get() == null) { // a static method has been called, before Randomly was instantiated diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java index 1469ec0da..4f5a089ca 100644 --- a/src/sqlancer/SQLProviderAdapter.java +++ b/src/sqlancer/SQLProviderAdapter.java @@ -25,9 +25,15 @@ protected void checkViewsAreValid(G globalState) { for (AbstractTable view : views) { SQLQueryAdapter q = new SQLQueryAdapter("SELECT 1 FROM " + view.getName() + " LIMIT 1"); try { - q.execute(globalState); + if (!q.execute(globalState)) { + throw new AssertionError(); + } } catch (Throwable t) { - throw new IgnoreMeException(); + try { + globalState.executeStatement(new SQLQueryAdapter("DROP VIEW " + view.getName(), true)); + } catch (Throwable t2) { + throw new IgnoreMeException(); + } } } } diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 332f51c4c..3a980a8c7 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -3,13 +3,52 @@ public final class CockroachDBBugs { // https://github.com/cockroachdb/cockroach/issues/46915 - public static boolean bug46915 = true; + public static boolean bug46915; // https://github.com/cockroachdb/cockroach/issues/45703 - public static boolean bug45703 = true; + public static boolean bug45703; // https://github.com/cockroachdb/cockroach/issues/44757 - public static boolean bug44757 = true; + public static boolean bug44757; + + // https://github.com/cockroachdb/cockroach/issues/83792 + public static boolean bug83792 = true; + + // https://github.com/cockroachdb/cockroach/issues/83874 + public static boolean bug83874 = true; + + // https://github.com/cockroachdb/cockroach/issues/83973 + public static boolean bug83973 = true; + + // https://github.com/cockroachdb/cockroach/issues/83976 + public static boolean bug83976 = true; + + // https://github.com/cockroachdb/cockroach/issues/84154 + public static boolean bug84154 = true; + + // https://github.com/cockroachdb/cockroach/issues/85356 + public static boolean bug85356 = true; + + // https://github.com/cockroachdb/cockroach/issues/85371 + public static boolean bug85371 = true; + + // https://github.com/cockroachdb/cockroach/issues/85389 + public static boolean bug85389 = true; + + // https://github.com/cockroachdb/cockroach/issues/85390 + public static boolean bug85390 = true; + + // https://github.com/cockroachdb/cockroach/issues/85393 + public static boolean bug85393 = true; + + // https://github.com/cockroachdb/cockroach/issues/85394 + public static boolean bug85394 = true; + + // https://github.com/cockroachdb/cockroach/issues/85441 + public static boolean bug85441 = true; + + // https://github.com/cockroachdb/cockroach/issues/85499 + public static boolean bug85499 = true; private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 14c74d039..8fd79cb99 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -33,9 +33,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("expected -9223372036854775809 to be of type int, found type decimal"); errors.add("to be of type int4, found type decimal"); - errors.add("as type bool: invalid bool value"); - errors.add("as type int: strconv.ParseInt"); - errors.add("as type float: strconv.ParseFloat: parsing"); + errors.add("as type bool"); + errors.add("as type int"); + errors.add("as type float"); errors.add("is not in select list"); errors.add("non-integer constant in ORDER BY"); @@ -68,9 +68,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("unsupported comparison operator: = "); errors.add("unsupported comparison operator: <= "); errors.add("to be of type string, found type bytes"); - errors.add("unknown signature: left(string, int) (desired )"); errors.add("unknown signature: bit_length(collatedstring"); - errors.add("ERROR: unknown signature: left(collatedstring"); errors.add("unsupported comparison operator: !~ NOT LIKE )"); + errors.add("unknown signature: left"); errors.add("numeric constant out of int64 range"); - errors.add("unknown signature: overlay(string, string, decimal)"); - errors.add("unknown signature: substring(string, int, decimal)"); + errors.add("unknown signature: overlay"); + errors.add("unknown signature: substring"); errors.add("unsupported binary operator: + (desired )"); errors.add("unsupported comparison operator"); errors.add("unknown signature: chr(decimal) (desired )"); @@ -122,16 +119,13 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("incompatible value type: expected rowid to be of type decimal, found type int"); errors.add("unknown signature: to_english(decimal)"); errors.add("unknown signature: chr(decimal)"); - errors.add(" unknown signature: left(string, int2) (desired )"); - errors.add("unknown signature: split_part(string, string, decimal) (desired )"); - errors.add(" unknown signature: substring(string, "); + errors.add("unknown signature: split_part"); errors.add("division by zero"); errors.add("as int, found type: decimal"); errors.add("value type decimal doesn't match type int2 "); errors.add("has type decimal"); errors.add("to be of type decimal, found type int"); errors.add("value type decimal doesn't match type int"); - errors.add("unknown signature: substring(string, decimal, int)"); errors.add("unsupported binary operator: / (desired )"); errors.add("(desired )"); errors.add("(desired )"); @@ -139,8 +133,6 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("found type: decimal"); errors.add("(desired )"); errors.add("unknown signature: to_hex(decimal)"); - errors.add("unknown signature: split_part(string, string, decimal)"); - errors.add("unknown signature: left(bytes, decimal)"); errors.add("division undefined"); errors.add("decimal out of range"); errors.add("unknown signature: xor_agg(decimal)"); @@ -155,14 +147,45 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("could not parse JSON: unable to decode JSON: EOF"); errors.add("could not parse JSON: unable to decode JSON: unexpected EOF"); errors.add("can't order by column type jsonb"); + errors.add("odd length hex string"); // TODO: better control what is generated in a view errors.add("aggregate functions are not allowed in GROUP BY"); errors.add(" must appear in the GROUP BY clause or be used in an aggregate function"); - if (CockroachDBBugs.bug44757) { + if (CockroachDBBugs.bug83874) { errors.add("no builtin aggregate"); } + if (CockroachDBBugs.bug83792) { + errors.add("comparison overload not found"); + } + if (CockroachDBBugs.bug83973) { + errors.add("invalid memory address"); + } + if (CockroachDBBugs.bug83976) { + errors.add("cannot execute distinct on no columns"); + } + if (CockroachDBBugs.bug85356) { + errors.add("inconsistent Case return types"); + } + if (CockroachDBBugs.bug85371) { + errors.add("index out of range"); + } + if (CockroachDBBugs.bug85389) { + errors.add("no volatility for cast decimal"); + } + if (CockroachDBBugs.bug85390) { + errors.add("lookup for ComparisonExpr"); + } + if (CockroachDBBugs.bug85393) { + errors.add("no output column equivalent to"); + } + if (CockroachDBBugs.bug85441) { + errors.add("cannot cast jsonb numeric to type bool"); + } + if (CockroachDBBugs.bug85499) { + errors.add("estimated row count must be non-zero"); + } errors.add("unable to vectorize execution plan"); // SET vectorize=experimental_always; errors.add(" mismatched physical types at index"); // SET vectorize=experimental_always; @@ -223,7 +246,7 @@ private static void addArrayErrors(ExpectedErrors errors) { errors.add("unimplemented: nested arrays not supported"); // e.g., casting a string {{1}} to an array errors.add("malformed array"); - errors.add("https://github.com/cockroachdb/cockroach/issues/35707"); // arrays don't support ORDER BY + // errors.add("https://github.com/cockroachdb/cockroach/issues/35707"); // arrays don't support ORDER BY errors.add("as bytes[], found type: varbit[]"); errors.add("to be of type decimal[], found type float[]"); @@ -234,7 +257,7 @@ private static void addArrayErrors(ExpectedErrors errors) { private static void addIntervalTypeErrors(ExpectedErrors errors) { errors.add("overflow during Encode"); - errors.add("as type interval"); + errors.add("type interval"); } private static void addJoinTypes(ExpectedErrors errors) { @@ -269,7 +292,6 @@ private static void addFunctionErrors(ExpectedErrors errors) { // functions errors.add("abs of min integer value (-9223372036854775808) not defined"); // ABS errors.add("the input string must not be empty"); // ASCII - errors.add("unknown signature: substring(string, decimal)"); // overflow errors.add("overlay(): non-positive substring length not allowed"); // overlay errors.add("non-positive substring length not allowed"); // overlay errors.add("lpad(): requested length too large"); // lpad diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index f8ab8e07f..aa41259b0 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -97,7 +97,7 @@ public TestOracle create(CockroachDBGlobalState globalState) throws SQLException public boolean testHashIndexes = true; @Parameter(names = { "--test-temp-tables" }, description = "Test TEMPORARY tables") - public boolean testTempTables = true; + public boolean testTempTables; // default: false https://github.com/cockroachdb/cockroach/issues/85388 @Parameter(names = { "--increased-vectorization" }, description = "Generate VECTORIZE=on with a higher probability (which found a number of bugs in the past)") diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 5c21a1107..1c1dd3531 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -274,6 +274,14 @@ public CockroachDBTable(String tableName, List columns, List< } + public int getIndexCount() { + int count = 0; + for (CockroachDBTable table : getDatabaseTables()) { + count += table.getIndexes().size(); + } + return count; + } + public static CockroachDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); @@ -285,6 +293,10 @@ public static CockroachDBSchema fromConnection(SQLConnection con, String databas for (CockroachDBColumn c : databaseColumns) { c.setTable(t); } + // To avoid some situations that columns can not be got. + if (databaseColumns.isEmpty()) { + continue; + } databaseTables.add(t); } @@ -334,6 +346,11 @@ private static List getTableColumns(SQLConnection con, String isNullable); columns.add(c); } + } catch (SQLException e) { + if (CockroachDBBugs.bug85394 && e.getMessage().contains("incompatible type annotation for ARRAY")) { + return columns; + } + throw e; } } return columns; diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java b/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java index 84e2cb996..aa4dabcc4 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java @@ -230,20 +230,22 @@ public static CockroachDBExpression createBitConstantWithSize(int size) { } public static CockroachDBExpression createTimestampConstant(long integer) { - return new CockroachDBTimeRelatedConstant("TIMESTAMP", integer, "yyyy-MM-dd"); + return new CockroachDBTimeRelatedConstant("TIMESTAMP", integer, "yyyy-MM-dd'T'HH:mm:ss"); } public static CockroachDBExpression createTimeConstant(long integer) { - return new CockroachDBTimeRelatedConstant("TIME", integer, "HH:mm:ss"); + return new CockroachDBTimeRelatedConstant("TIME", integer, "yyyy-MM-dd'T'HH:mm:ss"); } public static CockroachDBExpression createTimetz(long integer) { - return new CockroachDBTimeRelatedConstant("TIMETZ", integer, "HH:mm:ss"); // TODO: support the complete format + return new CockroachDBTimeRelatedConstant("TIMETZ", integer, "yyyy-MM-dd'T'HH:mm:ss"); // TODO: support the + // complete format } public static CockroachDBExpression createTimestamptzConstant(long integer) { - return new CockroachDBTimeRelatedConstant("TIMESTAMPTZ", integer, "HH:mm:ss"); // TODO: support the complete - // format + return new CockroachDBTimeRelatedConstant("TIMESTAMPTZ", integer, "yyyy-MM-dd'T'HH:mm:ss"); // TODO: support the + // complete + // format } public static CockroachDBExpression createIntervalConstant(long year, long month, long day, long hour, long minute, diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java index 81920cd4f..b8895a34e 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java @@ -22,12 +22,7 @@ public static SQLQueryAdapter create(CockroachDBGlobalState globalState) { sb.append(" FROM "); sb.append(randomTable.getName()); - return new SQLQueryAdapter(sb.toString(), - ExpectedErrors.from("current transaction is aborted, commands ignored until end of transaction block", - "ERROR: unable to encode table key: *tree.DArray" /* - * https://github.com/cockroachdb/cockroach/ - * issues/46964 - */, "overflow during Encode")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("overflow during Encode")); // https://github.com/cockroachdb/cockroach/issues/84078 } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 66c2936fd..e5b708338 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -3,6 +3,7 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; @@ -30,6 +31,10 @@ public void buildStatement() { errors.add("the following columns are not indexable due to their type"); // array types are not indexable errors.add("cannot determine type of empty array. Consider annotating with the desired type"); errors.add("incompatible IF expression"); // TODO: investigate; seems to be a bug + if (CockroachDBBugs.bug84154) { + errors.add("overflow during Encode"); + errors.add("of type interval"); + } CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append("CREATE "); if (Randomly.getBoolean()) { @@ -43,15 +48,19 @@ public void buildStatement() { && Randomly.getBooleanWithSmallProbability(); if (hashSharded) { sb.append(" USING HASH WITH BUCKET_COUNT="); - sb.append(Randomly.getNotCachedInteger(2, Short.MAX_VALUE)); + sb.append(Randomly.getNotCachedInteger(2, 2048)); errors.add("null value in column"); errors.add("cannot create a sharded index on a computed column"); } if (Randomly.getBoolean()) { - sb.append(" "); - sb.append(Randomly.fromOptions("STORING", "COVERING")); - sb.append(" "); - addColumns(sb, table.getRandomNonEmptyColumnSubset(), false); + List columns2 = table.getRandomNonEmptyColumnSubset(); + columns2.removeAll(columns); + if (!columns2.isEmpty()) { + sb.append(" "); + sb.append(Randomly.fromOptions("STORING", "COVERING")); + sb.append(" "); + addColumns(sb, columns2, false); + } } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 2d411f26e..cef9862de 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -15,12 +15,12 @@ private CockroachDBSetClusterSettingGenerator() { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBClusterSetting { - BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), + // BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), - SQL_DEFAULTS_VECTORIZE_ROW_COUNT_THRESHOLD("sql.defaults.vectorize_row_count_threshold", - (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), + // SQL_DEFAULTS_VECTORIZE_ROW_COUNT_THRESHOLD("sql.defaults.vectorize_row_count_threshold", + // (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), // SQL_DEFAULTS_EXPERIMENTAL_OPTIMIZER_FOREIGN_KEYS_ENABLED("sql.defaults.experimental_optimizer_foreign_keys.enabled", // CockroachDBSetSessionGenerator::onOff), SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index 48bb42fb7..e99eaaa27 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -29,8 +29,7 @@ private enum CockroachDBSetting { SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), // EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), REORDER_JOINS_LIMIT((g) -> g.getRandomly().getInteger(0, Integer.MAX_VALUE)), // - SQL_SAFE_UPDATES((g) -> "off"), - // TRACING(CockroachDBSetSessionGenerator::onOff) + SQL_SAFE_UPDATES((g) -> "off"), TRACING(CockroachDBSetSessionGenerator::onOff), /* * CockroachDB enables vectorized (column-oriented) execution by default. Row-oriented execution can be enforced * by setting vectorized to "off". Some examples of bugs found in the vectorized execution engine are: diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index 720f2a300..4e99ab31a 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -72,7 +72,13 @@ public void buildStatement() { && cockroachDBColumn.getType().getPrimitiveDataType() != CockroachDBDataType.SERIAL; if (generatedColumn) { sb.append(" AS ("); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(cockroachDBColumn.getType()))); + // To generate an expression exclude of the current column + List generatedColumns = new ArrayList<>(columns); + generatedColumns.remove(i); + CockroachDBExpressionGenerator genGeneratedColumn = new CockroachDBExpressionGenerator(globalState) + .setColumns(generatedColumns); + sb.append(CockroachDBVisitor + .asString(genGeneratedColumn.generateExpression(cockroachDBColumn.getType()))); sb.append(") STORED"); errors.add("computed columns cannot reference other computed columns"); errors.add("context-dependent operators are not allowed in computed column"); @@ -144,28 +150,6 @@ public void buildStatement() { sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); sb.append(")"); } - if (Randomly.getBoolean() && !globalState.getSchema().getDatabaseTables().isEmpty()) { - sb.append(", "); - // TODO: also allow referencing itself - List subset = Randomly.nonEmptySubset(columns); - sb.append(" FOREIGN KEY ("); - sb.append(subset.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(") REFERENCES "); - CockroachDBTable otherTable = globalState.getSchema().getRandomTable(); - sb.append(otherTable.getName()); - sb.append("("); - for (int i = 0; i < subset.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(otherTable.getRandomColumn().getName()); - } - sb.append(")"); - // TODO: ensure that the column types match - errors.add("does not match foreign key"); - errors.add("computed column"); - errors.add("there is no unique constraint matching given keys for referenced table"); - } sb.append(")"); errors.add("collatedstring"); CockroachDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index 02579cf32..d10000352 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -16,7 +16,7 @@ public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { errors.add("is referenced by foreign key"); // https://github.com/cockroachdb/cockroach/issues/47030 - errors.add("unexpected value: "); + // errors.add("unexpected value: "); StringBuilder sb = new StringBuilder(); sb.append("TRUNCATE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index e08389bf5..7e7f03e26 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -54,7 +54,7 @@ public void check() throws SQLException { List generateFetchColumns() { List columns = new ArrayList<>(); - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() || targetTables.getColumns().size() == 0) { columns.add(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); } else { columns.addAll(Randomly.nonEmptySubset(targetTables.getColumns()).stream() diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java index d5fcf9812..42abad00b 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java @@ -33,11 +33,12 @@ public void check() throws SQLException { String originalQueryString = CockroachDBVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); select.setHavingClause(predicate); String firstQueryString = CockroachDBVisitor.asString(select); - select.setHavingClause(negatedPredicate); + select.setHavingClause(gen.negatePredicate(predicate)); String secondQueryString = CockroachDBVisitor.asString(select); - select.setHavingClause(isNullPredicate); + select.setHavingClause(gen.isNull(predicate)); String thirdQueryString = CockroachDBVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 01fcc8539..c2e6706cc 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -96,17 +96,9 @@ public String getUnterminatedQueryString() { } public void checkException(Exception e) throws AssertionError { - Throwable ex = e; - - while (ex != null) { - if (expectedErrors.errorIsExpected(ex.getMessage())) { - return; - } else { - ex = ex.getCause(); - } + if (!expectedErrors.errorIsExpected(e.getMessage())) { + throw new AssertionError(query, e); } - - throw new AssertionError(query, e); } @Override diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index 08d5d423d..a4f5cc2dd 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -4,46 +4,58 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/15987 - public static boolean bug15987 = true; + public static boolean bug15987; // // https://github.com/pingcap/tidb/issues/15988 - public static boolean bug15988 = true; + public static boolean bug15988; // https://github.com/pingcap/tidb/issues/16028 - public static boolean bug16028 = true; + public static boolean bug16028; // https://github.com/pingcap/tidb/issues/16020 - public static boolean bug16020 = true; + public static boolean bug16020; // https://github.com/pingcap/tidb/issues/15990 - public static boolean bug15990 = true; + public static boolean bug15990; // https://github.com/pingcap/tidb/issues/15844 - public static boolean bug15844 = true; + public static boolean bug15844; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/10 - public static boolean bug10 = true; + public static boolean bug10; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/14 - public static boolean bug14 = true; + public static boolean bug14; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/15 - public static boolean bug15 = true; + public static boolean bug15; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/16 - public static boolean bug16 = true; + public static boolean bug16; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/19 - public static boolean bug19 = true; + public static boolean bug19; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/48 - public static boolean bug48 = true; + public static boolean bug48; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/50 - public static boolean bug50 = true; + public static boolean bug50; // https://github.com/citusdata/citus/issues/4079 - public static boolean bug4079 = true; + public static boolean bug4079; + + // https://github.com/pingcap/tidb/issues/35677 + public static boolean bug35677 = true; + + // https://github.com/pingcap/tidb/issues/35522 + public static boolean bug35522 = true; + + // https://github.com/pingcap/tidb/issues/35652 + public static boolean bug35652 = true; + + // https://github.com/pingcap/tidb/issues/38295 + public static boolean bug38295 = true; private TiDBBugs() { } diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index bb3d3ca62..95979264c 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -25,13 +25,14 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("is not valid for CHARACTER SET"); // known issue: https://github.com/pingcap/tidb/issues/14819 - errors.add("Wrong plan type for dataReaderBuilder"); + // errors.add("Wrong plan type for dataReaderBuilder"); errors.add("DOUBLE value is out of range"); - errors.add("index out of range"); // https://github.com/pingcap/tidb/issues/15810 - errors.add("baseBuiltinFunc.evalString() should never be called, please contact the TiDB team for help"); // https://github.com/pingcap/tidb/issues/15847 - errors.add("unsupport column type for encode 6"); // https://github.com/pingcap/tidb/issues/15850 + // errors.add("index out of range"); // https://github.com/pingcap/tidb/issues/15810 + // errors.add("baseBuiltinFunc.evalString() should never be called, please contact the TiDB team for help"); // + // https://github.com/pingcap/tidb/issues/15847 + // errors.add("unsupport column type for encode 6"); // https://github.com/pingcap/tidb/issues/15850 errors.add("Data truncation: %s value is out of range in '%s'"); errors.add("Truncated incorrect FLOAT value"); @@ -39,13 +40,32 @@ public static void addExpressionErrors(ExpectedErrors errors) { // regex errors.add("error parsing regexp"); + errors.add("from regexp"); + + // To avoid bugs + errors.add("Unknown column"); // https://github.com/pingcap/tidb/issues/35522 + errors.add("Can\'t find column"); // https://github.com/pingcap/tidb/issues/35527 + errors.add("Cannot convert"); // https://github.com/pingcap/tidb/issues/35652 // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/57 - errors.add("For input string: \"+Inf\""); + // errors.add("For input string: \"+Inf\""); + + // errors.add("inconsistent index"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/58 - errors.add("inconsistent index"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/58 + // errors.add("Illegal mix of collations"); - errors.add("Illegal mix of collations"); + if (TiDBBugs.bug35677) { + errors.add("for function inet_aton"); + } + if (TiDBBugs.bug35522) { + errors.add("ERROR 1054 (42S22)"); + } + if (TiDBBugs.bug35652) { + errors.add("from binary to utf8"); + } + if (TiDBBugs.bug38295) { + errors.add("assertion failed"); + } } public static void addExpressionHavingErrors(ExpectedErrors errors) { @@ -73,10 +93,10 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("error parsing regexp"); errors.add("is not valid for CHARACTER SET"); - if (true) { - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/54 - errors.add("Miss column"); - } + // if (true) { + // // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/54 + // errors.add("Miss column"); + // } } } diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index a960363fe..20f48f04a 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -20,7 +20,6 @@ import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; import sqlancer.tidb.ast.TiDBCase; import sqlancer.tidb.ast.TiDBCastOperation; -import sqlancer.tidb.ast.TiDBCollate; import sqlancer.tidb.ast.TiDBColumnReference; import sqlancer.tidb.ast.TiDBConstant; import sqlancer.tidb.ast.TiDBExpression; @@ -47,7 +46,7 @@ private enum Gen { UNARY_POSTFIX, // CONSTANT, // COLUMN, // - COMPARISON, REGEX, COLLATE, FUNCTION, BINARY_LOGICAL, BINARY_BIT, CAST, DEFAULT, CASE + COMPARISON, REGEX, FUNCTION, BINARY_LOGICAL, BINARY_BIT, CAST, DEFAULT, CASE // BINARY_ARITHMETIC } @@ -86,9 +85,9 @@ protected TiDBExpression generateExpression(int depth) { case REGEX: return new TiDBRegexOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBRegexOperator.getRandom()); - case COLLATE: - return new TiDBCollate(generateExpression(depth + 1), - Randomly.fromOptions("utf8mb4_bin", "latin1_bin", "binary", "ascii_bin", "utf8_bin")); + // case COLLATE: + // return new TiDBCollate(generateExpression(depth + 1), + // Randomly.fromOptions("utf8mb4_bin", "latin1_bin", "binary", "ascii_bin", "utf8_bin")); case FUNCTION: TiDBFunction func = TiDBFunction.getRandom(); return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); @@ -105,13 +104,9 @@ protected TiDBExpression generateExpression(int depth) { // return new TiDBBinaryArithmeticOperation(generateExpression(depth + 1), generateExpression(depth + 1), // TiDBBinaryArithmeticOperator.getRandom()); case CAST: - return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions( - /* - * "BINARY" https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 - */ "CHAR", - /* - * "DATE", "DATETIME", "TIME", https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 - */ "DECIMAL", "SIGNED"/* , "UNSIGNED" https://github.com/pingcap/tidb/issues/16028 */)); + return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions("BINARY", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 + "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 + "DECIMAL", "SIGNED", "UNSIGNED" /* https://github.com/pingcap/tidb/issues/16028 */)); case CASE: if (TiDBBugs.bug19) { throw new IgnoreMeException(); @@ -180,4 +175,28 @@ public TiDBExpression isNull(TiDBExpression expr) { return new TiDBUnaryPostfixOperation(expr, TiDBUnaryPostfixOperator.IS_NULL); } + public TiDBExpression generateConstant(TiDBDataType type) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return TiDBConstant.createNullConstant(); + } + switch (type) { + case INT: + return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + case BLOB: + case TEXT: + return TiDBConstant.createStringConstant(globalState.getRandomly().getString()); + case BOOL: + return TiDBConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOATING: + return TiDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); + case CHAR: + return TiDBConstant.createStringConstant(globalState.getRandomly().getChar()); + case DECIMAL: + case NUMERIC: + return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(); + } + } + } diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index a611543fc..b147ab8ab 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -56,6 +56,23 @@ public boolean isNumeric() { throw new AssertionError(this); } } + + public boolean canHaveDefault() { + switch (this) { + case INT: + case DECIMAL: + case FLOATING: + case BOOL: + case CHAR: + return true; + case NUMERIC: + case TEXT: + case BLOB: + return false; + default: + throw new AssertionError(this); + } + } } public static class TiDBCompositeDataType { @@ -176,6 +193,14 @@ public TiDBTables getRandomTableNonEmptyTables() { return new TiDBTables(Randomly.nonEmptySubset(getDatabaseTables())); } + public int getIndexCount() { + int count = 0; + for (TiDBTable table : getDatabaseTables()) { + count += table.getIndexes().size(); + } + return count; + } + private static TiDBCompositeDataType getColumnType(String typeString) { String trimmedStringType = typeString.replace(" zerofill", "").replace(" unsigned", ""); if (trimmedStringType.contains("decimal")) { @@ -211,14 +236,17 @@ private static TiDBCompositeDataType getColumnType(String typeString) { case "null": primitiveType = TiDBDataType.INT; break; + case "tinyint(3)": case "tinyint(4)": primitiveType = TiDBDataType.INT; size = 1; break; + case "smallint(5)": case "smallint(6)": primitiveType = TiDBDataType.INT; size = 2; break; + case "int(10)": case "int(11)": primitiveType = TiDBDataType.INT; size = 4; @@ -252,6 +280,10 @@ public static TiDBSchema fromConnection(SQLConnection con, String databaseName) List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); + // Ignore invalid views + if (databaseColumns.isEmpty()) { + continue; + } List indexes = getIndexes(con, tableName); boolean isView = tableName.startsWith("v"); TiDBTable t = new TiDBTable(tableName, databaseColumns, indexes, isView); @@ -302,6 +334,8 @@ private static List getTableColumns(SQLConnection con, String tableN columns.add(c); } } + } catch (SQLException e) { + } return columns; } diff --git a/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java index 4efc554f0..c7240e4b7 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java @@ -14,8 +14,8 @@ public enum TiDBComparisonOperator implements Operator { GREATER_EQUALS(">="), // SMALLER("<"), // SMALLER_EQUALS("<="), // - NOT_EQUALS("!="); // - // NULL_SAFE_EQUALS("<=>"); https://github.com/tidb-challenge-program/bug-hunting-issue/issues/5 + NOT_EQUALS("!="), // + NULL_SAFE_EQUALS("<=>"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/5 private String textRepr; diff --git a/src/sqlancer/tidb/ast/TiDBFunctionCall.java b/src/sqlancer/tidb/ast/TiDBFunctionCall.java index 4aded9ad2..1aaa87827 100644 --- a/src/sqlancer/tidb/ast/TiDBFunctionCall.java +++ b/src/sqlancer/tidb/ast/TiDBFunctionCall.java @@ -92,11 +92,10 @@ public int getNrArgs() { // [...] REPLACE(3), // REVERSE(1), // - RIGHT(2), // + // RIGHT(2), // https://github.com/pingcap/tidb/issues/35623 // RPAD TODO RTRIM(1), // - // SPACE(1), - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/6 + SPACE(1), // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/6 STRCMP(2), // SUBSTRING(2), // TODO: support other versions SUBSTRING_INDEX(3), // diff --git a/src/sqlancer/tidb/ast/TiDBRegexOperation.java b/src/sqlancer/tidb/ast/TiDBRegexOperation.java index f5f011dd6..c8da018d8 100644 --- a/src/sqlancer/tidb/ast/TiDBRegexOperation.java +++ b/src/sqlancer/tidb/ast/TiDBRegexOperation.java @@ -9,8 +9,9 @@ public class TiDBRegexOperation extends BinaryOperatorNode !t.isView()); TiDBColumn column = table.getRandomColumn(); @@ -58,9 +64,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append(Randomly.fromOptions("ENABLE", "DISABLE")); sb.append(" KEYS"); break; - case FORCE: - sb.append("FORCE"); - break; case DROP_PRIMARY_KEY: if (!column.isPrimaryKey()) { throw new IgnoreMeException(); diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index b52e2cc25..d84b117c2 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -3,7 +3,6 @@ import java.sql.SQLException; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -31,13 +30,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(Randomly.getNotCachedInteger(1, 1024)); sb.append(" BUCKETS"); } - return new SQLQueryAdapter(sb.toString(), - ExpectedErrors.from("https://github.com/pingcap/tidb/issues/15993", - /* https://github.com/pingcap/tidb/issues/15993 */ "doesn't have a default value", - "Index 'PRIMARY' in field list does not exist in table" /* - * https://github. com/pingcap/tidb/ - * issues/15993 - */)); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 5ed96f707..b48ea31cc 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -45,9 +45,8 @@ private enum Action { TIDB_ENABLE_FAST_ANALYZE("tidb_enable_fast_analyze", (r) -> Randomly.fromOptions(0, 1)), // TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), - // TODO: global - // TIDB_SCATTER_REGION("tidb_scatter_region", (r) -> Randomly.fromOptions(0, 1)); - TIDB_ENABLE_STMT_SUMMARY("tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // + TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions(0, 1)), + TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // TIDB_ENABLE_CHUNK_RPC("tidb_enable_chunk_rpc", (r) -> Randomly.fromOptions(0, 1)); private String name; diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index c252a7b9c..97859d1ad 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -59,10 +58,7 @@ private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { sb.append(columns.get(i).getName()); sb.append(" "); TiDBCompositeDataType type; - do { - type = TiDBCompositeDataType.getRandom(); - } while (type.getPrimitiveDataType() == TiDBDataType.INT && type.getSize() < 4 - || type.getPrimitiveDataType() == TiDBDataType.BOOL); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/49 + type = TiDBCompositeDataType.getRandom(); appendType(sb, type); sb.append(" "); boolean isGeneratedColumn = Randomly.getBooleanWithRatherLowProbability(); @@ -72,7 +68,6 @@ private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { sb.append(") "); sb.append(Randomly.fromOptions("STORED", "VIRTUAL")); sb.append(" "); - errors.add("You have an error in your SQL syntax"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/53 errors.add("Generated column can refer only to generated columns defined prior to it"); errors.add( "'Defining a virtual generated column as primary key' is not supported for generated columns."); @@ -87,10 +82,9 @@ private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { if (Randomly.getBooleanWithRatherLowProbability()) { sb.append("NOT NULL "); } - if (Randomly.getBoolean() && type.getPrimitiveDataType() != TiDBDataType.TEXT - && type.getPrimitiveDataType() != TiDBDataType.BLOB && !isGeneratedColumn) { + if (Randomly.getBoolean() && type.getPrimitiveDataType().canHaveDefault() && !isGeneratedColumn) { sb.append("DEFAULT "); - sb.append(TiDBVisitor.asString(gen.generateConstant())); + sb.append(TiDBVisitor.asString(gen.generateConstant(type.getPrimitiveDataType()))); sb.append(" "); errors.add("Invalid default value"); errors.add( @@ -137,27 +131,16 @@ && canUseAsUnique(type) && !isGeneratedColumn) { errors.add("UnknownType: *ast.WhenClause"); } } - List actions = Randomly.nonEmptySubset(Action.values()); - for (Action a : actions) { - sb.append(" "); - switch (a) { - case AUTO_INCREMENT: - sb.append("AUTO_INCREMENT="); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - break; - case PRE_SPLIT_REGIONS: - sb.append("PRE_SPLIT_REGIONS="); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - break; - case SHARD_ROW_ID_BITS: - sb.append("SHARD_ROW_ID_BITS="); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - errors.add("Unsupported shard_row_id_bits for table with primary key as row id"); - break; - default: - throw new AssertionError(a); - } - } + // Buggy code and disable it temporarily + /* + * List actions = Randomly.nonEmptySubset(Action.values()); for (Action a : actions) { sb.append(" "); + * switch (a) { case AUTO_INCREMENT: sb.append("AUTO_INCREMENT="); sb.append(Randomly.getNotCachedInteger(0, + * Integer.MAX_VALUE)); break; case PRE_SPLIT_REGIONS: sb.append("PRE_SPLIT_REGIONS="); + * sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); break; case SHARD_ROW_ID_BITS: + * sb.append("SHARD_ROW_ID_BITS="); sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); + * errors.add("Unsupported shard_row_id_bits for table with primary key as row id"); break; default: throw new + * AssertionError(a); } } + */ } private boolean canUseAsUnique(TiDBCompositeDataType type) { @@ -165,18 +148,14 @@ private boolean canUseAsUnique(TiDBCompositeDataType type) { } private void appendType(StringBuilder sb, TiDBCompositeDataType type) { - if (type.getPrimitiveDataType() == TiDBDataType.CHAR) { - throw new IgnoreMeException(); - } + // if (type.getPrimitiveDataType() == TiDBDataType.CHAR) { + // throw new IgnoreMeException(); + // } sb.append(type.toString()); appendSpecifiers(sb, type.getPrimitiveDataType()); appendSizeSpecifiers(sb, type.getPrimitiveDataType()); } - private enum Action { - AUTO_INCREMENT, PRE_SPLIT_REGIONS, SHARD_ROW_ID_BITS - } - private void appendSizeSpecifiers(StringBuilder sb, TiDBDataType type) { if (type.isNumeric() && Randomly.getBoolean() && !TiDBBugs.bug16028) { sb.append(" UNSIGNED"); diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 38eba8e78..0c2459b28 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -43,7 +43,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); sb.append(TiDBVisitor.asString(gen.generateExpression())); - errors.add("Data Too Long"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/43 + // errors.add("Data Too Long"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/43 } TiDBErrors.addInsertErrors(errors); From 1bbbe7989fd10b2a89cd329df3cc0e49a0b7094a Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Wed, 5 Oct 2022 10:52:04 +0800 Subject: [PATCH 0617/1284] clean comments --- src/sqlancer/ComparatorHelper.java | 5 +++-- src/sqlancer/SQLProviderAdapter.java | 16 +++++++++----- .../cockroachdb/CockroachDBErrors.java | 2 -- .../cockroachdb/CockroachDBSchema.java | 2 +- ...CockroachDBSetClusterSettingGenerator.java | 5 ----- .../gen/CockroachDBSetSessionGenerator.java | 2 -- .../gen/CockroachDBTruncateGenerator.java | 2 -- .../common/query/SQLQueryAdapter.java | 12 ++++++++-- src/sqlancer/tidb/TiDBErrors.java | 22 ------------------- src/sqlancer/tidb/TiDBSchema.java | 3 +-- src/sqlancer/tidb/ast/TiDBFunctionCall.java | 2 +- src/sqlancer/tidb/ast/TiDBRegexOperation.java | 5 ++--- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 13 ----------- .../tidb/gen/TiDBUpdateGenerator.java | 1 - 14 files changed, 28 insertions(+), 64 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 016f41f75..880d3ae72 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -59,9 +59,10 @@ public static List getResultSetFirstColumnAsString(String queryString, E while (result.next()) { String resultTemp = result.getString(1); if (resultTemp != null) { - resultTemp = resultTemp.replaceAll("[\\.]0+$", ""); + resultTemp = resultTemp.replaceAll("[\\.]0+$", ""); // Remove the trailing zeros as many DBMS treat + // it as non-bugs } - resultSet.add(resultTemp); // Remove the trailing zeros as many DBMS treat it as non-bugs + resultSet.add(resultTemp); } } catch (Exception e) { if (e instanceof IgnoreMeException) { diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java index 4f5a089ca..8de7523a3 100644 --- a/src/sqlancer/SQLProviderAdapter.java +++ b/src/sqlancer/SQLProviderAdapter.java @@ -26,15 +26,19 @@ protected void checkViewsAreValid(G globalState) { SQLQueryAdapter q = new SQLQueryAdapter("SELECT 1 FROM " + view.getName() + " LIMIT 1"); try { if (!q.execute(globalState)) { - throw new AssertionError(); + dropView(globalState, view.getName()); } } catch (Throwable t) { - try { - globalState.executeStatement(new SQLQueryAdapter("DROP VIEW " + view.getName(), true)); - } catch (Throwable t2) { - throw new IgnoreMeException(); - } + dropView(globalState, view.getName()); } } } + + private void dropView(G globalState, String viewName) { + try { + globalState.executeStatement(new SQLQueryAdapter("DROP VIEW " + viewName, true)); + } catch (Throwable t2) { + throw new IgnoreMeException(); + } + } } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 8fd79cb99..66a5e5d7d 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -246,8 +246,6 @@ private static void addArrayErrors(ExpectedErrors errors) { errors.add("unimplemented: nested arrays not supported"); // e.g., casting a string {{1}} to an array errors.add("malformed array"); - // errors.add("https://github.com/cockroachdb/cockroach/issues/35707"); // arrays don't support ORDER BY - errors.add("as bytes[], found type: varbit[]"); errors.add("to be of type decimal[], found type float[]"); errors.add("to be of type int[], found type decimal[]"); diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 1c1dd3531..c245f4cce 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -293,7 +293,7 @@ public static CockroachDBSchema fromConnection(SQLConnection con, String databas for (CockroachDBColumn c : databaseColumns) { c.setTable(t); } - // To avoid some situations that columns can not be got. + // To avoid some situations that columns can not be retrieved. if (databaseColumns.isEmpty()) { continue; } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index cef9862de..42740506d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -15,14 +15,9 @@ private CockroachDBSetClusterSettingGenerator() { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBClusterSetting { - // BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), - // SQL_DEFAULTS_VECTORIZE_ROW_COUNT_THRESHOLD("sql.defaults.vectorize_row_count_threshold", - // (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), - // SQL_DEFAULTS_EXPERIMENTAL_OPTIMIZER_FOREIGN_KEYS_ENABLED("sql.defaults.experimental_optimizer_foreign_keys.enabled", - // CockroachDBSetSessionGenerator::onOff), SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), SQL_STATS_HISTOGRAM_COLLECTION_ENABLED("sql.stats.histogram_collection.enabled", CockroachDBSetSessionGenerator::onOff); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index e99eaaa27..0ade4a3ef 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -25,9 +25,7 @@ private enum CockroachDBSetting { ENABLE_IMPLICIT_SELECT_FOR_UPDATE(CockroachDBSetSessionGenerator::onOff), ENABLE_INSERT_FAST_PATH(CockroachDBSetSessionGenerator::onOff), ENABLE_ZIGZAG_JOIN(CockroachDBSetSessionGenerator::onOff), - // EXPERIMENTAL_ENABLE_HASH_SHARDED_INDEXES(CockroachDBSetSessionGenerator::onOff), SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), - // EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), REORDER_JOINS_LIMIT((g) -> g.getRandomly().getInteger(0, Integer.MAX_VALUE)), // SQL_SAFE_UPDATES((g) -> "off"), TRACING(CockroachDBSetSessionGenerator::onOff), /* diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index d10000352..904ee3a9f 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -15,8 +15,6 @@ public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add("is referenced by foreign key"); - // https://github.com/cockroachdb/cockroach/issues/47030 - // errors.add("unexpected value: "); StringBuilder sb = new StringBuilder(); sb.append("TRUNCATE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index c2e6706cc..01fcc8539 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -96,9 +96,17 @@ public String getUnterminatedQueryString() { } public void checkException(Exception e) throws AssertionError { - if (!expectedErrors.errorIsExpected(e.getMessage())) { - throw new AssertionError(query, e); + Throwable ex = e; + + while (ex != null) { + if (expectedErrors.errorIsExpected(ex.getMessage())) { + return; + } else { + ex = ex.getCause(); + } } + + throw new AssertionError(query, e); } @Override diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 95979264c..f37bdb685 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -21,19 +21,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { // functions errors.add("BIGINT value is out of range"); errors.add("doesn't have a default value"); // default - errors.add("is not valid for CHARACTER SET"); - - // known issue: https://github.com/pingcap/tidb/issues/14819 - // errors.add("Wrong plan type for dataReaderBuilder"); - errors.add("DOUBLE value is out of range"); - // errors.add("index out of range"); // https://github.com/pingcap/tidb/issues/15810 - // errors.add("baseBuiltinFunc.evalString() should never be called, please contact the TiDB team for help"); // - // https://github.com/pingcap/tidb/issues/15847 - // errors.add("unsupport column type for encode 6"); // https://github.com/pingcap/tidb/issues/15850 - errors.add("Data truncation: %s value is out of range in '%s'"); errors.add("Truncated incorrect FLOAT value"); errors.add("Bad Number"); @@ -47,13 +37,6 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Can\'t find column"); // https://github.com/pingcap/tidb/issues/35527 errors.add("Cannot convert"); // https://github.com/pingcap/tidb/issues/35652 - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/57 - // errors.add("For input string: \"+Inf\""); - - // errors.add("inconsistent index"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/58 - - // errors.add("Illegal mix of collations"); - if (TiDBBugs.bug35677) { errors.add("for function inet_aton"); } @@ -92,11 +75,6 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Incorrect decimal value"); errors.add("error parsing regexp"); errors.add("is not valid for CHARACTER SET"); - - // if (true) { - // // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/54 - // errors.add("Miss column"); - // } } } diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index b147ab8ab..51bdd6072 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -334,8 +334,7 @@ private static List getTableColumns(SQLConnection con, String tableN columns.add(c); } } - } catch (SQLException e) { - + } catch (SQLException e) { // Happens when } return columns; } diff --git a/src/sqlancer/tidb/ast/TiDBFunctionCall.java b/src/sqlancer/tidb/ast/TiDBFunctionCall.java index 1aaa87827..c7bdb2d99 100644 --- a/src/sqlancer/tidb/ast/TiDBFunctionCall.java +++ b/src/sqlancer/tidb/ast/TiDBFunctionCall.java @@ -92,7 +92,7 @@ public int getNrArgs() { // [...] REPLACE(3), // REVERSE(1), // - // RIGHT(2), // https://github.com/pingcap/tidb/issues/35623 + RIGHT(2), // // RPAD TODO RTRIM(1), // SPACE(1), // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/6 diff --git a/src/sqlancer/tidb/ast/TiDBRegexOperation.java b/src/sqlancer/tidb/ast/TiDBRegexOperation.java index c8da018d8..f5f011dd6 100644 --- a/src/sqlancer/tidb/ast/TiDBRegexOperation.java +++ b/src/sqlancer/tidb/ast/TiDBRegexOperation.java @@ -9,9 +9,8 @@ public class TiDBRegexOperation extends BinaryOperatorNode actions = Randomly.nonEmptySubset(Action.values()); for (Action a : actions) { sb.append(" "); - * switch (a) { case AUTO_INCREMENT: sb.append("AUTO_INCREMENT="); sb.append(Randomly.getNotCachedInteger(0, - * Integer.MAX_VALUE)); break; case PRE_SPLIT_REGIONS: sb.append("PRE_SPLIT_REGIONS="); - * sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); break; case SHARD_ROW_ID_BITS: - * sb.append("SHARD_ROW_ID_BITS="); sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - * errors.add("Unsupported shard_row_id_bits for table with primary key as row id"); break; default: throw new - * AssertionError(a); } } - */ } private boolean canUseAsUnique(TiDBCompositeDataType type) { @@ -148,9 +138,6 @@ private boolean canUseAsUnique(TiDBCompositeDataType type) { } private void appendType(StringBuilder sb, TiDBCompositeDataType type) { - // if (type.getPrimitiveDataType() == TiDBDataType.CHAR) { - // throw new IgnoreMeException(); - // } sb.append(type.toString()); appendSpecifiers(sb, type.getPrimitiveDataType()); appendSizeSpecifiers(sb, type.getPrimitiveDataType()); diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 0c2459b28..4912899fe 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -43,7 +43,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); sb.append(TiDBVisitor.asString(gen.generateExpression())); - // errors.add("Data Too Long"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/43 } TiDBErrors.addInsertErrors(errors); From f99004f74cd938b14747706384b7ccf1a7645cfb Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Wed, 5 Oct 2022 16:08:44 +0800 Subject: [PATCH 0618/1284] Bypass errors in Alter --- src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index fb0d42892..057bb2789 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -31,6 +31,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("charset"); errors.add("not supported"); errors.add("SQL syntax"); + errors.add("can't drop"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); From d57fdbfee07f43556f8bc6d08b009cb46a554884 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Wed, 5 Oct 2022 16:50:28 +0800 Subject: [PATCH 0619/1284] support datetime in TiDB --- src/sqlancer/tidb/TiDBSchema.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 51bdd6072..7caff2974 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -256,6 +256,13 @@ private static TiDBCompositeDataType getColumnType(String typeString) { case "tinyblob": primitiveType = TiDBDataType.BLOB; break; + case "date": + case "datetime": + case "timestamp": + case "time": + case "year": + primitiveType = TiDBDataType.NUMERIC; + break; default: throw new AssertionError(trimmedStringType); } From 8b8f4e4853edb9e30c10ff3b578fb32f5a975b48 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 13:05:14 +0400 Subject: [PATCH 0620/1284] Minor review fixes --- src/sqlancer/yugabyte/YugabyteBugs.java | 6 ++++ src/sqlancer/yugabyte/ycql/YCQLProvider.java | 11 +++--- src/sqlancer/yugabyte/ycql/YCQLSchema.java | 7 +--- src/sqlancer/yugabyte/ysql/YSQLProvider.java | 29 +++++++-------- .../ysql/gen/YSQLAlterTableGenerator.java | 4 +-- .../yugabyte/ysql/gen/YSQLCommon.java | 23 ++---------- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 15 ++------ .../ysql/gen/YSQLTruncateGenerator.java | 12 ------- .../ysql/gen/YSQLVacuumGenerator.java | 5 +-- .../yugabyte/ysql/gen/YSQLViewGenerator.java | 14 -------- .../yugabyte/ysql/oracle/YSQLCatalog.java | 8 +++-- .../yugabyte/ysql/oracle/YSQLFuzzer.java | 36 +++---------------- 12 files changed, 45 insertions(+), 125 deletions(-) create mode 100644 src/sqlancer/yugabyte/YugabyteBugs.java diff --git a/src/sqlancer/yugabyte/YugabyteBugs.java b/src/sqlancer/yugabyte/YugabyteBugs.java new file mode 100644 index 000000000..1e944e0e2 --- /dev/null +++ b/src/sqlancer/yugabyte/YugabyteBugs.java @@ -0,0 +1,6 @@ +package sqlancer.yugabyte; + +public interface YugabyteBugs { + // https://github.com/yugabyte/yugabyte-db/issues/11357 + boolean bug11357 = true; +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLProvider.java b/src/sqlancer/yugabyte/ycql/YCQLProvider.java index 3d408f8c1..8bc8cbf84 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLProvider.java +++ b/src/sqlancer/yugabyte/ycql/YCQLProvider.java @@ -122,9 +122,10 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep e.printStackTrace(); } final String host = globalState.getOptions().getHost(); - final String url = "jdbc:cassandra://%s:9042/%s?localdatacenter=%s"; + final int port = globalState.getOptions().getPort(); + final String url = "jdbc:cassandra://%s:%s/%s?localdatacenter=%s"; final Connection connection = DriverManager.getConnection( - String.format(url, host, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); + String.format(url, host, port, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); try (Statement stmt = connection.createStatement()) { try { @@ -132,7 +133,7 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep } catch (Exception se) { // try again List tableNames = getTableNames( - new SQLConnection(DriverManager.getConnection(String.format(url, host, + new SQLConnection(DriverManager.getConnection(String.format(url, host, port, globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))), globalState.getDatabaseName()); for (String tableName : tableNames) { @@ -144,8 +145,8 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep stmt.execute("CREATE KEYSPACE IF NOT EXISTS " + globalState.getDatabaseName()); } - return new SQLConnection(DriverManager.getConnection(String.format(url, host, globalState.getDatabaseName(), - globalState.getDbmsSpecificOptions().datacenter))); + return new SQLConnection(DriverManager.getConnection(String.format(url, host, port, + globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))); } @Override diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java index 4845520d2..1534b9b56 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLSchema.java +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -7,7 +7,6 @@ import java.util.Collections; import java.util.List; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.DBMSCommon; @@ -195,12 +194,8 @@ private static YCQLCompositeDataType getColumnType(String typeString) { case "TIMESTAMP": primitiveType = YCQLDataType.TIMESTAMP; break; - case "INTERVAL": - throw new IgnoreMeException(); - // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP - // '1969-12-29 06:50:27')) default: - throw new IgnoreMeException(); + throw new AssertionError(); } return new YCQLCompositeDataType(primitiveType, size); } diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java index 278e5316e..f70b042d9 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLProvider.java +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -22,7 +22,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.yugabyte.ysql.YSQLOptions.YSQLOracleFactory; +import sqlancer.yugabyte.YugabyteBugs; import sqlancer.yugabyte.ysql.gen.YSQLAlterTableGenerator; import sqlancer.yugabyte.ysql.gen.YSQLAnalyzeGenerator; import sqlancer.yugabyte.ysql.gen.YSQLCommentGenerator; @@ -45,7 +45,8 @@ @AutoService(DatabaseProvider.class) public class YSQLProvider extends SQLProviderAdapter { - public static final Object CREATION_LOCK = new Object(); + // TODO Due to yugabyte problems with parallel DDL we need this lock object + public static final Object DDL_LOCK = new Object(); /** * Generate only data types and expressions that are understood by PQS. */ @@ -132,11 +133,6 @@ public void generateDatabase(YSQLGlobalState globalState) throws Exception { @Override public SQLConnection createDatabase(YSQLGlobalState globalState) throws SQLException { - if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() - .anyMatch((o) -> o == YSQLOracleFactory.PQS)) { - generateOnlyKnown = true; - } - username = globalState.getOptions().getUserName(); password = globalState.getOptions().getPassword(); host = globalState.getOptions().getHost(); @@ -197,7 +193,7 @@ public String getDBMSName() { // for some reason yugabyte unable to create few databases simultaneously private void createDatabaseSync(YSQLGlobalState globalState, String entryDatabaseName) throws SQLException { - synchronized (CREATION_LOCK) { + synchronized (DDL_LOCK) { exceptionLessSleep(5000); Connection con = createConnectionSafely(entryURL, username, password); @@ -246,7 +242,7 @@ protected void readFunctions(YSQLGlobalState globalState) throws SQLException { } protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { - synchronized (CREATION_LOCK) { + synchronized (DDL_LOCK) { boolean prevCreationFailed = false; // small optimization - wait only after failed requests while (globalState.getSchema().getDatabaseTables().size() < numTables) { if (!prevCreationFailed) { @@ -270,7 +266,7 @@ private void exceptionLessSleep(long timeout) { try { Thread.sleep(timeout); } catch (InterruptedException e) { - e.printStackTrace(); + throw new AssertionError(); } } @@ -296,10 +292,15 @@ private String getCreateDatabaseCommand(YSQLGlobalState state) { sb.append(Randomly.fromOptions("utf8")); sb.append("' "); } - // disabled due to https://github.com/yugabyte/yugabyte-db/issues/11357 - // if (Randomly.getBoolean()) { - // sb.append("COLOCATED = true "); - // } + + if (Randomly.getBoolean()) { + if (YugabyteBugs.bug11357) { + throw new IgnoreMeException(); + } + + sb.append("COLOCATED = true "); + } + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java index 035b2fbe4..d489b169e 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java @@ -99,7 +99,7 @@ public SQLQueryAdapter generate() { break; case ADD_TABLE_CONSTRAINT: sb.append("ADD "); - sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); + sb.append("CONSTRAINT ").append(r.getAlphabeticChar()).append(" "); YSQLCommon.addTableConstraint(sb, randomTable, globalState, errors); errors.add("already exists"); errors.add("multiple primary keys for table"); @@ -130,7 +130,7 @@ public SQLQueryAdapter generate() { break; case ADD_TABLE_CONSTRAINT_USING_INDEX: sb.append("ADD "); - sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); + sb.append("CONSTRAINT ").append(r.getAlphabeticChar()).append(" "); sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); sb.append(" USING INDEX "); sb.append(randomTable.getRandomIndex().getIndexName()); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java index e92af3e0f..89f3dd3a8 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java @@ -8,6 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.YugabyteBugs; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; @@ -213,27 +214,7 @@ public static boolean appendDataType(YSQLDataType type, StringBuilder sb, boolea public static void generateWith(StringBuilder sb, YSQLGlobalState globalState, ExpectedErrors errors, List columnsToBeAdded, boolean isTemporaryTable) { if (Randomly.getBoolean()) { - if (Randomly.getBoolean()) { - sb.append(" "); - // disabled https://github.com/yugabyte/yugabyte-db/issues/11357 - // sb.append(" WITH ("); - // ArrayList values = new ArrayList<>(Arrays.asList(StorageParameters.values())); - // errors.add("unrecognized parameter"); - // errors.add("ALTER TABLE / ADD CONSTRAINT USING INDEX is not supported on partitioned tables"); - // List subset = Randomly.nonEmptySubset(values); - // int i = 0; - // for (StorageParameters parameter : subset) { - // if (i++ != 0) { - // sb.append(", "); - // } - // sb.append(parameter.parameter); - // sb.append("="); - // sb.append(parameter.op.apply(globalState.getRandomly())); - // } - // sb.append(")"); - } else { - sb.append(" WITHOUT OIDS "); - } + sb.append(" WITHOUT OIDS "); } else if (Randomly.getBoolean() && !isTemporaryTable) { if (Randomly.getBoolean()) { sb.append(" SPLIT INTO "); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index faa7908b0..9f814025d 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -28,13 +29,6 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { sb.append(" UNIQUE"); } sb.append(" INDEX "); - /* - * Commented out as a workaround for https://www.postgresql.org/message-id/CA%2Bu7OA4XYhc- - * qyCgJqwwgMGZDWAyeH821oa5oMzm_HEifZ4BeA%40mail.gmail.com - */ - // if (Randomly.getBoolean()) { - // sb.append("CONCURRENTLY "); - // } YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized // views String indexName = getNewIndexName(randomTable); @@ -71,11 +65,6 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { sb.append(")"); } - // if (Randomly.getBoolean()) { - // sb.append(" "); - // sb.append("COLLATE "); - // sb.append(Randomly.fromOptions("C", "POSIX")); - // } if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" "); sb.append(globalState.getRandomOpclass()); @@ -97,7 +86,7 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { if (Randomly.getBoolean() && method != IndexType.HASH) { sb.append(" INCLUDE("); List columns = randomTable.getRandomNonEmptyColumnSubset(); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); sb.append(")"); } if (Randomly.getBoolean()) { diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java index d10bccee6..449654d99 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java @@ -19,21 +19,9 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { if (Randomly.getBoolean()) { sb.append(" TABLE"); } - // TODO partitions - // if (Randomly.getBoolean()) { - // sb.append(" ONLY"); - // } sb.append(" "); sb.append(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty().stream().map(AbstractTable::getName) .collect(Collectors.joining(", "))); - // if (Randomly.getBoolean()) { - // sb.append(" "); - // sb.append(Randomly.fromOptions("RESTART IDENTITY", "CONTINUE IDENTITY")); - // } - // if (Randomly.getBoolean()) { - // sb.append(" "); - // sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); - // } return new SQLQueryAdapter(sb.toString(), ExpectedErrors .from("cannot truncate a table referenced in a foreign key constraint", "is not a table")); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java index f873f7812..9efe027b1 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java @@ -10,10 +10,7 @@ private YSQLVacuumGenerator() { } public static SQLQueryAdapter create(YSQLGlobalState globalState) { - String sb = "VACUUM"; - ExpectedErrors errors = new ExpectedErrors(); - errors.add("VACUUM cannot run inside a transaction block"); - return new SQLQueryAdapter(sb, errors); + return new SQLQueryAdapter("VACUUM", ExpectedErrors.from("VACUUM cannot run inside a transaction block")); } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java index a5b57fbd8..a34be987b 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -16,11 +16,8 @@ private YSQLViewGenerator() { public static SQLQueryAdapter create(YSQLGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE"); - // boolean materialized; - // boolean recursive = false; if (Randomly.getBoolean()) { sb.append(" MATERIALIZED"); - // materialized = true; } else { if (Randomly.getBoolean()) { sb.append(" OR REPLACE"); @@ -28,11 +25,6 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { if (Randomly.getBoolean()) { sb.append(Randomly.fromOptions(" TEMP", " TEMPORARY")); } - // if (Randomly.getBoolean()) { - // sb.append(" RECURSIVE"); - // recursive = true; - // } - // materialized = false; } sb.append(" VIEW "); int i = 0; @@ -54,12 +46,6 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { sb.append(DBMSCommon.createColumnName(i)); } sb.append(")"); - // if (Randomly.getBoolean() && !materialized && !recursive) { - // sb.append(" WITH "); - // sb.append(Randomly.fromOptions("CASCADED", "LOCAL")); - // sb.append(" CHECK OPTION"); - // errors.add("WITH CHECK OPTION is supported only on automatically updatable views"); - // } sb.append(" AS ("); YSQLSelect select = YSQLRandomQueryGenerator.createRandomQuery(nrColumns, globalState); sb.append(YSQLVisitor.asString(select)); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java index a873ce505..1eb6b9ff4 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java @@ -1,6 +1,6 @@ package sqlancer.yugabyte.ysql.oracle; -import static sqlancer.yugabyte.ysql.YSQLProvider.CREATION_LOCK; +import static sqlancer.yugabyte.ysql.YSQLProvider.DDL_LOCK; import java.util.Arrays; import java.util.List; @@ -49,12 +49,14 @@ private YSQLProvider.Action getRandomAction(List actions) { } protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { - synchronized (CREATION_LOCK) { + synchronized (DDL_LOCK) { while (globalState.getSchema().getDatabaseTables().size() < numTables) { + // TODO concurrent DDLs may produce a lot of noise in test logs so its disabled right now + // added timeout to avoid possible catalog collisions try { Thread.sleep(1000); } catch (InterruptedException e) { - e.printStackTrace(); + throw new AssertionError(); } try { diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java index e8f791b3f..a5af7111f 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java @@ -2,11 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Locale; -import java.util.Map; - -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; import sqlancer.Randomly; import sqlancer.common.oracle.TestOracle; @@ -40,33 +35,12 @@ public YSQLFuzzer(YSQLGlobalState globalState) { errors.add("Connection refused"); errors.add("Connection to"); - // get config from -Dconfig.file="path/to/fuzzer.conf" testQueries = new ArrayList<>(); - try { - Config config = ConfigFactory.load(); - ArrayList queriesList = (ArrayList) config.getList("queries").unwrapped(); - for (Object configValue : queriesList) { - String type = ((String) ((Map) configValue).get("type")).toUpperCase(Locale.ROOT); - Integer weight = (Integer) ((Map) configValue).get("weight"); - - Query query = type.equalsIgnoreCase("SELECT") ? new SelectQuery() - : new ActionQuery(YSQLProvider.Action.valueOf(type)); - - for (int i = 0; i < weight; i++) { - testQueries.add(query); - } - } - } catch (Exception e) { - // do nothing - } finally { - if (testQueries.isEmpty()) { - System.out.println("No configuration found. Using just random select statements"); - testQueries.add(new SelectQuery()); - testQueries.add(new ActionQuery(YSQLProvider.Action.UPDATE)); - testQueries.add(new ActionQuery(YSQLProvider.Action.DELETE)); - testQueries.add(new ActionQuery(YSQLProvider.Action.INSERT)); - } - } + + testQueries.add(new SelectQuery()); + testQueries.add(new ActionQuery(YSQLProvider.Action.UPDATE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.DELETE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.INSERT)); } @Override From 4c3304135c1846d42b13f08f4d900638a0a78184 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 14:49:45 +0400 Subject: [PATCH 0621/1284] Fixed README, CI Added Test and YSQLErrors --- .github/workflows/main.yml | 21 +++ README.md | 27 ++-- src/sqlancer/yugabyte/YugabyteBugs.java | 9 +- src/sqlancer/yugabyte/ycql/YCQLOptions.java | 9 +- src/sqlancer/yugabyte/ycql/YCQLProvider.java | 16 +- src/sqlancer/yugabyte/ysql/YSQLErrors.java | 151 ++++++++++++++++++ src/sqlancer/yugabyte/ysql/YSQLProvider.java | 14 +- .../ysql/gen/YSQLAlterTableGenerator.java | 7 +- .../yugabyte/ysql/gen/YSQLCommon.java | 145 +---------------- .../ysql/gen/YSQLDeleteGenerator.java | 3 +- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 3 +- .../ysql/gen/YSQLInsertGenerator.java | 7 +- .../yugabyte/ysql/gen/YSQLTableGenerator.java | 7 +- .../ysql/gen/YSQLUpdateGenerator.java | 5 +- .../yugabyte/ysql/gen/YSQLViewGenerator.java | 7 +- .../yugabyte/ysql/oracle/YSQLCatalog.java | 6 +- .../yugabyte/ysql/oracle/YSQLFuzzer.java | 10 +- .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 6 +- .../YSQLPivotedQuerySynthesisOracle.java | 6 +- .../oracle/tlp/YSQLTLPAggregateOracle.java | 4 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 6 +- .../ysql/oracle/tlp/YSQLTLPHavingOracle.java | 4 +- test/sqlancer/dbms/TestYugabyte.java | 29 ++++ 23 files changed, 298 insertions(+), 204 deletions(-) create mode 100644 src/sqlancer/yugabyte/ysql/YSQLErrors.java create mode 100644 test/sqlancer/dbms/TestYugabyte.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bfbbe60cc..7bf958bc1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -297,6 +297,27 @@ jobs: - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test + yugabyte: + name: DBMS Tests (Yugabyte) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Setup Yugabyte + run: | + docker pull yugabytedb/yugabyte:latest + docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false + sleep 5 + - name: Run Tests + run: YUGABYTE_AVAILABLE=true mvn -Dtest=TestYugabyte test + java13: name: Java 13 Compatibility (DuckDB) runs-on: ubuntu-latest diff --git a/README.md b/README.md index 2209e7589..d163f049c 100644 --- a/README.md +++ b/README.md @@ -47,19 +47,20 @@ Please find the `.bib` entries [here](docs/PAPERS.md). Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. -| DBMS | Status | Expression Generation | Description | -|-------------|-------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | -| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | -| PostgreSQL | Working | Typed | | -| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | -| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | -| CockroachDB | Working | Typed | | -| TiDB | Working | Untyped | | -| DuckDB | Working | Untyped, Generic | | -| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | -| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | -| OceanBase | Working | Untyped | | +| DBMS | Status | Expression Generation | Description | +|------------------------------|-------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | +| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | +| PostgreSQL | Working | Typed | | +| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | +| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | +| CockroachDB | Working | Typed | | +| TiDB | Working | Untyped | | +| DuckDB | Working | Untyped, Generic | | +| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | +| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | +| OceanBase | Working | Untyped | | +| YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | # Using SQLancer diff --git a/src/sqlancer/yugabyte/YugabyteBugs.java b/src/sqlancer/yugabyte/YugabyteBugs.java index 1e944e0e2..424ba3217 100644 --- a/src/sqlancer/yugabyte/YugabyteBugs.java +++ b/src/sqlancer/yugabyte/YugabyteBugs.java @@ -1,6 +1,11 @@ package sqlancer.yugabyte; -public interface YugabyteBugs { +public final class YugabyteBugs { + // https://github.com/yugabyte/yugabyte-db/issues/11357 - boolean bug11357 = true; + public static boolean bug11357 = true; + + private YugabyteBugs() { + } + } diff --git a/src/sqlancer/yugabyte/ycql/YCQLOptions.java b/src/sqlancer/yugabyte/ycql/YCQLOptions.java index 5955d28dd..7e35e2cea 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLOptions.java +++ b/src/sqlancer/yugabyte/ycql/YCQLOptions.java @@ -14,9 +14,14 @@ import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.test.YCQLFuzzer; -@Parameters(commandDescription = "YCQL") +@Parameters(separators = "=", commandDescription = "YCQL (default port: " + YCQLOptions.DEFAULT_PORT + + ", default host: " + YCQLOptions.DEFAULT_HOST) public class YCQLOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 9042; + public static final String DEFAULT_DATACENTER = "datacenter1"; + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) public int maxNumDeletes = 1; @@ -24,7 +29,7 @@ public class YCQLOptions implements DBMSSpecificOptions { public int maxNumUpdates = 5; @Parameter(names = "--datacenter", description = "YCQL datacenter, can be found in system.local table", arity = 1) - public String datacenter = "datacenter1"; + public String datacenter = DEFAULT_DATACENTER; @Parameter(names = "--oracle") public List oracles = Arrays.asList(YCQLOracleFactory.FUZZER); diff --git a/src/sqlancer/yugabyte/ycql/YCQLProvider.java b/src/sqlancer/yugabyte/ycql/YCQLProvider.java index 8bc8cbf84..59624f588 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLProvider.java +++ b/src/sqlancer/yugabyte/ycql/YCQLProvider.java @@ -13,6 +13,7 @@ import sqlancer.AbstractAction; import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; @@ -119,10 +120,19 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep try { Class.forName("com.ing.data.cassandra.jdbc.CassandraDriver"); } catch (ClassNotFoundException e) { - e.printStackTrace(); + throw new AssertionError(); } - final String host = globalState.getOptions().getHost(); - final int port = globalState.getOptions().getPort(); + + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + + if (host == null) { + host = YCQLOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = YCQLOptions.DEFAULT_PORT; + } + final String url = "jdbc:cassandra://%s:%s/%s?localdatacenter=%s"; final Connection connection = DriverManager.getConnection( String.format(url, host, port, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); diff --git a/src/sqlancer/yugabyte/ysql/YSQLErrors.java b/src/sqlancer/yugabyte/ysql/YSQLErrors.java new file mode 100644 index 000000000..a2cee1add --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLErrors.java @@ -0,0 +1,151 @@ +package sqlancer.yugabyte.ysql; + +import sqlancer.common.query.ExpectedErrors; + +public final class YSQLErrors { + + private YSQLErrors() { + } + + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.add("An I/O error occurred while sending to the backend"); + errors.add("Conflicts with committed transaction"); + errors.add("cannot be changed"); + errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); + + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); + errors.add("but it cannot be referenced from this part of the query"); + errors.add("missing FROM-clause entry for table"); + + errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); + } + + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.add("PRIMARY KEY containing column of type 'INET' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'VARBIT' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'INT4RANGE' not yet supported"); + errors.add("INDEX on column of type 'INET' not yet supported"); + errors.add("INDEX on column of type 'VARBIT' not yet supported"); + errors.add("INDEX on column of type 'INT4RANGE' not yet supported"); + errors.add("is not commutative"); // exclude + errors.add("cannot be changed"); + errors.add("operator requires run-time type coercion"); // exclude + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.add("syntax error at or near \"(\""); + errors.add("does not exist"); + errors.add("is not unique"); + errors.add("cannot be changed"); + errors.add("invalid reference to FROM-clause entry for table"); + + errors.add("Invalid column number"); + errors.add("specified more than once"); + errors.add("You might need to add explicit type casts"); + errors.add("invalid regular expression"); + errors.add("could not determine which collation to use"); + errors.add("invalid input syntax for integer"); + errors.add("invalid regular expression"); + errors.add("operator does not exist"); + errors.add("quantifier operand invalid"); + errors.add("collation mismatch"); + errors.add("collations are not supported"); + errors.add("operator is not unique"); + errors.add("is not a valid binary digit"); + errors.add("invalid hexadecimal digit"); + errors.add("invalid hexadecimal data: odd number of digits"); + errors.add("zero raised to a negative power is undefined"); + errors.add("cannot convert infinity to numeric"); + errors.add("division by zero"); + errors.add("invalid input syntax for type money"); + errors.add("invalid input syntax for type"); + errors.add("cannot cast type"); + errors.add("value overflows numeric format"); + errors.add("is of type boolean but expression is of type text"); + errors.add("a negative number raised to a non-integer power yields a complex result"); + errors.add("could not determine polymorphic type because input has type unknown"); + + addToCharFunctionErrors(errors); + addBitStringOperationErrors(errors); + addFunctionErrors(errors); + addCommonRangeExpressionErrors(errors); + addCommonRegexExpressionErrors(errors); + } + + public static void addToCharFunctionErrors(ExpectedErrors errors) { + errors.add("multiple decimal points"); + errors.add("and decimal point together"); + errors.add("multiple decimal points"); + errors.add("cannot use \"S\" twice"); + errors.add("must be ahead of \"PR\""); + errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); + errors.add("cannot use \"S\" and \"SG\" together"); + errors.add("cannot use \"S\" and \"MI\" together"); + errors.add("cannot use \"S\" and \"PL\" together"); + errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); + errors.add("is not a number"); + } + + public static void addBitStringOperationErrors(ExpectedErrors errors) { + errors.add("cannot XOR bit strings of different sizes"); + errors.add("cannot AND bit strings of different sizes"); + errors.add("cannot OR bit strings of different sizes"); + errors.add("must be type boolean, not type text"); + } + + public static void addFunctionErrors(ExpectedErrors errors) { + errors.add("out of valid range"); // get_bit/get_byte + errors.add("cannot take logarithm of a negative number"); + errors.add("cannot take logarithm of zero"); + errors.add("requested character too large for encoding"); // chr + errors.add("null character not permitted"); // chr + errors.add("requested character not valid for encoding"); // chr + errors.add("requested length too large"); // repeat + errors.add("invalid memory alloc request size"); // repeat + errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + } + + public static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + errors.add("is not a valid hexadecimal digit"); + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.add("range lower bound must be less than or equal to range upper bound"); + errors.add("result of range difference would not be contiguous"); + errors.add("out of range"); + errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.add("value too long for type character"); + errors.add("not found in view targetlist"); + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in GROUP BY"); + } + + public static void addViewErrors(ExpectedErrors errors) { + errors.add("already exists"); + errors.add("cannot drop columns from view"); + errors.add("non-integer constant in ORDER BY"); // TODO + errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO + errors.add("cannot change data type of view column"); + errors.add("specified more than once"); // TODO + errors.add("materialized views must not use temporary tables or views"); + errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); + errors.add("is not a view"); + errors.add("non-integer constant in DISTINCT ON"); + errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java index f70b042d9..e72790344 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLProvider.java +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -22,7 +22,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.yugabyte.YugabyteBugs; import sqlancer.yugabyte.ysql.gen.YSQLAlterTableGenerator; import sqlancer.yugabyte.ysql.gen.YSQLAnalyzeGenerator; import sqlancer.yugabyte.ysql.gen.YSQLCommentGenerator; @@ -142,6 +141,13 @@ public SQLConnection createDatabase(YSQLGlobalState globalState) throws SQLExcep String entryDatabaseName = entryPath.substring(1); databaseName = globalState.getDatabaseName(); + if (host == null) { + host = YSQLOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = YSQLOptions.DEFAULT_PORT; + } + try { URI uri = new URI(entryURL); String userInfoURI = uri.getUserInfo(); @@ -294,9 +300,9 @@ private String getCreateDatabaseCommand(YSQLGlobalState state) { } if (Randomly.getBoolean()) { - if (YugabyteBugs.bug11357) { - throw new IgnoreMeException(); - } + // if (YugabyteBugs.bug11357) { + // throw new IgnoreMeException(); + // } sb.append("COLOCATED = true "); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java index d489b169e..c4a4effd8 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java @@ -6,6 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; @@ -26,9 +27,9 @@ public static SQLQueryAdapter create(YSQLTable randomTable, YSQLGlobalState glob } public List getActions(ExpectedErrors errors) { - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonInsertUpdateErrors(errors); - YSQLCommon.addCommonTableErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonInsertUpdateErrors(errors); + YSQLErrors.addCommonTableErrors(errors); errors.add("duplicate key value violates unique constraint"); errors.add("cannot drop key column"); errors.add("cannot drop desired object(s) because other objects depend on them"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java index 89f3dd3a8..09b559e4c 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.yugabyte.YugabyteBugs; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; @@ -22,126 +22,6 @@ public final class YSQLCommon { private YSQLCommon() { } - public static void addCommonFetchErrors(ExpectedErrors errors) { - errors.add("An I/O error occurred while sending to the backend"); - errors.add("Conflicts with committed transaction"); - errors.add("cannot be changed"); - errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); - - errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); - errors.add("but it cannot be referenced from this part of the query"); - errors.add("missing FROM-clause entry for table"); - - errors.add("canceling statement due to statement timeout"); - - errors.add("non-integer constant in"); - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("GROUP BY position"); - } - - public static void addCommonTableErrors(ExpectedErrors errors) { - errors.add("PRIMARY KEY containing column of type 'INET' not yet supported"); - errors.add("PRIMARY KEY containing column of type 'VARBIT' not yet supported"); - errors.add("PRIMARY KEY containing column of type 'INT4RANGE' not yet supported"); - errors.add("INDEX on column of type 'INET' not yet supported"); - errors.add("INDEX on column of type 'VARBIT' not yet supported"); - errors.add("INDEX on column of type 'INT4RANGE' not yet supported"); - errors.add("is not commutative"); // exclude - errors.add("cannot be changed"); - errors.add("operator requires run-time type coercion"); // exclude - } - - public static void addCommonExpressionErrors(ExpectedErrors errors) { - errors.add("syntax error at or near \"(\""); - errors.add("does not exist"); - errors.add("is not unique"); - errors.add("cannot be changed"); - errors.add("invalid reference to FROM-clause entry for table"); - - errors.add("Invalid column number"); - errors.add("specified more than once"); - errors.add("You might need to add explicit type casts"); - errors.add("invalid regular expression"); - errors.add("could not determine which collation to use"); - errors.add("invalid input syntax for integer"); - errors.add("invalid regular expression"); - errors.add("operator does not exist"); - errors.add("quantifier operand invalid"); - errors.add("collation mismatch"); - errors.add("collations are not supported"); - errors.add("operator is not unique"); - errors.add("is not a valid binary digit"); - errors.add("invalid hexadecimal digit"); - errors.add("invalid hexadecimal data: odd number of digits"); - errors.add("zero raised to a negative power is undefined"); - errors.add("cannot convert infinity to numeric"); - errors.add("division by zero"); - errors.add("invalid input syntax for type money"); - errors.add("invalid input syntax for type"); - errors.add("cannot cast type"); - errors.add("value overflows numeric format"); - errors.add("is of type boolean but expression is of type text"); - errors.add("a negative number raised to a non-integer power yields a complex result"); - errors.add("could not determine polymorphic type because input has type unknown"); - addToCharFunctionErrors(errors); - addBitStringOperationErrors(errors); - addFunctionErrors(errors); - addCommonRangeExpressionErrors(errors); - addCommonRegexExpressionErrors(errors); - } - - private static void addToCharFunctionErrors(ExpectedErrors errors) { - errors.add("multiple decimal points"); - errors.add("and decimal point together"); - errors.add("multiple decimal points"); - errors.add("cannot use \"S\" twice"); - errors.add("must be ahead of \"PR\""); - errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); - errors.add("cannot use \"S\" and \"SG\" together"); - errors.add("cannot use \"S\" and \"MI\" together"); - errors.add("cannot use \"S\" and \"PL\" together"); - errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); - errors.add("is not a number"); - } - - private static void addBitStringOperationErrors(ExpectedErrors errors) { - errors.add("cannot XOR bit strings of different sizes"); - errors.add("cannot AND bit strings of different sizes"); - errors.add("cannot OR bit strings of different sizes"); - errors.add("must be type boolean, not type text"); - } - - private static void addFunctionErrors(ExpectedErrors errors) { - errors.add("out of valid range"); // get_bit/get_byte - errors.add("cannot take logarithm of a negative number"); - errors.add("cannot take logarithm of zero"); - errors.add("requested character too large for encoding"); // chr - errors.add("null character not permitted"); // chr - errors.add("requested character not valid for encoding"); // chr - errors.add("requested length too large"); // repeat - errors.add("invalid memory alloc request size"); // repeat - errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii - errors.add("negative substring length not allowed"); // substr - errors.add("invalid mask length"); // set_masklen - } - - private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { - errors.add("is not a valid hexadecimal digit"); - } - - public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { - errors.add("range lower bound must be less than or equal to range upper bound"); - errors.add("result of range difference would not be contiguous"); - errors.add("out of range"); - errors.add("malformed range literal"); - errors.add("result of range union would not be contiguous"); - } - - public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { - errors.add("value too long for type character"); - errors.add("not found in view targetlist"); - } - public static boolean appendDataType(YSQLDataType type, StringBuilder sb, boolean allowSerial, boolean generateOnlyKnown, List opClasses) throws AssertionError { boolean serial = false; @@ -310,7 +190,7 @@ private static void addTableConstraint(StringBuilder sb, YSQLTable table, YSQLGl TableConstraints t, ExpectedErrors errors) { List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); List otherColumns; - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); switch (t) { case CHECK: sb.append("CHECK("); @@ -381,27 +261,6 @@ private static void deleteOrUpdateAction(StringBuilder sb) { sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); } - public static void addGroupingErrors(ExpectedErrors errors) { - errors.add("non-integer constant in GROUP BY"); // TODO - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("is not in select list"); - errors.add("aggregate functions are not allowed in GROUP BY"); - } - - public static void addViewErrors(ExpectedErrors errors) { - errors.add("already exists"); - errors.add("cannot drop columns from view"); - errors.add("non-integer constant in ORDER BY"); // TODO - errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO - errors.add("cannot change data type of view column"); - errors.add("specified more than once"); // TODO - errors.add("materialized views must not use temporary tables or views"); - errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); - errors.add("is not a view"); - errors.add("non-integer constant in DISTINCT ON"); - errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); - } - public enum TableConstraints { CHECK, UNIQUE, PRIMARY_KEY, FOREIGN_KEY } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java index e2eeb1ceb..e0128707b 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -3,6 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; @@ -35,7 +36,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { sb.append( YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, table.getColumns()))); } - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); errors.add("out of range"); errors.add("cannot cast"); errors.add("invalid input syntax for"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index 9f814025d..6077dcb1e 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -8,6 +8,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -120,7 +121,7 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { errors.add("functions in index expression must be marked IMMUTABLE"); errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java index 1817891b3..c26459f31 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; @@ -22,9 +23,9 @@ public static SQLQueryAdapter insert(YSQLGlobalState globalState) { YSQLTable table = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); ExpectedErrors errors = new ExpectedErrors(); errors.add("cannot insert into column"); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonInsertUpdateErrors(errors); - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonInsertUpdateErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); errors.add("multiple assignments to same column"); errors.add("violates foreign key constraint"); errors.add("value too long for type character varying"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java index 5e376ab4c..37945c7ac 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -57,8 +58,8 @@ public YSQLTableGenerator(String tableName, boolean generateOnlyKnown, YSQLGloba errors.add("Cannot split table that does not have primary key"); errors.add("inherits from generated column but specifies identity"); errors.add("inherits from generated column but specifies default"); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonTableErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonTableErrors(errors); } public static SQLQueryAdapter generate(String tableName, boolean generateOnlyKnown, YSQLGlobalState globalState) { @@ -148,7 +149,7 @@ private void generatePartitionBy() { errors.add("unsupported UNIQUE constraint with partition key definition"); errors.add("does not accept data type"); int n = partitionOption.contentEquals("LIST") ? 1 : Randomly.smallNumber() + 1; - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); for (int i = 0; i < n; i++) { if (i != 0) { sb.append(", "); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index fe8e71ada..06e762440 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -5,6 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -33,7 +34,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { // table multiple times errors.add("new row violates check option for view"); List columns = randomTable.getRandomNonEmptyColumnSubset(); - YSQLCommon.addCommonInsertUpdateErrors(errors); + YSQLErrors.addCommonInsertUpdateErrors(errors); for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -62,7 +63,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { errors.add("violates check constraint"); errors.add("could not determine which collation to use for string comparison"); errors.add("but expression is of type"); - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { sb.append(" WHERE "); YSQLExpression where = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java index a34be987b..f70891cc2 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -4,6 +4,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLVisitor; import sqlancer.yugabyte.ysql.ast.YSQLSelect; @@ -50,9 +51,9 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { YSQLSelect select = YSQLRandomQueryGenerator.createRandomQuery(nrColumns, globalState); sb.append(YSQLVisitor.asString(select)); sb.append(")"); - YSQLCommon.addGroupingErrors(errors); - YSQLCommon.addViewErrors(errors); - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addGroupingErrors(errors); + YSQLErrors.addViewErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, true); } diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java index 1eb6b9ff4..d6763e799 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java @@ -13,9 +13,9 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; public class YSQLCatalog implements TestOracle { @@ -40,8 +40,8 @@ public YSQLCatalog(YSQLGlobalState globalState) { this.con = state.getConnection(); this.logger = state.getLogger(); this.options = state.getOptions(); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonFetchErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); } private YSQLProvider.Action getRandomAction(List actions) { diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java index a5af7111f..0aa16993e 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java @@ -7,10 +7,10 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.YSQLVisitor; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLRandomQueryGenerator; public class YSQLFuzzer implements TestOracle { @@ -21,10 +21,10 @@ public class YSQLFuzzer implements TestOracle { public YSQLFuzzer(YSQLGlobalState globalState) { this.globalState = globalState; - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonFetchErrors(errors); - YSQLCommon.addGroupingErrors(errors); - YSQLCommon.addViewErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); + YSQLErrors.addGroupingErrors(errors); + YSQLErrors.addViewErrors(errors); // remove timeout error from scope errors.add("canceling statement due to statement timeout"); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java index 863b63398..6b2a57560 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -16,6 +16,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; @@ -29,7 +30,6 @@ import sqlancer.yugabyte.ysql.ast.YSQLJoin; import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPBase; @@ -40,8 +40,8 @@ public class YSQLNoRECOracle extends NoRECBase implements TestO public YSQLNoRECOracle(YSQLGlobalState globalState) { super(globalState); this.s = globalState.getSchema(); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonFetchErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); } public static List getJoinStatements(YSQLGlobalState globalState, List columns, diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java index 95ac268f4..6099f5b87 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java @@ -10,6 +10,7 @@ import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -21,7 +22,6 @@ import sqlancer.yugabyte.ysql.ast.YSQLExpression; import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; public class YSQLPivotedQuerySynthesisOracle @@ -31,8 +31,8 @@ public class YSQLPivotedQuerySynthesisOracle public YSQLPivotedQuerySynthesisOracle(YSQLGlobalState globalState) throws SQLException { super(globalState); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonFetchErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); } /* diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java index aa6dfd530..26e4d9179 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -14,6 +14,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLVisitor; @@ -27,7 +28,6 @@ import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation.PrefixOperator; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { @@ -38,7 +38,7 @@ public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { public YSQLTLPAggregateOracle(YSQLGlobalState state) { super(state); - YSQLCommon.addGroupingErrors(errors); + YSQLErrors.addGroupingErrors(errors); } @Override diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java index 359f12d92..e574391de 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -10,6 +10,7 @@ import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; @@ -21,7 +22,6 @@ import sqlancer.yugabyte.ysql.ast.YSQLExpression; import sqlancer.yugabyte.ysql.ast.YSQLJoin; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; @@ -35,8 +35,8 @@ public class YSQLTLPBase extends TernaryLogicPartitioningOracleBase Date: Wed, 5 Oct 2022 16:19:27 +0400 Subject: [PATCH 0622/1284] Tuned YCQL test --- test/sqlancer/dbms/TestYugabyte.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/sqlancer/dbms/TestYugabyte.java b/test/sqlancer/dbms/TestYugabyte.java index 083eb2c91..168982c22 100644 --- a/test/sqlancer/dbms/TestYugabyte.java +++ b/test/sqlancer/dbms/TestYugabyte.java @@ -14,7 +14,8 @@ public class TestYugabyte { @Test public void testYCQL() { assumeTrue(yugabyteIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", + assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "cassandra", "--password", "cassandra", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "ycql")); } From f58ee48dfbf5bf2e43a8b82ec16fbab35edfa29c Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 16:20:37 +0400 Subject: [PATCH 0623/1284] Reformat YCQL test --- test/sqlancer/dbms/TestYugabyte.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/sqlancer/dbms/TestYugabyte.java b/test/sqlancer/dbms/TestYugabyte.java index 168982c22..57798924d 100644 --- a/test/sqlancer/dbms/TestYugabyte.java +++ b/test/sqlancer/dbms/TestYugabyte.java @@ -14,9 +14,10 @@ public class TestYugabyte { @Test public void testYCQL() { assumeTrue(yugabyteIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "cassandra", "--password", "cassandra", "--num-threads", - "4", "--num-queries", TestConfig.NUM_QUERIES, "ycql")); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "cassandra", "--password", "cassandra", "--num-threads", "4", "--num-queries", + TestConfig.NUM_QUERIES, "ycql")); } @Test From cae924702c141eca43fe5efb8a2afcebb51afef6 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 16:48:43 +0400 Subject: [PATCH 0624/1284] Fixed YugabyteDB naming in CI --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7bf958bc1..68ef0b2bf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -298,7 +298,7 @@ jobs: run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test yugabyte: - name: DBMS Tests (Yugabyte) + name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From cfa76ecb51f94509cf2d75532197b98488bf6ead Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 17:10:46 +0400 Subject: [PATCH 0625/1284] Reduced number of threads in test to avoid sporadic failures Added some allowed errors and DDL locks for YCQL --- src/sqlancer/yugabyte/ycql/YCQLProvider.java | 31 ++++++++++--------- .../ysql/gen/YSQLTableGroupGenerator.java | 1 + test/sqlancer/dbms/TestYugabyte.java | 4 +-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/sqlancer/yugabyte/ycql/YCQLProvider.java b/src/sqlancer/yugabyte/ycql/YCQLProvider.java index 59624f588..b82f5b0a2 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLProvider.java +++ b/src/sqlancer/yugabyte/ycql/YCQLProvider.java @@ -1,6 +1,7 @@ package sqlancer.yugabyte.ycql; import static sqlancer.yugabyte.ycql.YCQLSchema.getTableNames; +import static sqlancer.yugabyte.ysql.YSQLProvider.DDL_LOCK; import java.sql.Connection; import java.sql.DriverManager; @@ -137,22 +138,24 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep final Connection connection = DriverManager.getConnection( String.format(url, host, port, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); - try (Statement stmt = connection.createStatement()) { - try { - stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); - } catch (Exception se) { - // try again - List tableNames = getTableNames( - new SQLConnection(DriverManager.getConnection(String.format(url, host, port, - globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))), - globalState.getDatabaseName()); - for (String tableName : tableNames) { - stmt.execute("DROP TABLE " + globalState.getDatabaseName() + "." + tableName); + synchronized (DDL_LOCK) { + try (Statement stmt = connection.createStatement()) { + try { + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); + } catch (Exception se) { + // try again + List tableNames = getTableNames( + new SQLConnection(DriverManager.getConnection(String.format(url, host, port, + globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))), + globalState.getDatabaseName()); + for (String tableName : tableNames) { + stmt.execute("DROP TABLE " + globalState.getDatabaseName() + "." + tableName); + } + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); } - stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); - } - stmt.execute("CREATE KEYSPACE IF NOT EXISTS " + globalState.getDatabaseName()); + stmt.execute("CREATE KEYSPACE IF NOT EXISTS " + globalState.getDatabaseName()); + } } return new SQLConnection(DriverManager.getConnection(String.format(url, host, port, diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java index da1b5c573..8bc292348 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java @@ -19,6 +19,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { StringBuilder sb = new StringBuilder("CREATE TABLEGROUP "); String tableGroupName = "tg" + UNIQUE_TABLEGROUP_COUNTER.incrementAndGet(); sb.append(tableGroupName); + errors.add("cannot use tablegroups in a colocated database"); return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/test/sqlancer/dbms/TestYugabyte.java b/test/sqlancer/dbms/TestYugabyte.java index 57798924d..10387948b 100644 --- a/test/sqlancer/dbms/TestYugabyte.java +++ b/test/sqlancer/dbms/TestYugabyte.java @@ -16,7 +16,7 @@ public void testYCQL() { assumeTrue(yugabyteIsAvailable); assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "cassandra", "--password", "cassandra", "--num-threads", "4", "--num-queries", + "cassandra", "--password", "cassandra", "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "ycql")); } @@ -25,7 +25,7 @@ public void testYSQL() { assumeTrue(yugabyteIsAvailable); assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "yugabyte", "--password", "yugabyte", "--num-threads", "4", "--num-queries", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "ysql")); } } From 7d46ee0c7a2173825ad3ca6ac60ee84069e26754 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 18:07:31 +0400 Subject: [PATCH 0626/1284] Added bug14330 --- src/sqlancer/yugabyte/YugabyteBugs.java | 3 +++ .../yugabyte/ycql/gen/YCQLExpressionGenerator.java | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/sqlancer/yugabyte/YugabyteBugs.java b/src/sqlancer/yugabyte/YugabyteBugs.java index 424ba3217..713ae6f88 100644 --- a/src/sqlancer/yugabyte/YugabyteBugs.java +++ b/src/sqlancer/yugabyte/YugabyteBugs.java @@ -5,6 +5,9 @@ public final class YugabyteBugs { // https://github.com/yugabyte/yugabyte-db/issues/11357 public static boolean bug11357 = true; + // https://github.com/yugabyte/yugabyte-db/issues/14330 + public static boolean bug14330 = true; + private YugabyteBugs() { } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java index 296be911d..77a4076cf 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.ColumnReferenceNode; @@ -23,6 +24,8 @@ import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import static sqlancer.yugabyte.YugabyteBugs.bug14330; + public final class YCQLExpressionGenerator extends UntypedExpressionGenerator, YCQLColumn> { private final YCQLGlobalState globalState; @@ -82,6 +85,10 @@ protected Node generateColumn() { @Override public Node generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { + if (bug14330) { + throw new IgnoreMeException(); + } + return YCQLConstant.createNullConstant(); } YCQLDataType type = YCQLDataType.getRandom(); From c3baf28af1f7689533085029481a5d4a5da5ad5e Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 21:40:03 +0400 Subject: [PATCH 0627/1284] Fixed code style for bug14330 --- src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java index 77a4076cf..7a0a7ed1b 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java @@ -1,5 +1,7 @@ package sqlancer.yugabyte.ycql.gen; +import static sqlancer.yugabyte.YugabyteBugs.bug14330; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -24,8 +26,6 @@ import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; -import static sqlancer.yugabyte.YugabyteBugs.bug14330; - public final class YCQLExpressionGenerator extends UntypedExpressionGenerator, YCQLColumn> { private final YCQLGlobalState globalState; From ffd1ac9a22671975ca737146d1da5dc9a5e4d4f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 08:20:41 +0000 Subject: [PATCH 0628/1284] Bump spotbugs-maven-plugin from 4.7.1.1 to 4.7.2.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.1.1 to 4.7.2.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.1.1...spotbugs-maven-plugin-4.7.2.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fdb287a10..1b550a2bc 100644 --- a/pom.xml +++ b/pom.xml @@ -252,7 +252,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.1.1 + 4.7.2.0 spotbugs From dd27b0b69887ad9eb681b062eb21959975b0d07c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 08:46:24 +0000 Subject: [PATCH 0629/1284] Bump junit-jupiter-engine from 5.9.0 to 5.9.1 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.9.0 to 5.9.1. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.9.0...r5.9.1) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1b550a2bc..1b76c1dd7 100644 --- a/pom.xml +++ b/pom.xml @@ -319,7 +319,7 @@ org.junit.jupiter junit-jupiter-engine - 5.9.0 + 5.9.1 test From fd39eea8b00f46c4d930f3814ffb0204496b8792 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 09:42:26 +0000 Subject: [PATCH 0630/1284] Bump mariadb-java-client from 3.0.7 to 3.0.8 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.7 to 3.0.8. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.7...3.0.8) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1b76c1dd7..529f86a51 100644 --- a/pom.xml +++ b/pom.xml @@ -309,7 +309,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.7 + 3.0.8 org.duckdb From b429f5218ea8470a539659354c6b2d8a994db1de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 10:14:01 +0000 Subject: [PATCH 0631/1284] Bump duckdb_jdbc from 0.4.0 to 0.5.1 Bumps [duckdb_jdbc](https://github.com/duckdb/duckdb) from 0.4.0 to 0.5.1. - [Release notes](https://github.com/duckdb/duckdb/releases) - [Commits](https://github.com/duckdb/duckdb/compare/v0.4.0...v0.5.1) --- updated-dependencies: - dependency-name: org.duckdb:duckdb_jdbc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 529f86a51..bca0450bc 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 0.4.0 + 0.5.1 org.junit.jupiter From 6850356f27a125ca777477c5476f7ec214da1f18 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Fri, 7 Oct 2022 10:12:44 +0800 Subject: [PATCH 0632/1284] Fix ALTER grammar in TiDB --- src/sqlancer/tidb/TiDBExpressionGenerator.java | 6 ------ src/sqlancer/tidb/TiDBSchema.java | 6 ++++++ src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java | 12 +++++------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index 20f48f04a..535647c53 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -85,9 +85,6 @@ protected TiDBExpression generateExpression(int depth) { case REGEX: return new TiDBRegexOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBRegexOperator.getRandom()); - // case COLLATE: - // return new TiDBCollate(generateExpression(depth + 1), - // Randomly.fromOptions("utf8mb4_bin", "latin1_bin", "binary", "ascii_bin", "utf8_bin")); case FUNCTION: TiDBFunction func = TiDBFunction.getRandom(); return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); @@ -100,9 +97,6 @@ protected TiDBExpression generateExpression(int depth) { } return new TiDBBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryLogicalOperator.getRandom()); - // case BINARY_ARITHMETIC: - // return new TiDBBinaryArithmeticOperation(generateExpression(depth + 1), generateExpression(depth + 1), - // TiDBBinaryArithmeticOperator.getRandom()); case CAST: return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions("BINARY", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 7caff2974..676475d4a 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -227,14 +227,20 @@ private static TiDBCompositeDataType getColumnType(String typeString) { primitiveType = TiDBDataType.TEXT; break; case "float": + size = 4; + primitiveType = TiDBDataType.FLOATING; + break; case "double": + size = 8; primitiveType = TiDBDataType.FLOATING; break; case "tinyint(1)": primitiveType = TiDBDataType.BOOL; + size = 1; break; case "null": primitiveType = TiDBDataType.INT; + size = 1; break; case "tinyint(3)": case "tinyint(4)": diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 057bb2789..a327dde39 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -9,6 +9,7 @@ import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; import sqlancer.tidb.TiDBSchema.TiDBDataType; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -25,11 +26,9 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add( "Information schema is changed during the execution of the statement(for example, table definition may be updated by other DDL ran in parallel)"); - errors.add("Data truncated"); - errors.add("Data truncation"); + errors.add("Data truncat"); errors.add("without a key length"); - errors.add("charset"); - errors.add("not supported"); + errors.add("supported"); errors.add("SQL syntax"); errors.add("can't drop"); StringBuilder sb = new StringBuilder("ALTER TABLE "); @@ -46,8 +45,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append("MODIFY "); sb.append(column.getName()); sb.append(" "); - sb.append(TiDBDataType.getRandom()); - errors.add("Unsupported modify column"); + sb.append(TiDBCompositeDataType.getRandom().toString()); break; case DROP_COLUMN: sb.append(" DROP "); @@ -102,7 +100,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append(" "); sb.append(column.getName()); sb.append(" "); - sb.append(column.getType().getPrimitiveDataType()); + sb.append(column.getType().toString()); sb.append(" NOT NULL "); errors.add("Invalid use of NULL value"); errors.add("Unsupported modify column:"); From f35e92bbf63eb3153589e66f71239d97441a8891 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 03:56:45 +0000 Subject: [PATCH 0633/1284] build(deps): bump maven-dependency-plugin from 3.2.0 to 3.3.0 Bumps [maven-dependency-plugin](https://github.com/apache/maven-dependency-plugin) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/apache/maven-dependency-plugin/releases) - [Commits](https://github.com/apache/maven-dependency-plugin/compare/maven-dependency-plugin-3.2.0...maven-dependency-plugin-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-dependency-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bca0450bc..dc01ce457 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.2.0 + 3.3.0 copy-dependencies From 12059f0a30d22754f42d1f8be77deb66b535c298 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 04:10:46 +0000 Subject: [PATCH 0634/1284] Bump maven-jar-plugin from 3.2.2 to 3.3.0 Bumps [maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 3.2.2 to 3.3.0. - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.2...maven-jar-plugin-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dc01ce457..d446aeb67 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.2 + 3.3.0 true From 903272af7e128bf6ae134495ad494207dc304faa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 05:39:35 +0000 Subject: [PATCH 0635/1284] Bump maven-shade-plugin from 3.3.0 to 3.4.0 Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.3.0 to 3.4.0. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.3.0...maven-shade-plugin-3.4.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-shade-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d446aeb67..d5d73cdd0 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.3.0 + 3.4.0 package From 4a1977fd0a75215da788a74ada95cbb153b9aa0b Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 9 Oct 2022 14:55:41 +0800 Subject: [PATCH 0636/1284] docs: add databend info Signed-off-by: Yisong Han --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d163f049c..eb0fcd779 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | | OceanBase | Working | Untyped | | | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | - +| Databend | Working | Typed | | # Using SQLancer From b3096a110c0997dfd4d58a0e3a6b1f699c2dc8d4 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 9 Oct 2022 15:41:20 +0800 Subject: [PATCH 0637/1284] chore: update unit test and databenderror etc. Signed-off-by: Yisong Han --- .gitignore | 1 + src/sqlancer/databend/DatabendErrors.java | 110 +++------------------- test/sqlancer/dbms/TestDatabend.java | 1 - 3 files changed, 14 insertions(+), 98 deletions(-) diff --git a/.gitignore b/.gitignore index 48efcf83a..a3ecf6076 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ target/ .settings/org.eclipse.core.resources.prefs .settings/org.eclipse.m2e.core.prefs .settings/org.eclipse.jdt.core.prefs +.vscode .project .checkstyle *.DS_Store diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 60ea7ca9c..7e1f80c4e 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -8,109 +8,25 @@ private DatabendErrors() { } public static void addExpressionErrors(ExpectedErrors errors) { - errors.add("with non-constant precision is not supported"); - errors.add("Like pattern must not end with escape character"); - errors.add("Could not convert string"); - errors.add("ORDER term out of range - should be between "); - errors.add("You might need to add explicit type casts."); - errors.add("can't be cast because the value is out of range for the destination type"); - errors.add("field value out of range"); - errors.add("Unimplemented type for cast"); - - errors.add("Escape string must be empty or one character."); - errors.add("Type mismatch when combining rows"); // BETWEEN - - errors.add("invalid UTF-8"); // TODO - errors.add("String value is not valid UTF8"); - - errors.add("Invalid TypeId "); // TODO - - errors.add("GROUP BY clause cannot contain aggregates!"); // investigate - - addRegexErrors(errors); - - addFunctionErrors(errors); - - errors.add("Overflow in multiplication"); - errors.add("Out of Range"); - errors.add("Date out of range"); - - // collate - errors.add("Cannot combine types with different collation!"); - errors.add("collations are only supported for type varchar"); - - // // https://github.com/cwida/Databend/issues/532 - errors.add("Not implemented type: DATE"); - errors.add("Not implemented type: TIMESTAMP"); - errors.add("Like pattern must not end with escape character!"); // LIKE - - errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a - // table supports rowids - - errors.add("does not have a column named"); // TODO: this only happens for views whose underlying table has a - // removed column - errors.add("Contents of view were altered: types don't match!"); - errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); - // TODO Databend待修复的bug（union schema error mismatch） - errors.add("unexpected end of file (failed to fill whole buffer)"); - } - - private static void addRegexErrors(ExpectedErrors errors) { - errors.add("missing ]"); - errors.add("missing )"); - errors.add("invalid escape sequence"); - errors.add("no argument for repetition operator: "); - errors.add("bad repetition operator"); - errors.add("trailing \\"); - errors.add("invalid perl operator"); - errors.add("invalid character class range"); - errors.add("width is not integer"); - } - - private static void addFunctionErrors(ExpectedErrors errors) { - errors.add("SUBSTRING cannot handle negative lengths"); - errors.add("is undefined outside [-1,1]"); // ACOS etc - errors.add("invalid type specifier"); // PRINTF - errors.add("argument index out of range"); // PRINTF - errors.add("invalid format string"); // PRINTF - errors.add("number is too big"); // PRINTF - errors.add("Like pattern must not end with escape character!"); // LIKE - errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part - errors.add("extract specifier"); // date_part - errors.add("not recognized"); // date_part - errors.add("not supported"); // date_part - errors.add("Failed to cast"); - errors.add("Conversion Error"); - errors.add("Could not cast value"); - errors.add("Insufficient padding in RPAD"); // RPAD - errors.add("Could not choose a best candidate function for the function call"); // monthname - errors.add("expected a numeric precision field"); // ROUND - errors.add("with non-constant precision is not supported"); // ROUND + errors.add("Division by zero"); + + /* + * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE + * databend2; CREATE TABLE t0(c0VARCHAR VARCHAR NULL, c1VARCHAR VARCHAR NULL, c2FLOAT FLOAT NOT NULL + * DEFAULT(NULL)); CREATE TABLE t1(c0INT BIGINT NULL); INSERT INTO t0(c1varchar, c0varchar) VALUES ('067596', + * '19'), ('', '87'); + */ + errors.add("Can't cast column from null into non-nullable type"); } public static void addInsertErrors(ExpectedErrors errors) { - addRegexErrors(errors); - addFunctionErrors(errors); - - errors.add("NOT NULL constraint failed"); - errors.add("PRIMARY KEY or UNIQUE constraint violated"); - errors.add("duplicate key"); - errors.add("can't be cast because the value is out of range for the destination type"); - errors.add("Could not convert string"); - errors.add("Unimplemented type for cast"); - errors.add("field value out of range"); - errors.add("CHECK constraint failed"); - errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid - errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if - // the table has a primary key - errors.add("Could not cast value"); - errors.add("create unique index, table contains duplicate data"); - errors.add("Failed to cast"); + errors.add("Division by zero"); + errors.add("Can't cast column from null into non-nullable type"); } public static void addGroupByErrors(ExpectedErrors errors) { - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("GROUP BY term out of range"); + errors.add("Division by zero"); + errors.add("Can't cast column from null into non-nullable type"); } } diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index a9575fd7d..326a43283 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -21,7 +21,6 @@ public void testDatabendNoREC() { "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); } - // TODO Databend待修复的bug（union schema error mismatch）https://github.com/datafuselabs/databend/issues/7463 @Test public void testDatabendTLPQueryPartitioning() { String databendAvailable = System.getenv("DATABEND_AVAILABLE"); From 72434c08523b5bb92a334b955e6c0aeb02856311 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 06:23:48 +0000 Subject: [PATCH 0638/1284] build(deps): bump checkstyle from 10.3.3 to 10.3.4 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3.3 to 10.3.4. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3.3...checkstyle-10.3.4) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d5d73cdd0..4c4f5b61c 100644 --- a/pom.xml +++ b/pom.xml @@ -209,7 +209,7 @@ com.puppycrawl.tools checkstyle - 10.3.3 + 10.3.4 From 93c26a4b52ad56cc80358a877a9ebcb0c8a9aa06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 09:15:44 +0000 Subject: [PATCH 0639/1284] build(deps): bump spotbugs-maven-plugin from 4.7.2.0 to 4.7.2.1 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.2.0 to 4.7.2.1. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.2.0...spotbugs-maven-plugin-4.7.2.1) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4c4f5b61c..1f9ebbe36 100644 --- a/pom.xml +++ b/pom.xml @@ -252,7 +252,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.2.0 + 4.7.2.1 spotbugs From 7b372ede548727584d69ee5855cb832cce88a887 Mon Sep 17 00:00:00 2001 From: Suri Zhang Date: Wed, 12 Oct 2022 13:56:27 +0800 Subject: [PATCH 0640/1284] Add Provider and Schema for QuestDB --- pom.xml | 7 +- src/sqlancer/GlobalState.java | 2 +- src/sqlancer/questdb/QuestDBErrors.java | 4 + src/sqlancer/questdb/QuestDBOptions.java | 55 ++++ src/sqlancer/questdb/QuestDBProvider.java | 106 ++++++ src/sqlancer/questdb/QuestDBSchema.java | 308 ++++++++++++++++++ .../questdb/gen/QuestDBDeleteGenerator.java | 15 + .../questdb/gen/QuestDBIndexGenerator.java | 14 + .../questdb/gen/QuestDBInsertGenerator.java | 35 ++ .../questdb/gen/QuestDBTableGenerator.java | 43 +++ .../questdb/gen/QuestDBUpdateGenerator.java | 16 + .../questdb/gen/QuestDBViewGenerator.java | 14 + 12 files changed, 617 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/questdb/QuestDBErrors.java create mode 100644 src/sqlancer/questdb/QuestDBOptions.java create mode 100644 src/sqlancer/questdb/QuestDBProvider.java create mode 100644 src/sqlancer/questdb/QuestDBSchema.java create mode 100644 src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBIndexGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBInsertGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBTableGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBViewGenerator.java diff --git a/pom.xml b/pom.xml index fdb287a10..9a736cc77 100644 --- a/pom.xml +++ b/pom.xml @@ -347,6 +347,11 @@ arangodb-java-driver 6.9.0 + + org.questdb + questdb + 6.5.3 + @@ -446,4 +451,4 @@ - + \ No newline at end of file diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 64b5c731e..2b93012c2 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -131,7 +131,7 @@ public S getSchema() { try { updateSchema(); } catch (Exception e) { - throw new AssertionError(); + throw new AssertionError(e.getMessage()); } } return schema; diff --git a/src/sqlancer/questdb/QuestDBErrors.java b/src/sqlancer/questdb/QuestDBErrors.java new file mode 100644 index 000000000..534c5c41d --- /dev/null +++ b/src/sqlancer/questdb/QuestDBErrors.java @@ -0,0 +1,4 @@ +package sqlancer.questdb; + +public class QuestDBErrors { +} diff --git a/src/sqlancer/questdb/QuestDBOptions.java b/src/sqlancer/questdb/QuestDBOptions.java new file mode 100644 index 000000000..ad2b593af --- /dev/null +++ b/src/sqlancer/questdb/QuestDBOptions.java @@ -0,0 +1,55 @@ +package sqlancer.questdb; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.questdb.QuestDBOptions.QuestDBOracleFactory; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +@Parameters(separators = "=", commandDescription = "QuestDB (default port: " + QuestDBOptions.DEFAULT_PORT + + " default host: " + QuestDBOptions.DEFAULT_HOST + ")") +public class QuestDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 8812; + + @Parameter(names = "--oracle") + public QuestDBOracleFactory oracle = QuestDBOracleFactory.WHERE; + + public enum QuestDBOracleFactory implements OracleFactory { + // TODO: implement test oracles + WHERE { + @Override + public TestOracle create(QuestDBGlobalState globalState) throws SQLException { + return null; + } + }; + + } + + @Parameter(names = "--username", description = "The user name used to log into QuestDB") + private String userName = "admin"; // NOPMD + + @Parameter(names = "--password", description = "The password used to log into QuestDB") + private String password = "quest"; // NOPMD + + @Override + public List getTestOracleFactory() { + return Arrays.asList(oracle); + } + + public String getUserName() { + return userName; + } + + public String getPassword() { + return password; + } + +} diff --git a/src/sqlancer/questdb/QuestDBProvider.java b/src/sqlancer/questdb/QuestDBProvider.java new file mode 100644 index 000000000..997ab40da --- /dev/null +++ b/src/sqlancer/questdb/QuestDBProvider.java @@ -0,0 +1,106 @@ +package sqlancer.questdb; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.gen.QuestDBDeleteGenerator; +import sqlancer.questdb.gen.QuestDBIndexGenerator; +import sqlancer.questdb.gen.QuestDBInsertGenerator; +import sqlancer.questdb.gen.QuestDBTableGenerator; + +@AutoService(DatabaseProvider.class) +public class QuestDBProvider extends SQLProviderAdapter { + public QuestDBProvider() { + super(QuestDBGlobalState.class, QuestDBOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(QuestDBInsertGenerator::getQuery), // + CREATE_INDEX(QuestDBIndexGenerator::getQuery), // + DELETE(QuestDBDeleteGenerator::generate); // + // TODO: maybe implement these later + // UPDATE(QuestDBUpdateGenerator::getQuery), // + // CREATE_VIEW(QuestDBViewGenerator::generate), // + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(QuestDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + public static class QuestDBGlobalState extends SQLGlobalState { + + @Override + protected QuestDBSchema readSchema() throws SQLException { + return QuestDBSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(QuestDBGlobalState globalState) throws Exception { + // TODO: should follow duckdb or cockrachdb? what's the difference between generate and create db? + } + + @Override + public SQLConnection createDatabase(QuestDBGlobalState globalState) throws Exception { + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = QuestDBOptions.DEFAULT_HOST; + } + if (port == sqlancer.MainOptions.NO_SET_PORT) { + port = QuestDBOptions.DEFAULT_PORT; + } + // TODO(anxing): maybe not hardcode here... + String databaseName = "qdb"; + String tableName = "test"; + String url = String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName); + // use QuestDB default username & password for Postgres JDBC + Properties properties = new Properties(); + properties.setProperty("user", globalState.getDbmsSpecificOptions().getUserName()); + properties.setProperty("password", globalState.getDbmsSpecificOptions().getPassword()); + properties.setProperty("sslmode", "disable"); + + Connection con = DriverManager.getConnection(url, properties); + // QuestDB cannot create or drop `DATABASE`, can only create or drop `TABLE` + globalState.getState().logStatement("DROP TABLE IF EXISTS " + tableName + " CASCADE"); + SQLQueryAdapter createTableCommand = new QuestDBTableGenerator().getQuery(globalState); + globalState.getState().logStatement(createTableCommand); + + try (Statement s = con.createStatement()) { + s.execute("DROP TABLE IF EXISTS " + tableName); + } + try (Statement s = con.createStatement()) { + s.execute(createTableCommand.getQueryString()); + } + con.close(); + con = DriverManager.getConnection(url, properties); + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "questdb"; + } + +} diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java new file mode 100644 index 000000000..ef2026706 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -0,0 +1,308 @@ +package sqlancer.questdb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; + +public class QuestDBSchema extends AbstractSchema { + + public enum QuestDBDataType { + + BOOLEAN, CHAR, + /* STRING, */ + INT, FLOAT, + /* SYMBOL, */ + DATE, TIMESTAMP, + /* GEOHASH, */ + NULL; + + public static QuestDBDataType getRandomWithoutNull() { + QuestDBDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == QuestDBDataType.NULL); + return dt; + } + + } + + public static class QuestDBCompositeDataType { + + private final QuestDBDataType dataType; + + private final int size; + + private final boolean isNullable; + + public QuestDBCompositeDataType(QuestDBDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + + switch (dataType) { + case INT: + switch (size) { + case 1: + case 2: + isNullable = false; + break; + default: + isNullable = true; + break; + } + break; + case BOOLEAN: + isNullable = false; + break; + default: + isNullable = true; + } + } + + public QuestDBDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public boolean isNullable() { + return isNullable; + } + + public static QuestDBCompositeDataType getRandomWithoutNull() { + QuestDBDataType type = QuestDBDataType.getRandomWithoutNull(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8, 32); + break; + case BOOLEAN: + case CHAR: + case DATE: + case TIMESTAMP: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new QuestDBCompositeDataType(type, size); + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 1: + return Randomly.fromOptions("BYTE"); + case 2: + return Randomly.fromOptions("SHORT"); + case 4: + return Randomly.fromOptions("INT"); + default: + throw new AssertionError(size); + } + case CHAR: + return "CHAR"; + case FLOAT: + switch (size) { + case 4: + return Randomly.fromOptions("FLOAT"); + case 8: + return Randomly.fromOptions(/* "DOUBLE", */"LONG"); + case 32: + return Randomly.fromOptions("LONG256"); + default: + throw new AssertionError(size); + } + case BOOLEAN: + return Randomly.fromOptions("BOOLEAN"); + case TIMESTAMP: + return Randomly.fromOptions("TIMESTAMP"); + case DATE: + return Randomly.fromOptions("DATE"); + case NULL: + return Randomly.fromOptions("NULL"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + } + + public static class QuestDBColumn extends AbstractTableColumn { + private final boolean isIndexed; + private final boolean isNullable; + + public QuestDBColumn(String name, QuestDBCompositeDataType columnType, boolean isIndexed) { + super(name, null, columnType); + this.isIndexed = isIndexed; + this.isNullable = columnType.isNullable(); + } + + public boolean isIndexed() { + return isIndexed; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class QuestDBTables extends AbstractTables { + public static final Set RESERVED_TABLES = new HashSet<>( + Arrays.asList("sys.column_versions_purge_log", "telemetry_config", "telemetry")); + + public QuestDBTables(List tables) { + super(tables); + } + } + + public QuestDBSchema(List databaseTables) { + super(databaseTables); + } + + public QuestDBTables getRandomTableNonEmptyTables() { + return new QuestDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + private static QuestDBCompositeDataType getColumnType(String typeString) { + QuestDBDataType primitiveType; + int size = -1; + + switch (typeString) { + case "INT": + primitiveType = QuestDBDataType.INT; + size = 4; + break; + case "CHAR": + primitiveType = QuestDBDataType.CHAR; + break; + case "FLOAT": + primitiveType = QuestDBDataType.FLOAT; + size = 4; + break; + case "LONG": + primitiveType = QuestDBDataType.FLOAT; + size = 8; + break; + case "LONG256": + primitiveType = QuestDBDataType.FLOAT; + size = 32; + break; + case "BOOLEAN": + primitiveType = QuestDBDataType.BOOLEAN; + break; + case "DATE": + primitiveType = QuestDBDataType.DATE; + break; + case "TIMESTAMP": + primitiveType = QuestDBDataType.TIMESTAMP; + break; + case "BYTE": + primitiveType = QuestDBDataType.INT; + size = 1; + break; + case "SHORT": + primitiveType = QuestDBDataType.INT; + size = 2; + break; + case "NULL": + primitiveType = QuestDBDataType.NULL; + break; + default: + throw new AssertionError(typeString); + } + return new QuestDBCompositeDataType(primitiveType, size); + } + + public static class QuestDBTable extends AbstractRelationalTable { + + public QuestDBTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static QuestDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + QuestDBTable t = new QuestDBTable(tableName, databaseColumns, isView); + for (QuestDBColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new QuestDBSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES;")) { + while (rs.next()) { + String tName = rs.getString("table"); + // exclude reserved tables for testing + if (!QuestDBTables.RESERVED_TABLES.contains(tName)) { + tableNames.add(rs.getString("table")); + } + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("SHOW COLUMNS FROM %s;", tableName))) { + while (rs.next()) { + String columnName = rs.getString("column"); + String dataType = rs.getString("type"); + boolean isIndexed = rs.getString("indexed").contains("true"); + QuestDBColumn c = new QuestDBColumn(columnName, getColumnType(dataType), isIndexed); + columns.add(c); + } + } + } + if (columns.stream().noneMatch(c -> c.isIndexed())) { + // TODO: implement an option to enable/disable rowids + columns.add(new QuestDBColumn("rowid", new QuestDBCompositeDataType(QuestDBDataType.INT, 4), false)); + } + return columns; + } + +} diff --git a/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java b/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java new file mode 100644 index 000000000..8cfefd093 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java @@ -0,0 +1,15 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBDeleteGenerator { + private QuestDBDeleteGenerator() { + + } + + public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { + // TODO + return null; + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java b/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java new file mode 100644 index 000000000..b994c22c0 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java @@ -0,0 +1,14 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBIndexGenerator { + private QuestDBIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + // TODO + return null; + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java new file mode 100644 index 000000000..807917b46 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -0,0 +1,35 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; + +public class QuestDBInsertGenerator extends AbstractInsertGenerator { + + private final QuestDBGlobalState globalState; + + // uncomment later: + // private final ExpectedErrors errors = new ExpectedErrors(); + + public QuestDBInsertGenerator(QuestDBGlobalState globalState) { + this.globalState = globalState; + } + + private SQLQueryAdapter generate() { + // TODO: below is dummy implementation to pass compilation + QuestDBSchema.QuestDBTable table = globalState.getSchema().getRandomTable(); + table.getRandomNonEmptyColumnSubset(); + return null; + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + return new QuestDBInsertGenerator(globalState).generate(); + } + + @Override + protected void insertValue(QuestDBColumn tiDBColumn) { + // TODO + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java new file mode 100644 index 000000000..360c9169b --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -0,0 +1,43 @@ +package sqlancer.questdb.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; + +public class QuestDBTableGenerator { + + public SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = "test"; // globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + sb.append(columns.get(i).getType()); + } + sb.append(");"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < sqlancer.Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + QuestDBCompositeDataType columnType = QuestDBCompositeDataType.getRandomWithoutNull(); + columns.add(new QuestDBColumn(columnName, columnType, false)); + } + return columns; + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java b/src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java new file mode 100644 index 000000000..cc7fba1df --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java @@ -0,0 +1,16 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBUpdateGenerator { + + private QuestDBUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + // TODO + return null; + } + +} diff --git a/src/sqlancer/questdb/gen/QuestDBViewGenerator.java b/src/sqlancer/questdb/gen/QuestDBViewGenerator.java new file mode 100644 index 000000000..0ff4d232b --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBViewGenerator.java @@ -0,0 +1,14 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBViewGenerator { + private QuestDBViewGenerator() { + } + + public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { + // TODO + return null; + } +} From 1108bbd91b84fbbbf5136b309f8e2245616b7591 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sat, 15 Oct 2022 12:13:32 +0800 Subject: [PATCH 0641/1284] fix: unstable deployment databend Signed-off-by: Yisong Han --- .github/workflows/main.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68ef0b2bf..5d069d3be 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -134,10 +134,8 @@ jobs: run: | sudo apt update sudo apt install mysql-client - LASTEST_TAG=$(curl -s GET https://api.github.com/repos/datafuselabs/databend/tags\?per_page\=1 | jq -r '.[].name') - curl -LJO https://github.com/datafuselabs/databend/releases/download/${LASTEST_TAG}/databend-${LASTEST_TAG}-x86_64-unknown-linux-musl.tar.gz - mkdir ./databend && tar xzvf databend-${LASTEST_TAG}-x86_64-unknown-linux-musl.tar.gz -C ./databend - ./databend/bin/databend-query & + docker pull datafuselabs/databend-query + docker run --net=host datafuselabs/databend-query & - name: Create SQLancer user run: mysql -uroot -h127.0.0.1 -P3307 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" - name: Run Tests From 4619e20034039f0c4527d5e56618fd64fd21ce3f Mon Sep 17 00:00:00 2001 From: kokrui Date: Mon, 17 Oct 2022 19:58:36 +0800 Subject: [PATCH 0642/1284] Add DuckDB canonicalization rule for -0 -> 0 --- .../duckdb/test/DuckDBQueryPartitioningBase.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 08c862973..68456ef26 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -3,7 +3,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -38,9 +37,16 @@ public DuckDBQueryPartitioningBase(DuckDBGlobalState state) { } public static String canonicalizeResultValue(String value) { - // Rule: -0.0 should be canonicalized to 0.0 - if (Objects.equals(value, "-0.0")) { + if (value == null) { + return value; + } + + switch (value) { + case "-0.0": return "0.0"; + case "-0": + return "0"; + default: } return value; From b70d306a8c76ae1af9091a23d736057354660824 Mon Sep 17 00:00:00 2001 From: kokrui Date: Mon, 17 Oct 2022 15:07:58 +0800 Subject: [PATCH 0643/1284] Remove now-obsolete DuckDB timestamp/date errors --- src/sqlancer/duckdb/DuckDBErrors.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index cb45bb96c..590a61d47 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -39,9 +39,6 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Cannot combine types with different collation!"); errors.add("collations are only supported for type varchar"); - // // https://github.com/cwida/duckdb/issues/532 - errors.add("Not implemented type: DATE"); - errors.add("Not implemented type: TIMESTAMP"); errors.add("Like pattern must not end with escape character!"); // LIKE errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a From 27abea1aa0694089f7e51ee5c6e80a924dc66f83 Mon Sep 17 00:00:00 2001 From: kokrui Date: Mon, 17 Oct 2022 15:17:15 +0800 Subject: [PATCH 0644/1284] Add expected DuckDB timestamp errors --- src/sqlancer/duckdb/DuckDBErrors.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 590a61d47..555bc2e88 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -48,6 +48,10 @@ public static void addExpressionErrors(ExpectedErrors errors) { // removed column errors.add("Contents of view were altered: types don't match!"); errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + + // timestamp + errors.add("Cannot subtract infinite timestamps"); + errors.add("Timestamp difference is out of bounds"); } private static void addRegexErrors(ExpectedErrors errors) { From 6b3a422fc4a6f21f23c2444dbfc09b853dde824c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 13:31:53 +0000 Subject: [PATCH 0645/1284] build(deps): bump jdbc-yugabytedb from 42.3.0 to 42.3.5-yb-1 Bumps [jdbc-yugabytedb](https://github.com/yugabyte/pgjdbc) from 42.3.0 to 42.3.5-yb-1. - [Release notes](https://github.com/yugabyte/pgjdbc/releases) - [Changelog](https://github.com/yugabyte/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/yugabyte/pgjdbc/commits) --- updated-dependencies: - dependency-name: com.yugabyte:jdbc-yugabytedb dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dfd0e2be7..098975cfb 100644 --- a/pom.xml +++ b/pom.xml @@ -294,7 +294,7 @@ com.yugabyte jdbc-yugabytedb - 42.3.0 + 42.3.5-yb-1 org.xerial From 294529cdf3317150069f4a5329acc886ce0beea1 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:49:39 +0200 Subject: [PATCH 0646/1284] more errors to ignore --- src/sqlancer/clickhouse/ClickHouseErrors.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 17993c7e4..908e5afec 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -37,10 +37,13 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); errors.add("Positional argument out of bounds"); + errors.add("Illegal value (aggregate function) for positional argument in GROUP BY"); + errors.add("Invalid escape sequence at the end of LIKE pattern"); } public static void addExpressionHavingErrors(ExpectedErrors errors) { errors.add("Memory limit"); + errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 } public static void addQueryErrors(ExpectedErrors errors) { @@ -62,6 +65,7 @@ public static void addTableManipulationErrors(ExpectedErrors errors) { errors.add("Sorting key cannot contain constants"); errors.add("Sampling expression must be present in the primary key"); errors.add("Cyclic aliases: default expression and column type are incompatible"); + errors.add("Positional argument out of bounds"); } } From 9e04f20b8ac3f7e14bc67e6bf8a037372c53127b Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 17 Oct 2022 21:00:06 +0200 Subject: [PATCH 0647/1284] add Having errors to TLPHaving oracle (#594) --- .../clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index 2cc2410d1..fc4f498cc 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -25,6 +25,7 @@ public ClickHouseTLPHavingOracle(ClickHouseProvider.ClickHouseGlobalState state) super(state); ClickHouseErrors.addExpectedExpressionErrors(errors); ClickHouseErrors.addGroupingErrors(errors); + ClickHouseErrors.addExpressionHavingErrors(errors); } @Override From efac7d93c18f0bb154119236dc066dd115ce9dbc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:37:59 +0000 Subject: [PATCH 0648/1284] build(deps): bump cassandra-jdbc-wrapper from 4.6.0 to 4.7.0 Bumps [cassandra-jdbc-wrapper](https://github.com/ing-bank/cassandra-jdbc-wrapper) from 4.6.0 to 4.7.0. - [Release notes](https://github.com/ing-bank/cassandra-jdbc-wrapper/releases) - [Changelog](https://github.com/ing-bank/cassandra-jdbc-wrapper/blob/master/CHANGELOG.md) - [Commits](https://github.com/ing-bank/cassandra-jdbc-wrapper/compare/v4.6.0...v4.7.0) --- updated-dependencies: - dependency-name: com.ing.data:cassandra-jdbc-wrapper dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 098975cfb..0b104497f 100644 --- a/pom.xml +++ b/pom.xml @@ -289,7 +289,7 @@ com.ing.data cassandra-jdbc-wrapper - 4.6.0 + 4.7.0 com.yugabyte From 4e328929ea9c6c917dd0a847bbced093f350c16b Mon Sep 17 00:00:00 2001 From: Elliana May Date: Wed, 19 Oct 2022 19:55:35 +0800 Subject: [PATCH 0649/1284] fix: don't pass username or password to duckdb It's an inmemory database and does not support them --- src/sqlancer/duckdb/DuckDBProvider.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 3164b7d6a..2384d894b 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -143,8 +143,7 @@ public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLExc String url = "jdbc:duckdb:" + databaseFile; tryDeleteDatabase(databaseFile); - Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); + Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement(); stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); stmt.close(); From 8a16b51e8753a58bb01846f9b8d997a60aba875c Mon Sep 17 00:00:00 2001 From: Elliana May Date: Wed, 19 Oct 2022 21:32:19 +0800 Subject: [PATCH 0650/1284] feat: add isDefaultUsername/isDefaultPassword methods to MainOptions --- src/sqlancer/MainOptions.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index ff588de5f..476dcb43f 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -5,9 +5,12 @@ import sqlancer.Randomly.StringGenerationStrategy; +import java.util.Objects; + @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { public static final int NO_SET_PORT = -1; + public static final MainOptions DEFAULT_OPTIONS = new MainOptions(); @Parameter(names = { "--help", "-h" }, description = "Lists all supported options and commands", help = true) private boolean help; // NOPMD @@ -218,6 +221,14 @@ public boolean isHelp() { return help; } + public boolean isDefaultPassword() { + return Objects.equals(password, DEFAULT_OPTIONS.password); + } + + public boolean isDefaultUsername() { + return Objects.equals(userName, DEFAULT_OPTIONS.userName); + } + public String getDatabasePrefix() { return databasePrefix; } From 6f255a596a65bb58483124094407d7c7c4b11d1b Mon Sep 17 00:00:00 2001 From: Elliana May Date: Wed, 19 Oct 2022 21:32:42 +0800 Subject: [PATCH 0651/1284] feat: throw an exception if someone attempts to pass creds to DuckDB --- src/sqlancer/duckdb/DuckDBProvider.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 2384d894b..59b26eac7 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -11,6 +11,7 @@ import sqlancer.AbstractAction; import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; @@ -143,6 +144,11 @@ public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLExc String url = "jdbc:duckdb:" + databaseFile; tryDeleteDatabase(databaseFile); + MainOptions options = globalState.getOptions(); + if (!(options.isDefaultUsername() && options.isDefaultPassword())) { + throw new AssertionError("DuckDB doesn't support credentials (username/password)"); + } + Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement(); stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); From 18dcbfcd925c9b633bc9a676dea5c0b4fb1d00a9 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Wed, 19 Oct 2022 21:46:22 +0800 Subject: [PATCH 0652/1284] chore: correct import order, per checkstyle error --- src/sqlancer/MainOptions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 476dcb43f..0b91dcc79 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -1,12 +1,12 @@ package sqlancer; +import java.util.Objects; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.Randomly.StringGenerationStrategy; -import java.util.Objects; - @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { public static final int NO_SET_PORT = -1; From 679044ac8e6e570410a7440c95b08eb25b40caf4 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 10 Oct 2022 17:15:15 -0400 Subject: [PATCH 0653/1284] Update CockroachDB to v22.2.0-beta.4 --- .github/workflows/main.yml | 6 +++--- src/sqlancer/cockroachdb/CockroachDBBugs.java | 4 ++++ src/sqlancer/cockroachdb/CockroachDBErrors.java | 9 +++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d069d3be..348bdd976 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,11 +109,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v21.1.7.linux-amd64.tgz | tar xvz - cd cockroach-v21.1.7.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.0-beta.4.linux-amd64.tgz | tar xvz + cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v21.1.7.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 3a980a8c7..0d708c9a2 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -50,6 +50,10 @@ public final class CockroachDBBugs { // https://github.com/cockroachdb/cockroach/issues/85499 public static boolean bug85499 = true; + // https://github.com/cockroachdb/cockroach/issues/88037 + // TODO: This should be fixed in v22.2.1. + public static boolean bug88037 = true; + private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 66a5e5d7d..bbbda7baf 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -186,6 +186,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (CockroachDBBugs.bug85499) { errors.add("estimated row count must be non-zero"); } + if (CockroachDBBugs.bug88037) { + errors.add("expected required columns to be a subset of output columns"); + } errors.add("unable to vectorize execution plan"); // SET vectorize=experimental_always; errors.add(" mismatched physical types at index"); // SET vectorize=experimental_always; @@ -203,6 +206,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("ERROR: for SELECT DISTINCT, ORDER BY expressions must appear in select list"); addArrayErrors(errors); + addComputedColumnErrors(errors); } private static void addArrayErrors(ExpectedErrors errors) { @@ -308,4 +312,9 @@ public static void addTransactionErrors(ExpectedErrors errors) { errors.add("current transaction is aborted"); } + private static void addComputedColumnErrors(ExpectedErrors errors) { + // computed columns + errors.add("computed column expressions cannot reference computed columns"); + } + } From 4c20a94b3ad2c037e1a66c0b637184f8c20faa7e Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 20 Oct 2022 22:13:23 +0200 Subject: [PATCH 0654/1284] [ClickHouse] Simplify errors skip lists (#596) Simplify ClickHouse error skip lists --- src/sqlancer/clickhouse/ClickHouseErrors.java | 72 ++++++++----------- .../gen/ClickHouseTableGenerator.java | 2 +- .../tlp/ClickHouseTLPAggregateOracle.java | 1 - .../oracle/tlp/ClickHouseTLPBase.java | 1 - .../oracle/tlp/ClickHouseTLPHavingOracle.java | 2 - .../oracle/tlp/ClickHouseTLPWhereOracle.java | 1 - 6 files changed, 29 insertions(+), 50 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 908e5afec..5aa973d22 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -8,64 +8,48 @@ private ClickHouseErrors() { } public static void addExpectedExpressionErrors(ExpectedErrors errors) { - errors.add("Illegal type"); + errors.add("is not under aggregate function and not in GROUP BY"); errors.add("Argument at index 1 for function like must be constant"); errors.add("Argument at index 1 for function notLike must be constant"); - errors.add("does not return a value of type UInt8"); - errors.add("invalid escape sequence"); - errors.add("invalid character class range"); - errors.add("Memory limit"); - errors.add("There is no supertype for types"); + errors.add("argument of function"); + errors.add("Attempt to read after eof: while converting"); errors.add("Bad get: has Int64, requested UInt64"); + errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 errors.add("Cannot convert string"); - errors.add("Cannot read floating point value"); + errors.add("Cannot insert NULL value into a column of type"); errors.add("Cannot parse infinity."); - errors.add("Attempt to read after eof: while converting"); - errors.add( - "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); - errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database - errors.add("is not under aggregate function"); - errors.add("Invalid type for filter in"); - errors.add("argument of function"); - errors.add(" is not under aggregate function and not in GROUP BY"); - errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () - errors.add("OptimizedRegularExpression: cannot compile re2"); - errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 - errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation + errors.add("Cannot parse Int32 from String, because value is too short"); errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 errors.add("Cannot parse number with a sign character but without any numeric character"); errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); + errors.add("Cannot parse string"); + errors.add("Cannot read floating point value"); + errors.add("Cyclic aliases: default expression and column type are incompatible"); + errors.add("Directory for table data"); + errors.add("Directory not empty"); + errors.add("does not return a value of type UInt8"); + errors.add("does not return a value of type UInt8"); + errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database + errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); - errors.add("Positional argument out of bounds"); + errors.add("Illegal type"); errors.add("Illegal value (aggregate function) for positional argument in GROUP BY"); - errors.add("Invalid escape sequence at the end of LIKE pattern"); - } - - public static void addExpressionHavingErrors(ExpectedErrors errors) { - errors.add("Memory limit"); errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 - } - - public static void addQueryErrors(ExpectedErrors errors) { - errors.add("Memory limit"); - } - - public static void addGroupingErrors(ExpectedErrors errors) { - errors.add("Memory limit"); - } - - public static void addTableManipulationErrors(ExpectedErrors errors) { + errors.add("invalid character class range"); + errors.add("Invalid escape sequence at the end of LIKE pattern"); + errors.add("invalid escape sequence"); + errors.add("Invalid type for filter in"); + errors.add("is not under aggregate function"); + errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation + errors.add( + "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); errors.add("Memory limit"); - errors.add("Directory for table data"); - errors.add("Directory not empty"); + errors.add("OptimizedRegularExpression: cannot compile re2"); errors.add("Partition key cannot contain constants"); - errors.add("Cannot convert string"); - errors.add("argument of function"); - errors.add("Attempt to read after eof: while converting"); - errors.add("Sorting key cannot contain constants"); - errors.add("Sampling expression must be present in the primary key"); - errors.add("Cyclic aliases: default expression and column type are incompatible"); errors.add("Positional argument out of bounds"); + errors.add("Sampling expression must be present in the primary key"); + errors.add("Sorting key cannot contain constants"); + errors.add("There is no supertype for types"); } } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index 4d6f97008..c2b2e89c2 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -37,7 +37,7 @@ public static SQLQueryAdapter createTableStatement(String tableName, ClickHouseTableGenerator chTableGenerator = new ClickHouseTableGenerator(tableName, globalState); chTableGenerator.start(); ExpectedErrors errors = new ExpectedErrors(); - ClickHouseErrors.addTableManipulationErrors(errors); + ClickHouseErrors.addExpectedExpressionErrors(errors); return new SQLQueryAdapter(chTableGenerator.sb.toString(), errors, true); } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 7bf1f01a7..1538e0a5f 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -26,7 +26,6 @@ public class ClickHouseTLPAggregateOracle extends ClickHouseTLPBase { public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState state) { super(state); ClickHouseErrors.addExpectedExpressionErrors(errors); - ClickHouseErrors.addQueryErrors(errors); } @Override diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 045aadc51..d3cfc6b11 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -31,7 +31,6 @@ public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase Date: Sat, 22 Oct 2022 03:59:04 +0000 Subject: [PATCH 0655/1284] build(deps): bump slf4j-simple from 1.7.36 to 2.0.3 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.36 to 2.0.3. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.36...v_2.0.3) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0b104497f..6dae29f62 100644 --- a/pom.xml +++ b/pom.xml @@ -325,7 +325,7 @@ org.slf4j slf4j-simple - 1.7.36 + 2.0.3 ru.yandex.clickhouse From 38b24ecd73a086147c5320d2527d520873e727e1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Oct 2022 12:15:01 +0800 Subject: [PATCH 0656/1284] Reference renamed SLF4 variable --- src/sqlancer/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a4404a6fa..a3f70b9f1 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -38,7 +38,7 @@ public final class Main { static boolean progressMonitorStarted; static { - System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "ERROR"); + System.setProperty(org.slf4j.simple.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "ERROR"); if (!LOG_DIRECTORY.exists()) { LOG_DIRECTORY.mkdir(); } From f32e0df931cc0a71e6cea1acb60124976e4ff9d7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Oct 2022 11:34:36 +0800 Subject: [PATCH 0657/1284] Remove the Travis CI references --- CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 81177e352..e7567316f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,12 +50,12 @@ You can run them using the following command: mvn verify ``` -We use [Travis-CI](https://travis-ci.com/) to automatically check PRs. +We use [GitHub Actions](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) to automatically check PRs. ## Testing -As part of the Travis-CI gate, we use smoke testing by running SQLancer on each supported DBMS for some minutes, to test that nothing is obviously broken. For DBMS for which all bugs have been fixed, we verify that SQLancer cannot find any further bugs (i.e., the return code is zero). +As part of the GitHub Actions check, we use smoke testing by running SQLancer on each supported DBMS for some minutes, to test that nothing is obviously broken. For DBMS for which all bugs have been fixed, we verify that SQLancer cannot find any further bugs (i.e., the return code is zero). In addition, we use [unit tests](https://github.com/sqlancer/sqlancer/tree/master/test/sqlancer) to test SQLancer's core functionality, such as random string and number generation as well as option passing. When fixing a bug, add a unit test, if it is easily possible. @@ -65,13 +65,13 @@ You can run the tests using the following command: mvn test ``` -Note that per default, the smoke testing is performed only for embedded DBMS (i.e., DuckDB and SQLite). To run smoke tests also for the other DBMS, you need to set environment variables. For example, you can run the MySQL smoke testing (and no other tests) using the following command: +Note that per default, the smoke testing is performed only for embedded DBMS (e.g., DuckDB and SQLite). To run smoke tests also for the other DBMS, you need to set environment variables. For example, you can run the MySQL smoke testing (and no other tests) using the following command: ``` MYSQL_AVAILABLE=true mvn -Dtest=TestMySQL test ``` -For up-to-date testing commands, check out the `.travis.yml` file. +For up-to-date testing commands, check out the `.github/workflows/main.yml` file. ## Reviewing @@ -88,4 +88,4 @@ Please pay attention to good commit messages (in particular subject lines). As b 2. Do not end the subject line with a period. For example, write "Refactor the handling of indexes" rather than "Refactor the handling of indexes.". 3. Use the imperative mood in the subject line. For example, write "Refactor the handling of indexes" rather than "Refactoring" or "Refactor**ed** the handling of indexes". -Please also pay attention to a clean commit history. Rather than merging with the main branch, use `git rebase` to rebase your commits on the main branch. Sometimes, it might happen that you discover an issue only after having already created a commit, for example, when an issue is found by `mvn verify` in the Travis CI. Do not introduce a separate commit for such issues. If the issue was introduced by the last commit, you can fix the issue, and use `git commit --amend` to change the latest commit. If the change was introduced by one of the previous commits, you can use `git rebase -i` to change the respective commit. If you already have a number of such commits, you can use `git squash` to "collapse" multiple commits into one. For more information, you might want to read [How (and Why!) to Keep Your Git Commit History Clean](https://about.gitlab.com/blog/2018/06/07/keeping-git-commit-history-clean/) written by Kushal Pandya. +Please also pay attention to a clean commit history. Rather than merging with the main branch, use `git rebase` to rebase your commits on the main branch. Sometimes, it might happen that you discover an issue only after having already created a commit, for example, when an issue is found by `mvn verify` in the CI checks. Do not introduce a separate commit for such issues. If the issue was introduced by the last commit, you can fix the issue, and use `git commit --amend` to change the latest commit. If the change was introduced by one of the previous commits, you can use `git rebase -i` to change the respective commit. If you already have a number of such commits, you can use `git squash` to "collapse" multiple commits into one. For more information, you might want to read [How (and Why!) to Keep Your Git Commit History Clean](https://about.gitlab.com/blog/2018/06/07/keeping-git-commit-history-clean/) written by Kushal Pandya. From eabd7b1440cc299b93c7cb62f6e1b8269c57307f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Oct 2022 11:51:32 +0800 Subject: [PATCH 0658/1284] Document how to deal with repeatedly triggered bugs --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e7567316f..b28a1be29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,10 @@ For a permissive DBMS, implementing the expression generator is easier, since th For a strict DBMS, the better approach is typically to attempt to generate expressions of the expected type. For PostgreSQL, the expression generator thus expects an additional type argument (see [PostgreSQLExpressionGenerator](https://github.com/sqlancer/sqlancer/blob/86647df8aa2dd8d167b5c3ce3297290f5b0b2bcd/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java#L251)). This type is propagated recursively. For example, if we require a predicate for the `WHERE` clause, we pass boolean as a type. The expression generator then calls a method `generateBooleanExpression` that attempts to produce a boolean expression, by, for example, generating a comparison (e.g., `<=`). For the comparison's operands, a random type is then selected and propagated. For example, if an integer type is selected, then `generateExpression` is called with this type once for the left operand, and once for the right operand. Note that this process does not guarantee that the expression will indeed have the expected type. It might happen, for example, that the expression generator attempts to produce an integer value, but that it produces a double value instead, namely when an integer overflow occurs, which, depending on the DBMS, implicitly converts the result to a floating-point value. +### Unfixed Bugs + +Often, some bugs are fixed only after an extended period, meaning that SQLancer will repeatedly report the same bug. In such cases, it might be possible to avoid generating the problematic pattern, or adding an expected error with the internal error message. Rather than, for example, commenting out the code with the bug-inducing pattern, a pattern implemented by the [TiDBBugs class](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java) should be applied. The core idea is to use a public, static flag for each issue, which is set to true as long as the issue persists (e.g., see [bug35652](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java#L55)). The work-around code is then executed—or the problematic pattern should not be generated—if the flag is set to true (e.g., [an expected error is added for bug35652](https://github.com/sqlancer/sqlancer/blob/59564d818d991d54b32fa5a79c9f733799c090f2/src/sqlancer/tidb/TiDBErrors.java#L47)). This makes it easy to later on identify and remove all such work-around code once the issue has been fixed. + ## Options SQLancer uses [JCommander](https://jcommander.org/) for handling options. The `MainOptions` class contains options that are expected to be supported by all DBMS-testing implementations. Furthermore, each `*Provider` class provides a method to return an additional set of supported options. From 75c626f1e90d8bec871412f08d23d982c22de702 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 24 Oct 2022 14:46:34 +0200 Subject: [PATCH 0659/1284] [ClickHouse] Added NoRec Oracle for ClickHouse and other improvements (#598) * Do not print failed CREATE and INSERT statements in reproduction scenario * ClickHouse passes check_names.py * More errors ignored * null results are now correctly reported from JDBC driver * Enable JOINs as only 1 table was generated before. Add USE statement for easier repro * Currently no support for estimated result. Drop dead code * Support NoRec. Added Alias operation. Co-authored-by: Manuel Rigger --- src/check_names.py | 2 +- src/sqlancer/Main.java | 4 +- src/sqlancer/MainOptions.java | 7 + src/sqlancer/clickhouse/ClickHouseErrors.java | 32 +++-- .../clickhouse/ClickHouseOptions.java | 7 + .../clickhouse/ClickHouseProvider.java | 5 +- src/sqlancer/clickhouse/ClickHouseSchema.java | 77 +++++++---- .../clickhouse/ClickHouseToStringVisitor.java | 11 +- .../clickhouse/ClickHouseVisitor.java | 5 + .../ast/ClickHouseAliasOperation.java | 28 ++++ .../ast/ClickHouseColumnReference.java | 17 +-- .../clickhouse/ast/ClickHouseExpression.java | 23 ++-- .../gen/ClickHouseExpressionGenerator.java | 33 ++++- .../gen/ClickHouseTableGenerator.java | 2 +- .../oracle/norec/ClickHouseNoRECOracle.java | 125 ++++++++++++++++++ .../tlp/ClickHouseTLPAggregateOracle.java | 2 +- .../oracle/tlp/ClickHouseTLPBase.java | 4 +- .../tlp/ClickHouseTLPGroupByOracle.java | 4 +- .../oracle/tlp/ClickHouseTLPHavingOracle.java | 2 +- test/sqlancer/dbms/TestClickHouse.java | 31 +++-- 20 files changed, 334 insertions(+), 87 deletions(-) create mode 100644 src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java create mode 100644 src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java diff --git a/src/check_names.py b/src/check_names.py index f76b881ab..bca448b19 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -18,7 +18,7 @@ def verify_prefix(prefix, files): print('The class name of ' + f + ' does not start with ' + prefix) exit(-1) -# TODO: ClickHouse (wait for https://github.com/sqlancer/sqlancer/pull/39) +verify_prefix('ClickHouse', get_java_files("sqlancer/clickhouse/")) verify_prefix('CockroachDB', get_java_files("sqlancer/cockroachdb/")) verify_prefix('DuckDB', get_java_files("sqlancer/duckdb")) verify_prefix('MariaDB', get_java_files("sqlancer/mariadb/")) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a3f70b9f1..7953386b7 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -222,10 +222,12 @@ public static class QueryManager { } public boolean execute(Query q, String... fills) throws Exception { - globalState.getState().logStatement(q); boolean success; success = q.execute(globalState, fills); Main.nrSuccessfulActions.addAndGet(1); + if (globalState.getOptions().loggerPrintFailed() || success) { + globalState.getState().logStatement(q); + } return success; } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 0b91dcc79..2916dbdd2 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -47,6 +47,9 @@ public class MainOptions { @Parameter(names = "--log-execution-time", description = "Logs the execution time of each statement (requires --log-each-select to be enabled)", arity = 1) private boolean logExecutionTime = true; // NOPMD + @Parameter(names = "--print-failed", description = "Logs failed insert, create and other statements without results", arity = 1) + private boolean loggerPrintFailed = true; // NOPMD + @Parameter(names = "--username", description = "The user name used to log into the DBMS") private String userName = "sqlancer"; // NOPMD @@ -141,6 +144,10 @@ public boolean logExecutionTime() { return logExecutionTime; } + public boolean loggerPrintFailed() { + return loggerPrintFailed; + } + public int getNrQueries() { return nrQueries; } diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 5aa973d22..88597d681 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -8,18 +8,15 @@ private ClickHouseErrors() { } public static void addExpectedExpressionErrors(ExpectedErrors errors) { - errors.add("is not under aggregate function and not in GROUP BY"); errors.add("Argument at index 1 for function like must be constant"); errors.add("Argument at index 1 for function notLike must be constant"); - errors.add("argument of function"); errors.add("Attempt to read after eof: while converting"); errors.add("Bad get: has Int64, requested UInt64"); - errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 errors.add("Cannot convert string"); errors.add("Cannot insert NULL value into a column of type"); - errors.add("Cannot parse infinity."); errors.add("Cannot parse Int32 from String, because value is too short"); errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 + errors.add("Cannot parse infinity."); errors.add("Cannot parse number with a sign character but without any numeric character"); errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); errors.add("Cannot parse string"); @@ -27,22 +24,12 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Cyclic aliases: default expression and column type are incompatible"); errors.add("Directory for table data"); errors.add("Directory not empty"); - errors.add("does not return a value of type UInt8"); - errors.add("does not return a value of type UInt8"); - errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); errors.add("Illegal type"); errors.add("Illegal value (aggregate function) for positional argument in GROUP BY"); - errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 - errors.add("invalid character class range"); errors.add("Invalid escape sequence at the end of LIKE pattern"); - errors.add("invalid escape sequence"); errors.add("Invalid type for filter in"); - errors.add("is not under aggregate function"); - errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation - errors.add( - "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); errors.add("Memory limit"); errors.add("OptimizedRegularExpression: cannot compile re2"); errors.add("Partition key cannot contain constants"); @@ -50,6 +37,23 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Sampling expression must be present in the primary key"); errors.add("Sorting key cannot contain constants"); errors.add("There is no supertype for types"); + errors.add("argument of function"); + errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 + errors.add("but its arguments considered equal according to constraints"); + errors.add("does not return a value of type UInt8"); + errors.add("does not return a value of type UInt8"); + errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database + errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 + errors.add("invalid character class range"); + errors.add("invalid escape sequence"); + errors.add("is not under aggregate function and not in GROUP BY"); + errors.add("is not under aggregate function"); + errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation + errors.add( + "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); + errors.add("there are only columns"); + errors.add("there are columns"); + errors.add("in block. (NOT_FOUND_COLUMN_IN_BLOCK)"); } } diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index c58697c98..8a2056c1b 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -11,6 +11,7 @@ import sqlancer.OracleFactory; import sqlancer.clickhouse.ClickHouseOptions.ClickHouseOracleFactory; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.oracle.norec.ClickHouseNoRECOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; @@ -60,6 +61,12 @@ public TestOracle create(ClickHouseGlobalState globalState) throws SQLException public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPHavingOracle(globalState); } + }, + NoREC { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseNoRECOracle(globalState); + } }; } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 7ff2de11b..e3a1fb21c 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -87,7 +87,7 @@ protected ClickHouseSchema readSchema() throws SQLException { @Override public void generateDatabase(ClickHouseGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(1); i++) { + for (int i = 0; i < Randomly.fromOptions(5); i++) { boolean success; do { String tableName = ClickHouseCommon.createTableName(i); @@ -96,6 +96,7 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws Exception } while (!success); } + // TODO: add more Actions to populate table StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), ClickHouseProvider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { @@ -126,6 +127,8 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ globalState.getState().logStatement(dropDatabaseCommand); String createDatabaseCommand = "CREATE DATABASE IF NOT EXISTS " + databaseName; globalState.getState().logStatement(createDatabaseCommand); + String useDatabaseCommand = "USE " + databaseName; // Noop. To reproduce easier. + globalState.getState().logStatement(useDatabaseCommand); try (Statement s = con.createStatement()) { s.execute(dropDatabaseCommand); Thread.sleep(1000); diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index 21b39e021..ff58fc7a9 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -60,14 +60,15 @@ public static class ClickHouseColumn extends AbstractTableColumn getTableColumns(SQLConnection con, String boolean isAlias = "ALIAS".compareTo(defaultType) == 0; boolean isMaterialized = "MATERIALIZED".compareTo(defaultType) == 0; ClickHouseColumn c = new ClickHouseColumn(columnName, getColumnType(dataType), isAlias, - isMaterialized); + isMaterialized, null); columns.add(c); } } diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index beaa37b49..2d78fcc8c 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; @@ -129,7 +130,8 @@ public void visit(ClickHouseCastOperation cast) { @Override public void visit(ClickHouseExpression.ClickHouseJoin join) { - + sb.append("ON "); + visit(join.getOnClause()); } @Override @@ -141,6 +143,13 @@ public void visit(ClickHouseColumnReference c) { } } + @Override + public void visit(ClickHouseAliasOperation alias) { + visit(alias.getExpression()); + sb.append(" AS "); + sb.append(alias.getAlias()); + } + public static String asString(ClickHouseExpression expr) { ClickHouseToStringVisitor visitor = new ClickHouseToStringVisitor(); visitor.visit(expr); diff --git a/src/sqlancer/clickhouse/ClickHouseVisitor.java b/src/sqlancer/clickhouse/ClickHouseVisitor.java index e1473f669..93a46d607 100644 --- a/src/sqlancer/clickhouse/ClickHouseVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseVisitor.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; @@ -51,6 +52,8 @@ default void visit(ClickHouseExpression.ClickHousePostfixText op) { void visit(ClickHouseCastOperation cast); + void visit(ClickHouseAliasOperation alias); + void visit(ClickHouseExpression.ClickHouseJoin join); void visit(ClickHouseAggregate aggregate); @@ -78,6 +81,8 @@ default void visit(ClickHouseExpression expr) { visit((ClickHouseExpression.ClickHousePostfixText) expr); } else if (expr instanceof ClickHouseAggregate) { visit((ClickHouseAggregate) expr); + } else if (expr instanceof ClickHouseAliasOperation) { + visit((ClickHouseAliasOperation) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java new file mode 100644 index 000000000..b7976ff46 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java @@ -0,0 +1,28 @@ +package sqlancer.clickhouse.ast; + +public class ClickHouseAliasOperation extends ClickHouseExpression { + + private final ClickHouseExpression expression; + private final String alias; + + public ClickHouseAliasOperation(ClickHouseExpression expression, String alias) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.alias = alias; + } + + @Override + public ClickHouseConstant getExpectedValue() { + return expression.getExpectedValue(); + } + + public ClickHouseExpression getExpression() { + return expression; + } + + public String getAlias() { + return alias; + } +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java index 818da0e7e..796eb8333 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java @@ -5,28 +5,17 @@ public class ClickHouseColumnReference extends ClickHouseExpression { private final ClickHouseColumn column; - private final ClickHouseConstant value; - public ClickHouseColumnReference(ClickHouseColumn column, ClickHouseConstant value) { + public ClickHouseColumnReference(ClickHouseColumn column) { this.column = column; - this.value = value; } - public static ClickHouseColumnReference create(ClickHouseColumn column, ClickHouseConstant value) { - return new ClickHouseColumnReference(column, value); + public static ClickHouseColumnReference create(ClickHouseColumn column) { + return new ClickHouseColumnReference(column); } public ClickHouseColumn getColumn() { return column; } - public ClickHouseConstant getValue() { - return value; - } - - @Override - public ClickHouseConstant getExpectedValue() { - return value; - } - } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index c3642c208..161fd0206 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -36,25 +36,28 @@ public ClickHouseExpression getExpression() { } public static class ClickHouseJoin extends ClickHouseExpression { - // TODO: support ANY, ALL, ASOF modifiers public enum JoinType { INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, NATURAL, LEFT_SEMI, RIGHT_SEMI, LEFT_ANTI, RIGHT_ANTI; } - private final ClickHouseSchema.ClickHouseTable table; + private final ClickHouseSchema.ClickHouseTable leftTable; + private final ClickHouseSchema.ClickHouseTable rightTable; private ClickHouseExpression onClause; private final ClickHouseJoin.JoinType type; - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable table, ClickHouseExpression onClause, - ClickHouseJoin.JoinType type) { - this.table = table; + public ClickHouseJoin(ClickHouseSchema.ClickHouseTable leftTable, ClickHouseSchema.ClickHouseTable rightTable, + ClickHouseJoin.JoinType type, ClickHouseExpression onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; this.onClause = onClause; this.type = type; } - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable table, ClickHouseJoin.JoinType type) { - this.table = table; + public ClickHouseJoin(ClickHouseSchema.ClickHouseTable leftTable, ClickHouseSchema.ClickHouseTable rightTable, + ClickHouseJoin.JoinType type) { + this.leftTable = leftTable; + this.rightTable = rightTable; if (type != ClickHouseJoin.JoinType.NATURAL) { throw new AssertionError(); } @@ -62,8 +65,12 @@ public ClickHouseJoin(ClickHouseSchema.ClickHouseTable table, ClickHouseJoin.Joi this.type = type; } + public ClickHouseSchema.ClickHouseTable getLeftTable() { + return leftTable; + } + public ClickHouseSchema.ClickHouseTable getTable() { - return table; + return rightTable; } public ClickHouseExpression getOnClause() { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 1054055f9..6e86b28ec 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -71,6 +71,14 @@ protected ClickHouseExpression generateExpression(ClickHouseLancerDataType type, } } + protected ClickHouseExpression generateJoinClause(ClickHouseSchema.ClickHouseTable leftTable, + ClickHouseSchema.ClickHouseTable rightTable) { + ClickHouseExpression leftExpr = getColumnNameFromTable(leftTable); + ClickHouseExpression rightExpr = getColumnNameFromTable(rightTable); + return new ClickHouseBinaryComparisonOperation(leftExpr, rightExpr, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.EQUALS); + } + @Override protected ClickHouseExpression generateColumn(ClickHouseLancerDataType type) { if (columns.isEmpty()) { @@ -80,7 +88,20 @@ protected ClickHouseExpression generateColumn(ClickHouseLancerDataType type) { .filter(c -> c.getType().getType().name().equals(type.getType().name())).collect(Collectors.toList()); ClickHouseColumn column = filteredColumns.isEmpty() ? Randomly.fromList(columns) : Randomly.fromList(filteredColumns); - return new ClickHouseColumnReference(column, null); + return new ClickHouseColumnReference(column); + } + + protected ClickHouseExpression getColumnNameFromTable(ClickHouseSchema.ClickHouseTable table) { + if (columns.isEmpty()) { + return generateConstant(ClickHouseLancerDataType.getRandom()); + } + List filteredColumns = columns.stream().filter(c -> c.getTable() == table) + .collect(Collectors.toList()); + if (filteredColumns.isEmpty()) { + return generateConstant(ClickHouseLancerDataType.getRandom()); + } + ClickHouseColumn column = Randomly.fromList(filteredColumns); + return new ClickHouseColumnReference(column); } @Override @@ -97,17 +118,17 @@ public List getRandomJoinClauses( if (Randomly.getBoolean() && tables.size() > 1) { int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); for (int i = 0; i < nrJoinClauses; i++) { - ClickHouseExpression joinClause = generateExpression(ClickHouseLancerDataType.getRandom()); - ClickHouseSchema.ClickHouseTable table = Randomly.fromList(tables); - tables.remove(table); + ClickHouseSchema.ClickHouseTable leftTable = Randomly.fromList(tables); + ClickHouseSchema.ClickHouseTable rightTable = Randomly.fromList(tables); + ClickHouseExpression joinClause = generateJoinClause(leftTable, rightTable); ClickHouseExpression.ClickHouseJoin.JoinType options; options = Randomly.fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); if (options == ClickHouseExpression.ClickHouseJoin.JoinType.NATURAL) { // NATURAL joins do not have an ON clause joinClause = null; } - ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(table, joinClause, - options); + ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(leftTable, rightTable, + options, joinClause); joinStatements.add(j); } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index c2b2e89c2..5b62e88bc 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -55,7 +55,7 @@ public void start() { sb.append(" ("); int nrColumns = 1 + Randomly.smallNumber(); for (int i = 0; i < nrColumns; i++) { - columns.add(ClickHouseSchema.ClickHouseColumn.createDummy(ClickHouseCommon.createColumnName(i))); + columns.add(ClickHouseSchema.ClickHouseColumn.createDummy(ClickHouseCommon.createColumnName(i), null)); } for (int i = 0; i < nrColumns; i++) { if (i != 0) { diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java new file mode 100644 index 000000000..b8da925e5 --- /dev/null +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -0,0 +1,125 @@ +package sqlancer.clickhouse.oracle.norec; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.clickhouse.ClickHouseErrors; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; +import sqlancer.clickhouse.ClickHouseToStringVisitor; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; +import sqlancer.clickhouse.ast.ClickHouseColumnReference; +import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.ast.ClickHouseTableReference; +import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; + +public class ClickHouseNoRECOracle extends NoRECBase implements TestOracle { + + private final ClickHouseSchema s; + + public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + ClickHouseErrors.addExpectedExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); + ClickHouseExpression randomWhereCondition = gen.generateExpression(ClickHouseLancerDataType.getRandom()); + List tables = randomTables.getTables(); + List tableList = tables.stream().map(t -> new ClickHouseTableReference(t)) + .collect(Collectors.toList()); + List joins = gen.getRandomJoinClauses(tables); + int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); + int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, + randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getSecondQuery(List tableList, ClickHouseExpression whereClause, + List joins) throws SQLException { + ClickHouseSelect select = new ClickHouseSelect(); + + ClickHouseExpression inner = new ClickHouseAliasOperation(whereClause, "check"); + + select.setFetchColumns(Arrays.asList(inner)); + select.setFromList(tableList); + select.setJoinClauses(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(check <> 0) FROM (" + ClickHouseToStringVisitor.asString(select) + + ") as res"; + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(SQLConnection con, List tableList, + List columns, ClickHouseExpression randomWhereCondition, + List joins) throws SQLException { + ClickHouseSelect select = new ClickHouseSelect(); + List allColumns = columns.stream().map((c) -> new ClickHouseColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions( + new ClickHouseExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + select.setJoinClauses(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = ClickHouseToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 1538e0a5f..220b8f6c0 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -76,7 +76,7 @@ public void check() throws SQLException { throw new AssertionError(); } else if (firstResult.isEmpty()) { return; - } else if (firstResult.size() == 1) { + } else if (firstResult.size() == 1 && secondResult.size() == 1) { if (firstResult.get(0).equals(secondResult.get(0))) { return; } else if (!ComparatorHelper.isEqualDouble(firstResult.get(0), secondResult.get(0))) { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index d3cfc6b11..245343830 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -50,8 +50,8 @@ public void check() throws SQLException { } List generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new ClickHouseColumnReference(c)) + .collect(Collectors.toList()); } @Override diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index bb3629604..0cdee3864 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -43,8 +43,8 @@ public void check() throws SQLException { @Override List generateFetchColumns() { List columns; - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new ClickHouseColumnReference(c)) + .collect(Collectors.toList()); return columns; } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index d015d8092..c7bde4b70 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -31,7 +31,7 @@ public void check() throws SQLException { ClickHouseSchema s = state.getSchema(); ClickHouseSchema.ClickHouseTables targetTables = s.getRandomTableNonEmptyTables(); List groupByColumns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); + .map(c -> new ClickHouseColumnReference(c)).collect(Collectors.toList()); List columns = targetTables.getColumns(); ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); ClickHouseExpressionGenerator aggrGen = new ClickHouseExpressionGenerator(state).allowAggregates(true) diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 2cf53eef2..073fdcbd3 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -35,8 +35,9 @@ public void testClickHouseTLPHaving() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPHaving")); + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "clickhouse", "--oracle", "TLPHaving")); } @Test @@ -45,8 +46,9 @@ public void testClickHouseTLPGroupBy() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPGroupBy")); + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "clickhouse", "--oracle", "TLPGroupBy")); } @Test @@ -55,8 +57,9 @@ public void testClickHouseTLPDistinct() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPDistinct")); + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "clickhouse", "--oracle", "TLPDistinct")); } @Test @@ -65,8 +68,20 @@ public void testClickHouseTLPAggregate() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPAggregate")); + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "clickhouse", "--oracle", "TLPAggregate")); + } + + @Test + public void testClickHouseNoREC() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "clickhouse", "--oracle", "NoREC")); } } From f2f1a99aeae389fffccf2640aab39d527b0fdba2 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Tue, 25 Oct 2022 12:34:42 +0200 Subject: [PATCH 0660/1284] fix (#601) --- .../clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 220b8f6c0..6275b6a87 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -74,7 +74,7 @@ public void check() throws SQLException { if (firstResult.size() != secondResult.size()) { throw new AssertionError(); - } else if (firstResult.isEmpty()) { + } else if (firstResult.isEmpty() || firstResult.equals(secondResult)) { return; } else if (firstResult.size() == 1 && secondResult.size() == 1) { if (firstResult.get(0).equals(secondResult.get(0))) { From 83f8bfb7991f16c39e70412000b8629f404aff1b Mon Sep 17 00:00:00 2001 From: Suri Zhang Date: Sun, 23 Oct 2022 20:54:45 +0800 Subject: [PATCH 0661/1284] Implement generators and ast for questdb --- src/sqlancer/questdb/QuestDBBugs.java | 10 ++ src/sqlancer/questdb/QuestDBErrors.java | 22 ++- src/sqlancer/questdb/QuestDBOptions.java | 12 +- src/sqlancer/questdb/QuestDBProvider.java | 62 +++++++- src/sqlancer/questdb/QuestDBSchema.java | 50 +++--- .../questdb/QuestDBToStringVisitor.java | 28 ++++ src/sqlancer/questdb/ast/QuestDBConstant.java | 89 +++++++++++ .../questdb/ast/QuestDBExpression.java | 4 + src/sqlancer/questdb/ast/QuestDBSelect.java | 16 ++ .../gen/QuestDBAlterIndexGenerator.java | 53 +++++++ .../questdb/gen/QuestDBDeleteGenerator.java | 15 -- .../gen/QuestDBExpressionGenerator.java | 146 ++++++++++++++++++ .../questdb/gen/QuestDBIndexGenerator.java | 14 -- .../questdb/gen/QuestDBInsertGenerator.java | 43 ++++-- .../questdb/gen/QuestDBTableGenerator.java | 33 +++- .../questdb/gen/QuestDBTruncateGenerator.java | 22 +++ .../test/QuestDBQueryPartitioningBase.java | 80 ++++++++++ .../QuestDBQueryPartitioningWhereTester.java | 41 +++++ 18 files changed, 656 insertions(+), 84 deletions(-) create mode 100644 src/sqlancer/questdb/QuestDBBugs.java create mode 100644 src/sqlancer/questdb/QuestDBToStringVisitor.java create mode 100644 src/sqlancer/questdb/ast/QuestDBConstant.java create mode 100644 src/sqlancer/questdb/ast/QuestDBExpression.java create mode 100644 src/sqlancer/questdb/ast/QuestDBSelect.java create mode 100644 src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java delete mode 100644 src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java delete mode 100644 src/sqlancer/questdb/gen/QuestDBIndexGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java create mode 100644 src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java create mode 100644 src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java diff --git a/src/sqlancer/questdb/QuestDBBugs.java b/src/sqlancer/questdb/QuestDBBugs.java new file mode 100644 index 000000000..bbba765a3 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBBugs.java @@ -0,0 +1,10 @@ +package sqlancer.questdb; + +public final class QuestDBBugs { + + // https://github.com/questdb/questdb/issues/2689 + public static boolean bug2689 = true; + + private QuestDBBugs() { + } +} diff --git a/src/sqlancer/questdb/QuestDBErrors.java b/src/sqlancer/questdb/QuestDBErrors.java index 534c5c41d..44dbea148 100644 --- a/src/sqlancer/questdb/QuestDBErrors.java +++ b/src/sqlancer/questdb/QuestDBErrors.java @@ -1,4 +1,24 @@ package sqlancer.questdb; -public class QuestDBErrors { +import sqlancer.common.query.ExpectedErrors; + +public final class QuestDBErrors { + + private QuestDBErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + // TODO (anxing) + } + + public static void addGroupByErrors(ExpectedErrors errors) { + // TODO (anxing) + } + + public static void addInsertErrors(ExpectedErrors errors) { + // TODO (anxing) + errors.add("Invalid column"); + errors.add("inconvertible types:"); + errors.add("inconvertible value:"); + } } diff --git a/src/sqlancer/questdb/QuestDBOptions.java b/src/sqlancer/questdb/QuestDBOptions.java index ad2b593af..527e80cc8 100644 --- a/src/sqlancer/questdb/QuestDBOptions.java +++ b/src/sqlancer/questdb/QuestDBOptions.java @@ -12,6 +12,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.questdb.QuestDBOptions.QuestDBOracleFactory; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.test.QuestDBQueryPartitioningWhereTester; @Parameters(separators = "=", commandDescription = "QuestDB (default port: " + QuestDBOptions.DEFAULT_PORT + " default host: " + QuestDBOptions.DEFAULT_HOST + ")") @@ -20,17 +21,16 @@ public class QuestDBOptions implements DBMSSpecificOptions public static final int DEFAULT_PORT = 8812; @Parameter(names = "--oracle") - public QuestDBOracleFactory oracle = QuestDBOracleFactory.WHERE; + public List oracle = Arrays.asList(QuestDBOracleFactory.WHERE); public enum QuestDBOracleFactory implements OracleFactory { - // TODO: implement test oracles + // TODO (anxing): implement test oracles WHERE { @Override public TestOracle create(QuestDBGlobalState globalState) throws SQLException { - return null; + return new QuestDBQueryPartitioningWhereTester(globalState); } - }; - + } } @Parameter(names = "--username", description = "The user name used to log into QuestDB") @@ -41,7 +41,7 @@ public TestOracle create(QuestDBGlobalState globalState) throws SQLException { @Override public List getTestOracleFactory() { - return Arrays.asList(oracle); + return oracle; } public String getUserName() { diff --git a/src/sqlancer/questdb/QuestDBProvider.java b/src/sqlancer/questdb/QuestDBProvider.java index 997ab40da..d9932c8a3 100644 --- a/src/sqlancer/questdb/QuestDBProvider.java +++ b/src/sqlancer/questdb/QuestDBProvider.java @@ -10,16 +10,19 @@ import sqlancer.AbstractAction; import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; -import sqlancer.questdb.gen.QuestDBDeleteGenerator; -import sqlancer.questdb.gen.QuestDBIndexGenerator; +import sqlancer.questdb.gen.QuestDBAlterIndexGenerator; import sqlancer.questdb.gen.QuestDBInsertGenerator; import sqlancer.questdb.gen.QuestDBTableGenerator; +import sqlancer.questdb.gen.QuestDBTruncateGenerator; @AutoService(DatabaseProvider.class) public class QuestDBProvider extends SQLProviderAdapter { @@ -29,9 +32,9 @@ public QuestDBProvider() { public enum Action implements AbstractAction { INSERT(QuestDBInsertGenerator::getQuery), // - CREATE_INDEX(QuestDBIndexGenerator::getQuery), // - DELETE(QuestDBDeleteGenerator::generate); // - // TODO: maybe implement these later + ALTER_INDEX(QuestDBAlterIndexGenerator::getQuery), // + TRUNCATE(QuestDBTruncateGenerator::generate); // + // TODO (anxing): maybe implement these later // UPDATE(QuestDBUpdateGenerator::getQuery), // // CREATE_VIEW(QuestDBViewGenerator::generate), // @@ -47,6 +50,20 @@ public SQLQueryAdapter getQuery(QuestDBGlobalState state) throws Exception { } } + private static int mapActions(QuestDBGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case ALTER_INDEX: + return r.getInteger(0, 3); + case TRUNCATE: + return r.getInteger(0, 5); + default: + throw new AssertionError("Unknown action: " + a); + } + } + public static class QuestDBGlobalState extends SQLGlobalState { @Override @@ -58,7 +75,23 @@ protected QuestDBSchema readSchema() throws SQLException { @Override public void generateDatabase(QuestDBGlobalState globalState) throws Exception { - // TODO: should follow duckdb or cockrachdb? what's the difference between generate and create db? + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new QuestDBTableGenerator().getQuery(globalState, null); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + QuestDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); } @Override @@ -84,15 +117,30 @@ public SQLConnection createDatabase(QuestDBGlobalState globalState) throws Excep Connection con = DriverManager.getConnection(url, properties); // QuestDB cannot create or drop `DATABASE`, can only create or drop `TABLE` globalState.getState().logStatement("DROP TABLE IF EXISTS " + tableName + " CASCADE"); - SQLQueryAdapter createTableCommand = new QuestDBTableGenerator().getQuery(globalState); + SQLQueryAdapter createTableCommand = new QuestDBTableGenerator().getQuery(globalState, tableName); globalState.getState().logStatement(createTableCommand); + globalState.getState().logStatement("DROP TABLE IF EXISTS " + tableName); try (Statement s = con.createStatement()) { s.execute("DROP TABLE IF EXISTS " + tableName); } + // TODO(anxing): Drop all previous tables in db + // List tableNames = + // globalState.getSchema().getDatabaseTables().stream().map(AbstractTable::getName).collect(Collectors.toList()); + // for (String tName : tableNames) { + // try (Statement s = con.createStatement()) { + // String query = "DROP TABLE IF EXISTS " + tName; + // globalState.getState().logStatement(query); + // s.execute(query); + // } + // } try (Statement s = con.createStatement()) { s.execute(createTableCommand.getQueryString()); } + // drop test table + try (Statement s = con.createStatement()) { + s.execute("DROP TABLE IF EXISTS " + tableName); + } con.close(); con = DriverManager.getConnection(url, properties); return new SQLConnection(con); diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index ef2026706..0b5563a31 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -25,11 +25,11 @@ public class QuestDBSchema extends AbstractSchema getTableNames(SQLConnection con) throws SQLException { + protected static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SHOW TABLES;")) { @@ -277,7 +277,7 @@ private static List getTableNames(SQLConnection con) throws SQLException String tName = rs.getString("table"); // exclude reserved tables for testing if (!QuestDBTables.RESERVED_TABLES.contains(tName)) { - tableNames.add(rs.getString("table")); + tableNames.add(tName); } } } @@ -298,10 +298,6 @@ private static List getTableColumns(SQLConnection con, String tab } } } - if (columns.stream().noneMatch(c -> c.isIndexed())) { - // TODO: implement an option to enable/disable rowids - columns.add(new QuestDBColumn("rowid", new QuestDBCompositeDataType(QuestDBDataType.INT, 4), false)); - } return columns; } diff --git a/src/sqlancer/questdb/QuestDBToStringVisitor.java b/src/sqlancer/questdb/QuestDBToStringVisitor.java new file mode 100644 index 000000000..7552b06cb --- /dev/null +++ b/src/sqlancer/questdb/QuestDBToStringVisitor.java @@ -0,0 +1,28 @@ +package sqlancer.questdb; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.questdb.ast.QuestDBConstant; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof QuestDBConstant) { + visit((QuestDBConstant) expr); + } else { // TODO: maybe implement QuestDBSelect & Join + throw new AssertionError("Unknown class: " + expr.getClass()); + } + } + + private void visit(QuestDBConstant constant) { + sb.append(constant.toString()); + } + + public static String asString(Node expr) { + QuestDBToStringVisitor visitor = new QuestDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBConstant.java b/src/sqlancer/questdb/ast/QuestDBConstant.java new file mode 100644 index 000000000..e0950e656 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBConstant.java @@ -0,0 +1,89 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.newast.Node; + +public class QuestDBConstant implements Node { + private QuestDBConstant() { + } + + public static class QuestDBNullConstant extends QuestDBConstant { + @Override + public String toString() { + return "NULL"; + } + } + + public static class QuestDBIntConstant extends QuestDBConstant { + private final long value; + + public QuestDBIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + } + + public static class QuestDBBooleanConstant extends QuestDBConstant { + private final boolean value; + + public QuestDBBooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public boolean getValue() { + return value; + } + } + + public static Node createIntConstant(long val) { + return new QuestDBIntConstant(val); + } + + public static class QuestDBDoubleConstant extends QuestDBConstant { + + private final double value; + + public QuestDBDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + } + + public static Node createBooleanConstant(boolean val) { + return new QuestDBBooleanConstant(val); + } + + public static Node createNullConstant() { + return new QuestDBNullConstant(); + } + + public static Node createFloatConstant(double val) { + return new QuestDBDoubleConstant(val); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBExpression.java b/src/sqlancer/questdb/ast/QuestDBExpression.java new file mode 100644 index 000000000..c2c7f1d74 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBExpression.java @@ -0,0 +1,4 @@ +package sqlancer.questdb.ast; + +public interface QuestDBExpression { +} diff --git a/src/sqlancer/questdb/ast/QuestDBSelect.java b/src/sqlancer/questdb/ast/QuestDBSelect.java new file mode 100644 index 000000000..023384487 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBSelect.java @@ -0,0 +1,16 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class QuestDBSelect extends SelectBase> implements Node { + private boolean isDistinct; + + public void setDistinct(boolean distinct) { + isDistinct = distinct; + } + + public boolean isDistinct() { + return isDistinct; + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java new file mode 100644 index 000000000..1e8242148 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.questdb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; + +public final class QuestDBAlterIndexGenerator { + private QuestDBAlterIndexGenerator() { + } + + enum Action { + ADD_INDEX, DROP_INDEX + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add(" does not have a column with name \"rowid\""); + errors.add("Table does not contain column rowid referenced in alter statement"); + errors.add("cannot create index"); + errors.add("Index flag is only supported for SYMBOL"); + errors.add("Invalid column: "); + + StringBuilder sb = new StringBuilder("ALTER TABLE "); + + QuestDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append(" "); + + sb.append("ALTER COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_INDEX: + sb.append("ADD INDEX"); + errors.add("already exists!"); + + break; + case DROP_INDEX: + sb.append("DROP INDEX"); + errors.add("Column is not indexed"); + break; + default: + throw new AssertionError("Unkown action:" + action); + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java b/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java deleted file mode 100644 index 8cfefd093..000000000 --- a/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java +++ /dev/null @@ -1,15 +0,0 @@ -package sqlancer.questdb.gen; - -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; - -public final class QuestDBDeleteGenerator { - private QuestDBDeleteGenerator() { - - } - - public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { - // TODO - return null; - } -} diff --git a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java new file mode 100644 index 000000000..82f82ac88 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java @@ -0,0 +1,146 @@ +package sqlancer.questdb.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBDataType; +import sqlancer.questdb.ast.QuestDBConstant; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBExpressionGenerator extends UntypedExpressionGenerator, QuestDBColumn> { + + private final QuestDBGlobalState globalState; + + public QuestDBExpressionGenerator(QuestDBGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, QuestDBUnaryPrefixOperator.NOT); + } + + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, QuestDBUnaryPostfixOperator.IS_NULL); + } + + @Override + public Node generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return QuestDBConstant.createNullConstant(); + } + QuestDBDataType type = QuestDBDataType.getRandomWithoutNull(); + switch (type) { + case INT: + return QuestDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + case BOOLEAN: + return QuestDBConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + return QuestDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); + // case CHAR: + // case DATE: + // case TIMESTAMP: + // throw new IgnoreMeException(); + default: + throw new AssertionError("Unknown type: " + type); + } + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } else { + throw new AssertionError("Expression generation failed, depth=" + depth); + } + } + + @Override + protected Node generateColumn() { + QuestDBColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode<>(column); + } + + public enum QuestDBUnaryPostfixOperator implements Operator { + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + QuestDBUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static QuestDBUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum QuestDBUnaryPrefixOperator implements Operator { + + NOT("NOT"); + + private String textRepr; + + QuestDBUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static QuestDBUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum QuestDBBinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum QuestDBBinaryComparisonOperator implements Operator { + EQUALS("="), GREATER_THAN(">"), GREATER_THAN_EQUALS(">="), LESS_THAN("<"), SMALLER_THAN_EQUALS("<="), + NOT_EQUALS("!="); + + private String textRepr; + + QuestDBBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java b/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java deleted file mode 100644 index b994c22c0..000000000 --- a/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java +++ /dev/null @@ -1,14 +0,0 @@ -package sqlancer.questdb.gen; - -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; - -public final class QuestDBIndexGenerator { - private QuestDBIndexGenerator() { - } - - public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { - // TODO - return null; - } -} diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java index 807917b46..754a0d2e5 100644 --- a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -1,27 +1,40 @@ package sqlancer.questdb.gen; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.questdb.QuestDBErrors; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; -import sqlancer.questdb.QuestDBSchema; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; +import sqlancer.questdb.QuestDBToStringVisitor; public class QuestDBInsertGenerator extends AbstractInsertGenerator { private final QuestDBGlobalState globalState; - // uncomment later: - // private final ExpectedErrors errors = new ExpectedErrors(); + private final ExpectedErrors errors = new ExpectedErrors(); public QuestDBInsertGenerator(QuestDBGlobalState globalState) { this.globalState = globalState; } private SQLQueryAdapter generate() { - // TODO: below is dummy implementation to pass compilation - QuestDBSchema.QuestDBTable table = globalState.getSchema().getRandomTable(); - table.getRandomNonEmptyColumnSubset(); - return null; + sb.append("INSERT INTO "); + QuestDBTable table = globalState.getSchema().getRandomTable(); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + QuestDBErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); } public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { @@ -29,7 +42,19 @@ public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { } @Override - protected void insertValue(QuestDBColumn tiDBColumn) { - // TODO + protected void insertColumns(List columns) { + sb.append("("); + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + insertValue(columns.get(nrColumn)); + } + sb.append(")"); + } + + @Override + protected void insertValue(QuestDBColumn questDBColumn) { + sb.append(QuestDBToStringVisitor.asString(new QuestDBExpressionGenerator(globalState).generateConstant())); } } diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java index 360c9169b..abd25e9d6 100644 --- a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -2,21 +2,30 @@ import java.util.ArrayList; import java.util.List; +import javax.annotation.Nullable; +import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBBugs; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; public class QuestDBTableGenerator { - public SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + public SQLQueryAdapter getQuery(QuestDBGlobalState globalState, @Nullable String tableName) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); - String tableName = "test"; // globalState.getSchema().getFreeTableName(); + String name = tableName; + if (tableName == null) { + name = globalState.getSchema().getFreeTableName(); + } sb.append("CREATE TABLE "); - sb.append(tableName); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(name); sb.append("("); List columns = getNewColumns(); for (int i = 0; i < columns.size(); i++) { @@ -27,13 +36,27 @@ public SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { sb.append(" "); sb.append(columns.get(i).getType()); } - sb.append(");"); + sb.append(")"); + // test index at Create Table + if (Randomly.getBooleanWithRatherLowProbability()) { + errors.add("cannot create index"); + // QuestDB does not support index for non-SYMBOL typed columns + errors.add("Index flag is only supported for SYMBOL"); + sb.append(","); + String index = String.format(" INDEX (%s)", Randomly.fromList(columns).getName()); + sb.append(index); + } + if (QuestDBBugs.bug2689) { + errors.add("Invalid metadata"); + } + sb.append(";"); + errors.add("table already exists"); return new SQLQueryAdapter(sb.toString(), errors, true); } private static List getNewColumns() { List columns = new ArrayList<>(); - for (int i = 0; i < sqlancer.Randomly.smallNumber() + 1; i++) { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { String columnName = String.format("c%d", i); QuestDBCompositeDataType columnType = QuestDBCompositeDataType.getRandomWithoutNull(); columns.add(new QuestDBColumn(columnName, columnType, false)); diff --git a/src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java b/src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java new file mode 100644 index 000000000..19c48f9f7 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java @@ -0,0 +1,22 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; + +public final class QuestDBTruncateGenerator { + private QuestDBTruncateGenerator() { + + } + + public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { + StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); + ExpectedErrors errors = new ExpectedErrors(); + QuestDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + QuestDBErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java new file mode 100644 index 000000000..fb6bd3c3f --- /dev/null +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -0,0 +1,80 @@ +package sqlancer.questdb.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; +import sqlancer.questdb.QuestDBSchema.QuestDBTables; +import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.questdb.ast.QuestDBSelect; +import sqlancer.questdb.gen.QuestDBExpressionGenerator; + +public class QuestDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, QuestDBGlobalState> implements TestOracle { + + QuestDBSchema s; + QuestDBTables targetTables; + QuestDBExpressionGenerator gen; + QuestDBSelect select; + + protected QuestDBQueryPartitioningBase(QuestDBGlobalState state) { + super(state); + QuestDBErrors.addExpressionErrors(errors); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new QuestDBColumn("*", null, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + } + return columns; + } + + public static String canonicalizeResultValue(String value) { + // Rule: -0.0 should be canonicalized to 0.0 + if (Objects.equals(value, "-0.0")) { + return "0.0"; + } + + return value; + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new QuestDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new QuestDBSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + // Ignore JOINs for now + select.setFromList(new ArrayList<>(tableList)); + select.setWhereClause(null); + } +} diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java new file mode 100644 index 000000000..fe5b5890d --- /dev/null +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java @@ -0,0 +1,41 @@ +package sqlancer.questdb.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBToStringVisitor; + +public class QuestDBQueryPartitioningWhereTester extends QuestDBQueryPartitioningBase { + public QuestDBQueryPartitioningWhereTester(QuestDBGlobalState state) { + super(state); + QuestDBErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = QuestDBToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + // Ignore OrderBy for now + + select.setWhereClause(predicate); + String firstQueryString = QuestDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = QuestDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = QuestDBToStringVisitor.asString(select); + + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, false, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, QuestDBQueryPartitioningBase::canonicalizeResultValue); + } +} From 488ca7d586fa3ba2c6e94216b902205f5ca5df1b Mon Sep 17 00:00:00 2001 From: Suri Zhang Date: Thu, 3 Nov 2022 15:22:33 +0800 Subject: [PATCH 0662/1284] Implement more expressions and add SELECT, WHERE to test --- src/sqlancer/questdb/QuestDBBugs.java | 3 - src/sqlancer/questdb/QuestDBErrors.java | 7 +++ .../questdb/QuestDBToStringVisitor.java | 41 ++++++++++++- src/sqlancer/questdb/ast/QuestDBConstant.java | 4 +- .../gen/QuestDBExpressionGenerator.java | 58 ++++++++++++++++++- .../questdb/gen/QuestDBTableGenerator.java | 13 ----- 6 files changed, 105 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/questdb/QuestDBBugs.java b/src/sqlancer/questdb/QuestDBBugs.java index bbba765a3..1bd565823 100644 --- a/src/sqlancer/questdb/QuestDBBugs.java +++ b/src/sqlancer/questdb/QuestDBBugs.java @@ -2,9 +2,6 @@ public final class QuestDBBugs { - // https://github.com/questdb/questdb/issues/2689 - public static boolean bug2689 = true; - private QuestDBBugs() { } } diff --git a/src/sqlancer/questdb/QuestDBErrors.java b/src/sqlancer/questdb/QuestDBErrors.java index 44dbea148..83c74fdc1 100644 --- a/src/sqlancer/questdb/QuestDBErrors.java +++ b/src/sqlancer/questdb/QuestDBErrors.java @@ -9,6 +9,13 @@ private QuestDBErrors() { public static void addExpressionErrors(ExpectedErrors errors) { // TODO (anxing) + errors.add("unexpected argument for function: "); + errors.add("unexpected token:"); // SELECT FROM multiple tables without WHERE/ JOIN clause + errors.add("boolean expression expected"); + errors.add("Column name expected"); + errors.add("too few arguments for 'in'"); + errors.add("cannot compare TIMESTAMP with type"); // WHERE column IN with nonTIMESTAMP arg + errors.add("constant expected"); } public static void addGroupByErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/questdb/QuestDBToStringVisitor.java b/src/sqlancer/questdb/QuestDBToStringVisitor.java index 7552b06cb..295a6d910 100644 --- a/src/sqlancer/questdb/QuestDBToStringVisitor.java +++ b/src/sqlancer/questdb/QuestDBToStringVisitor.java @@ -4,6 +4,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.questdb.ast.QuestDBConstant; import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.questdb.ast.QuestDBSelect; public class QuestDBToStringVisitor extends NewToStringVisitor { @@ -11,7 +12,9 @@ public class QuestDBToStringVisitor extends NewToStringVisitor expr) { if (expr instanceof QuestDBConstant) { visit((QuestDBConstant) expr); - } else { // TODO: maybe implement QuestDBSelect & Join + } else if (expr instanceof QuestDBSelect) { + visit((QuestDBSelect) expr); + } else { // TODO: maybe implement QuestDBJoin throw new AssertionError("Unknown class: " + expr.getClass()); } } @@ -20,6 +23,42 @@ private void visit(QuestDBConstant constant) { sb.append(constant.toString()); } + private void visit(QuestDBSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + // if (!select.getGroupByExpressions().isEmpty()) { + // sb.append(" GROUP BY "); + // visit(select.getGroupByExpressions()); + // } + // if (select.getHavingClause() != null) { + // sb.append(" HAVING "); + // visit(select.getHavingClause()); + // } + // if (!select.getOrderByExpressions().isEmpty()) { + // sb.append(" ORDER BY "); + // visit(select.getOrderByExpressions()); + // } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + } + public static String asString(Node expr) { QuestDBToStringVisitor visitor = new QuestDBToStringVisitor(); visitor.visit(expr); diff --git a/src/sqlancer/questdb/ast/QuestDBConstant.java b/src/sqlancer/questdb/ast/QuestDBConstant.java index e0950e656..a59ce1053 100644 --- a/src/sqlancer/questdb/ast/QuestDBConstant.java +++ b/src/sqlancer/questdb/ast/QuestDBConstant.java @@ -66,9 +66,9 @@ public double getValue() { @Override public String toString() { if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; + return "cast('Infinity' as double)"; } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; + return "cast('-Infinity' as double)"; } return String.valueOf(value); } diff --git a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java index 82f82ac88..8f9ea443a 100644 --- a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java @@ -1,8 +1,14 @@ package sqlancer.questdb.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewInOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; @@ -21,6 +27,10 @@ public QuestDBExpressionGenerator(QuestDBGlobalState globalState) { this.globalState = globalState; } + private enum Expression { + UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, IN + } + @Override public Node negatePredicate(Node predicate) { return new NewUnaryPrefixOperatorNode<>(predicate, QuestDBUnaryPrefixOperator.NOT); @@ -57,7 +67,31 @@ public Node generateConstant() { protected Node generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); - } else { + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + + switch (expr) { + case UNARY_PREFIX: + return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), + QuestDBUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), + QuestDBUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + QuestDBBinaryComparisonOperator.getRandom()); + case BINARY_ARITHMETIC: + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + QuestDBBinaryArithmeticOperator.getRandom()); + case BINARY_LOGICAL: + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + QuestDBBinaryLogicalOperator.getRandom()); + case IN: + return new NewInOperatorNode<>(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + default: throw new AssertionError("Expression generation failed, depth=" + depth); } } @@ -125,7 +159,7 @@ public static Operator getRandom() { public enum QuestDBBinaryComparisonOperator implements Operator { EQUALS("="), GREATER_THAN(">"), GREATER_THAN_EQUALS(">="), LESS_THAN("<"), SMALLER_THAN_EQUALS("<="), - NOT_EQUALS("!="); + NOT_EQUALS("!="), REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); private String textRepr; @@ -143,4 +177,24 @@ public String getTextRepresentation() { } } + + public enum QuestDBBinaryArithmeticOperator implements Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"); // , LSHIFT("<<"), + // RSHIFT(">>"); + + private String textRepr; + + QuestDBBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } } diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java index abd25e9d6..a308b8d10 100644 --- a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -7,7 +7,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.questdb.QuestDBBugs; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; @@ -37,18 +36,6 @@ public SQLQueryAdapter getQuery(QuestDBGlobalState globalState, @Nullable String sb.append(columns.get(i).getType()); } sb.append(")"); - // test index at Create Table - if (Randomly.getBooleanWithRatherLowProbability()) { - errors.add("cannot create index"); - // QuestDB does not support index for non-SYMBOL typed columns - errors.add("Index flag is only supported for SYMBOL"); - sb.append(","); - String index = String.format(" INDEX (%s)", Randomly.fromList(columns).getName()); - sb.append(index); - } - if (QuestDBBugs.bug2689) { - errors.add("Invalid metadata"); - } sb.append(";"); errors.add("table already exists"); return new SQLQueryAdapter(sb.toString(), errors, true); From 201f2751c13e1cfe2aff99f18a21496b8e71e37e Mon Sep 17 00:00:00 2001 From: "Anxing (Suri) Zhang" Date: Fri, 4 Nov 2022 12:33:35 +0800 Subject: [PATCH 0663/1284] Update README.md Add QuestDB to supported DBMS --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index eb0fcd779..af793f42c 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | OceanBase | Working | Untyped | | | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | | Databend | Working | Typed | | +| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | + # Using SQLancer From 913d53bceca42e576e010b035460b83267640119 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Fri, 4 Nov 2022 14:42:29 +0100 Subject: [PATCH 0664/1284] Use new ClickHouse DockerHub Updated clickhouse-jdbc, added functions support and many improvements in column and tables names --- .github/workflows/main.yml | 6 +- src/sqlancer/clickhouse/ClickHouseErrors.java | 9 + src/sqlancer/clickhouse/ClickHouseSchema.java | 20 +- .../clickhouse/ClickHouseToStringVisitor.java | 95 +- .../clickhouse/ClickHouseVisitor.java | 9 +- .../clickhouse/ast/ClickHouseAggregate.java | 3 +- .../ClickHouseBinaryArithmeticOperation.java | 65 + .../ClickHouseBinaryComparisonOperation.java | 18 +- .../ClickHouseBinaryFunctionOperation.java | 57 + .../ast/ClickHouseBinaryLogicalOperation.java | 23 +- .../clickhouse/ast/ClickHouseCast.java | 42 +- .../ast/ClickHouseCastOperation.java | 3 +- .../ast/ClickHouseColumnReference.java | 19 +- .../clickhouse/ast/ClickHouseConstant.java | 1894 +---------------- .../clickhouse/ast/ClickHouseExpression.java | 50 +- .../clickhouse/ast/ClickHouseSelect.java | 18 +- .../ast/ClickHouseTableReference.java | 19 +- .../ast/ClickHouseUnaryFunctionOperation.java | 52 + .../ast/ClickHouseUnaryPostfixOperation.java | 3 +- .../ast/ClickHouseUnaryPrefixOperation.java | 12 +- .../constant/ClickHouseBooleanConstant.java | 142 ++ .../constant/ClickHouseCreateConstant.java | 162 ++ .../constant/ClickHouseFloat32Constant.java | 136 ++ .../constant/ClickHouseFloat64Constant.java | 136 ++ .../constant/ClickHouseInt128Constant.java | 139 ++ .../ast/constant/ClickHouseInt16Constant.java | 130 ++ .../constant/ClickHouseInt256Constant.java | 139 ++ .../ast/constant/ClickHouseInt32Constant.java | 130 ++ .../ast/constant/ClickHouseInt64Constant.java | 131 ++ .../ast/constant/ClickHouseInt8Constant.java | 130 ++ .../ast/constant/ClickHouseNullConstant.java | 53 + .../constant/ClickHouseStringConstant.java | 125 ++ .../constant/ClickHouseUInt128Constant.java | 139 ++ .../constant/ClickHouseUInt16Constant.java | 130 ++ .../constant/ClickHouseUInt256Constant.java | 139 ++ .../constant/ClickHouseUInt32Constant.java | 130 ++ .../constant/ClickHouseUInt64Constant.java | 131 ++ .../ast/constant/ClickHouseUInt8Constant.java | 138 ++ .../clickhouse/gen/ClickHouseCommon.java | 2 +- .../gen/ClickHouseExpressionGenerator.java | 132 +- .../gen/ClickHouseTableGenerator.java | 16 +- .../oracle/norec/ClickHouseNoRECOracle.java | 29 +- .../tlp/ClickHouseTLPAggregateOracle.java | 60 +- .../oracle/tlp/ClickHouseTLPBase.java | 44 +- .../tlp/ClickHouseTLPGroupByOracle.java | 13 - .../oracle/tlp/ClickHouseTLPHavingOracle.java | 33 +- ...ickHouseBinaryComparisonOperationTest.java | 171 +- .../ast/ClickHouseOperatorsVisitorTest.java | 117 + .../ast/ClickHouseToStringVisitorTest.java | 345 +++ 49 files changed, 3535 insertions(+), 2204 deletions(-) create mode 100644 src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java create mode 100644 src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java create mode 100644 src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java create mode 100644 test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java create mode 100644 test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 348bdd976..67ade8e58 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,11 +87,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up ClickHouse run: | - docker pull yandex/clickhouse-server:latest - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest + docker pull clickhouse/clickhouse-server:latest + docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:latest sleep 5 - name: Run Tests - run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test + run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test cockroachdb: diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 88597d681..d2b6c700a 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -54,6 +54,15 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("there are only columns"); errors.add("there are columns"); errors.add("in block. (NOT_FOUND_COLUMN_IN_BLOCK)"); + errors.add("Missing columns"); + errors.add("Ambiguous column"); + errors.add("Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)"); + errors.add("Floating point partition key is not supported"); + errors.add("Cannot get JOIN keys from JOIN ON section"); + errors.add("ILLEGAL_DIVISION"); + errors.add("DECIMAL_OVERFLOW"); + errors.add("Cannot convert out of range floating point value to integer type"); + errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/42768 } } diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index ff58fc7a9..97f4534a1 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -8,12 +8,15 @@ import java.util.List; import java.util.Map; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseConstant; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; @@ -34,7 +37,7 @@ public ClickHouseLancerDataType(ClickHouseDataType type) { } public ClickHouseLancerDataType(String textRepr) { - this.clickHouseType = ClickHouseDataType.fromTypeString(textRepr); + this.clickHouseType = ClickHouseDataType.of(textRepr); this.textRepr = textRepr; } @@ -78,6 +81,11 @@ public boolean isAlias() { public boolean isMaterialized() { return isMaterialized; } + + public ClickHouseColumnReference asColumnReference(String tableAlias) { + return new ClickHouseColumnReference(this, null, tableAlias); + } + } public static ClickHouseConstant getConstant(ResultSet randomRowValues, int columnIndex, @@ -85,20 +93,20 @@ public static ClickHouseConstant getConstant(ResultSet randomRowValues, int colu Object value; ClickHouseConstant constant; if (randomRowValues.getString(columnIndex) == null) { - constant = ClickHouseConstant.createNullConstant(); + constant = ClickHouseCreateConstant.createNullConstant(); } else { switch (valueType) { case Int32: value = randomRowValues.getLong(columnIndex); - constant = ClickHouseConstant.createInt32Constant((long) value); + constant = ClickHouseCreateConstant.createInt32Constant((long) value); break; case Float64: value = randomRowValues.getDouble(columnIndex); - constant = ClickHouseConstant.createFloat64Constant((double) value); + constant = ClickHouseCreateConstant.createFloat64Constant((double) value); break; case String: value = randomRowValues.getString(columnIndex); - constant = ClickHouseConstant.createStringConstant((String) value); + constant = ClickHouseCreateConstant.createStringConstant((String) value); break; case AggregateFunction: case Array: diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index 2d78fcc8c..cbc622560 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -1,8 +1,10 @@ package sqlancer.clickhouse; +import java.util.List; + import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; -import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; @@ -21,17 +23,6 @@ public void visitSpecific(ClickHouseExpression expr) { ClickHouseVisitor.super.visit(expr); } - @Override - public void visit(ClickHouseBinaryComparisonOperation op) { - sb.append("("); - visit(op.getLeft()); - sb.append(") "); - sb.append(op.getOperator().getTextRepresentation()); - sb.append(" ("); - visit(op.getRight()); - sb.append(")"); - } - @Override public void visit(ClickHouseBinaryLogicalOperation op) { sb.append("("); @@ -83,8 +74,17 @@ public void visit(ClickHouseSelect select, boolean inner) { } visit(select.getFetchColumns()); - sb.append(" FROM "); - visit(select.getFromList()); + ClickHouseExpression fromClause = select.getFromClause(); + if (fromClause != null) { + sb.append(" FROM "); + visit(fromClause); + } + List joins = select.getJoinClauses(); + if (!joins.isEmpty()) { + for (ClickHouseExpression.ClickHouseJoin join : joins) { + visit(join); + } + } if (select.getWhereClause() != null) { sb.append(" WHERE "); visit(select.getWhereClause()); @@ -108,7 +108,12 @@ public void visit(ClickHouseSelect select, boolean inner) { @Override public void visit(ClickHouseTableReference tableReference) { - sb.append(tableReference.getTable().getName()); + sb.append(tableReference.getTable().getName()); // Original name, not alias. + String alias = tableReference.getAlias(); + if (alias != null) { + sb.append(" AS " + alias); + } + } @Override @@ -130,24 +135,76 @@ public void visit(ClickHouseCastOperation cast) { @Override public void visit(ClickHouseExpression.ClickHouseJoin join) { - sb.append("ON "); - visit(join.getOnClause()); + ClickHouseExpression.ClickHouseJoin.JoinType type = join.getType(); + if (type == ClickHouseExpression.ClickHouseJoin.JoinType.CROSS) { + sb.append(" JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.INNER) { + sb.append(" INNER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_OUTER) { + sb.append(" LEFT OUTER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_OUTER) { + sb.append(" RIGHT OUTER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.FULL_OUTER) { + sb.append(" FULL OUTER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_SEMI) { + sb.append(" LEFT SEMI JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_SEMI) { + sb.append(" RIGHT SEMI JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_ANTI) { + sb.append(" LEFT ANTI JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_ANTI) { + sb.append(" RIGHT ANTI JOIN "); + visit(join.getRightTable()); + } else { + throw new UnsupportedOperationException(); + } + ClickHouseExpression onClause = join.getOnClause(); + if (onClause != null) { + sb.append(" ON "); + visit(onClause); + } } @Override public void visit(ClickHouseColumnReference c) { - if (c.getColumn().getTable() == null) { + if (c.getTableAlias() != null) { + sb.append(c.getTableAlias()); + sb.append("."); + sb.append(c.getColumn().getName()); + } else if (c.getColumn().getTable() == null) { sb.append(c.getColumn().getName()); } else { sb.append(c.getColumn().getFullQualifiedName()); } + if (c.getAlias() != null) { + sb.append(" AS " + c.getAlias()); + } + } + + @Override + public void visit(ClickHouseBinaryFunctionOperation func) { + sb.append(func.getOperatorRepresentation()); + sb.append("("); + visit(func.getLeft()); + sb.append(","); + visit(func.getRight()); + sb.append(")"); } @Override public void visit(ClickHouseAliasOperation alias) { visit(alias.getExpression()); - sb.append(" AS "); + sb.append(" AS `"); sb.append(alias.getAlias()); + sb.append("`"); } public static String asString(ClickHouseExpression expr) { diff --git a/src/sqlancer/clickhouse/ClickHouseVisitor.java b/src/sqlancer/clickhouse/ClickHouseVisitor.java index 93a46d607..2966f93be 100644 --- a/src/sqlancer/clickhouse/ClickHouseVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseVisitor.java @@ -3,6 +3,7 @@ import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; @@ -58,8 +59,12 @@ default void visit(ClickHouseExpression.ClickHousePostfixText op) { void visit(ClickHouseAggregate aggregate); + void visit(ClickHouseBinaryFunctionOperation func); + default void visit(ClickHouseExpression expr) { - if (expr instanceof ClickHouseBinaryComparisonOperation) { + if (expr instanceof ClickHouseBinaryFunctionOperation) { + visit((ClickHouseBinaryFunctionOperation) expr); + } else if (expr instanceof ClickHouseBinaryComparisonOperation) { visit((ClickHouseBinaryComparisonOperation) expr); } else if (expr instanceof ClickHouseBinaryLogicalOperation) { visit((ClickHouseBinaryLogicalOperation) expr); @@ -83,6 +88,8 @@ default void visit(ClickHouseExpression expr) { visit((ClickHouseAggregate) expr); } else if (expr instanceof ClickHouseAliasOperation) { visit((ClickHouseAliasOperation) expr); + } else if (expr instanceof ClickHouseExpression.ClickHouseJoinOnClause) { + visit((ClickHouseExpression.ClickHouseJoinOnClause) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java index 172554b77..e868f82ca 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java @@ -4,7 +4,8 @@ import java.util.List; import java.util.stream.Collectors; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseSchema; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java new file mode 100644 index 000000000..fc0f8504a --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java @@ -0,0 +1,65 @@ +package sqlancer.clickhouse.ast; + +import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; + +public class ClickHouseBinaryArithmeticOperation extends ClickHouseExpression + implements BinaryOperation { + + public enum ClickHouseBinaryArithmeticOperator { + ADD("+"), // + MINUS("-"), // + MULT("*"), // + DIV("/"), // + MODULO("%"); // + + String textRepresentation; + + ClickHouseBinaryArithmeticOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static ClickHouseBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + + private final ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator operation; + private final ClickHouseExpression left; + private final ClickHouseExpression right; + + public ClickHouseBinaryArithmeticOperation(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator operation) { + this.left = left; + this.right = right; + this.operation = operation; + } + + public ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator getOperator() { + return operation; + } + + @Override + public ClickHouseExpression getLeft() { + return left; + } + + @Override + public ClickHouseExpression getRight() { + return right; + } + + @Override + public String getOperatorRepresentation() { + return operation.getTextRepresentation(); + } + + public static ClickHouseBinaryArithmeticOperation create(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator op) { + return new ClickHouseBinaryArithmeticOperation(left, right, op); + } +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java index 6696a3ee3..75dc259e8 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java @@ -1,8 +1,10 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.visitor.BinaryOperation; public class ClickHouseBinaryComparisonOperation extends ClickHouseExpression @@ -75,7 +77,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { } else if (lessThan.asInt() >= 1) { return lessThan; } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } } @@ -95,7 +97,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { && equals.getDataType() == ClickHouseDataType.UInt32 && equals.getDataType() == ClickHouseDataType.Int64 && equals.getDataType() == ClickHouseDataType.UInt64 && equals.asInt() == 1) { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } else { ClickHouseConstant applyLess = left.applyLess(right); if (applyLess == null) { @@ -121,7 +123,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { && lessThan.getDataType() == ClickHouseDataType.UInt32 && lessThan.getDataType() == ClickHouseDataType.Int64 && lessThan.getDataType() == ClickHouseDataType.UInt64 && lessThan.asInt() >= 1) { - return ClickHouseConstant.createTrue(); + return ClickHouseCreateConstant.createTrue(); } else { ClickHouseConstant applyLess = left.applyLess(right); if (applyLess == null) { @@ -146,14 +148,14 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { return null; } if (left.isNull() || right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { ClickHouseConstant applyEquals = left.applyEquals(right); if (applyEquals == null) { return null; } boolean equals = applyEquals.asInt() == 1; - return ClickHouseConstant.createBoolean(!equals); + return ClickHouseCreateConstant.createBoolean(!equals); } } @@ -165,7 +167,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { return null; } if (left.isNull() || right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } ClickHouseConstant leftStr = ClickHouseCast.castToText(left); ClickHouseConstant rightStr = ClickHouseCast.castToText(right); @@ -173,7 +175,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { return null; } boolean val = LikeImplementationHelper.match(leftStr.asString(), rightStr.asString(), 0, 0, false); - return ClickHouseConstant.createBoolean(val); + return ClickHouseCreateConstant.createBoolean(val); } }; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java new file mode 100644 index 000000000..65bf4c79e --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java @@ -0,0 +1,57 @@ +package sqlancer.clickhouse.ast; + +import sqlancer.Randomly; + +public class ClickHouseBinaryFunctionOperation extends ClickHouseExpression { + + public enum ClickHouseBinaryFunctionOperator { + INT_DIV("intDiv"), GCD("gcd"), LCM("lcm"), MAX2("max2"), MIN2("min2"), POW("pow"); + + String textRepresentation; + + ClickHouseBinaryFunctionOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static ClickHouseBinaryFunctionOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + + private final ClickHouseBinaryFunctionOperator operation; + private final ClickHouseExpression left; + private final ClickHouseExpression right; + + public ClickHouseBinaryFunctionOperation(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryFunctionOperator operation) { + this.left = left; + this.right = right; + this.operation = operation; + } + + public ClickHouseBinaryFunctionOperator getOperator() { + return operation; + } + + public ClickHouseExpression getLeft() { + return left; + } + + public ClickHouseExpression getRight() { + return right; + } + + public String getOperatorRepresentation() { + return operation.getTextRepresentation(); + } + + public static ClickHouseBinaryFunctionOperation create(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryFunctionOperator op) { + return new ClickHouseBinaryFunctionOperation(left, right, op); + } + +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java index bbd37e865..019fad752 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.ast; import sqlancer.Randomly; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; public class ClickHouseBinaryLogicalOperation extends ClickHouseExpression { @@ -14,22 +15,22 @@ public enum ClickHouseBinaryLogicalOperator { @Override public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { if (left.isNull() && right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else if (left.isNull()) { if (right.asBooleanNotNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } else if (right.isNull()) { if (left.asBooleanNotNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } else { - return left.asBooleanNotNull() && right.asBooleanNotNull() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); + return left.asBooleanNotNull() && right.asBooleanNotNull() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); } } }, @@ -37,13 +38,13 @@ public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant righ @Override public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { if (!left.isNull() && left.asBooleanNotNull()) { - return ClickHouseConstant.createTrue(); + return ClickHouseCreateConstant.createTrue(); } else if (!right.isNull() && right.asBooleanNotNull()) { - return ClickHouseConstant.createTrue(); + return ClickHouseCreateConstant.createTrue(); } else if (left.isNull() || right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } }; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCast.java b/src/sqlancer/clickhouse/ast/ClickHouseCast.java index dc35f1fc7..09c946717 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCast.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCast.java @@ -6,7 +6,9 @@ import java.util.Optional; import java.util.regex.Pattern; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; public final class ClickHouseCast extends ClickHouseExpression { @@ -50,18 +52,18 @@ public static Optional isTrue(ClickHouseConstant value) { public static ClickHouseConstant castToInt(ClickHouseConstant cons) { switch (cons.getDataType()) { case Nothing: - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); case Int32: return cons; case Float64: - return ClickHouseConstant.createInt32Constant((long) cons.asDouble()); + return ClickHouseCreateConstant.createInt32Constant((long) cons.asDouble()); case String: String asString = cons.asString(); while (startsWithWhitespace(asString)) { asString = asString.substring(1); } if (!asString.isEmpty() && unprintAbleCharThatLetsBecomeNumberZero(asString)) { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } for (int i = asString.length(); i >= 0; i--) { try { @@ -79,13 +81,13 @@ public static ClickHouseConstant castToInt(ClickHouseConstant cons) { result = Long.MAX_VALUE; } } - return ClickHouseConstant.createInt32Constant(result); + return ClickHouseCreateConstant.createInt32Constant(result); } } catch (Exception e) { } } - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); default: throw new AssertionError(); } @@ -95,7 +97,7 @@ public static ClickHouseConstant castToInt(ClickHouseConstant cons) { public static ClickHouseConstant castToReal(ClickHouseConstant cons) { ClickHouseConstant numericValue = castToNumeric(cons); if (numericValue.getDataType() == ClickHouseDataType.Int32) { - return ClickHouseConstant.createFloat64Constant(numericValue.asInt()); + return ClickHouseCreateConstant.createFloat64Constant(numericValue.asInt()); } else { return numericValue; } @@ -120,7 +122,7 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool boolean noNumIsRealZero, boolean convertIntToReal) throws AssertionError { switch (value.getDataType()) { case Nothing: - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); case Int32: case Float64: return value; @@ -130,11 +132,11 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool asString = asString.substring(1); } if (!asString.isEmpty() && unprintAbleCharThatLetsBecomeNumberZero(asString)) { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } if (asString.toLowerCase().startsWith("-infinity") || asString.toLowerCase().startsWith("infinity") || asString.startsWith("NaN")) { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } for (int i = asString.length(); i >= 0; i--) { try { @@ -151,17 +153,17 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool boolean isInteger = !isFloatingPointNumber && first.compareTo(second) == 0; if (doubleShouldBeConvertedToInt || isInteger && !convertIntToReal) { // see https://www.sqlite.org/src/tktview/afdc5a29dc - return ClickHouseConstant.createInt32Constant(first.longValue()); + return ClickHouseCreateConstant.createInt32Constant(first.longValue()); } else { - return ClickHouseConstant.createFloat64Constant(d); + return ClickHouseCreateConstant.createFloat64Constant(d); } } catch (Exception e) { } } if (noNumIsRealZero) { - return ClickHouseConstant.createFloat64Constant(0.0); + return ClickHouseCreateConstant.createFloat64Constant(0.0); } else { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } default: throw new AssertionError(value); @@ -222,14 +224,14 @@ public static ClickHouseConstant castToText(ClickHouseConstant cons) { } if (cons.getDataType() == ClickHouseDataType.Float64) { if (cons.asDouble() == Double.POSITIVE_INFINITY) { - return ClickHouseConstant.createStringConstant("Inf"); + return ClickHouseCreateConstant.createStringConstant("Inf"); } else if (cons.asDouble() == Double.NEGATIVE_INFINITY) { - return ClickHouseConstant.createStringConstant("-Inf"); + return ClickHouseCreateConstant.createStringConstant("-Inf"); } return castRealToText(cons); } if (cons.getDataType() == ClickHouseDataType.Int32) { - return ClickHouseConstant.createStringConstant(String.valueOf(cons.asInt())); + return ClickHouseCreateConstant.createStringConstant(String.valueOf(cons.asInt())); } return null; } @@ -237,7 +239,7 @@ public static ClickHouseConstant castToText(ClickHouseConstant cons) { private static synchronized ClickHouseConstant castRealToText(ClickHouseConstant cons) throws AssertionError { try (Statement s = castDatabase.createStatement()) { String castResult = s.executeQuery("SELECT CAST(" + cons.asDouble() + " AS TEXT)").getString(1); - return ClickHouseConstant.createStringConstant(castResult); + return ClickHouseCreateConstant.createStringConstant(castResult); } catch (Exception e) { throw new AssertionError(e); } @@ -246,9 +248,9 @@ private static synchronized ClickHouseConstant castRealToText(ClickHouseConstant public static ClickHouseConstant asBoolean(ClickHouseConstant val) { Optional boolVal = isTrue(val); if (boolVal.isPresent()) { - return ClickHouseConstant.createBoolean(boolVal.get()); + return ClickHouseCreateConstant.createBoolean(boolVal.get()); } else { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java index 490a77d37..a905b4e49 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; public class ClickHouseCastOperation extends ClickHouseExpression { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java index 796eb8333..6d75cf5fa 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java @@ -5,17 +5,30 @@ public class ClickHouseColumnReference extends ClickHouseExpression { private final ClickHouseColumn column; + private final String columnAlias; + private final String tableAlias; - public ClickHouseColumnReference(ClickHouseColumn column) { + public ClickHouseColumnReference(ClickHouseColumn column, String columnAlias, String tableAlias) { this.column = column; + this.columnAlias = columnAlias; + this.tableAlias = tableAlias; } - public static ClickHouseColumnReference create(ClickHouseColumn column) { - return new ClickHouseColumnReference(column); + public ClickHouseColumnReference(ClickHouseAliasOperation alias) { + this.column = new ClickHouseColumn(alias.getAlias(), null, true, false, null); + this.columnAlias = null; + this.tableAlias = null; } public ClickHouseColumn getColumn() { return column; } + public String getAlias() { + return columnAlias; + } + + public String getTableAlias() { + return tableAlias; + } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java index 048c14296..b38ad7e89 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java @@ -1,1899 +1,13 @@ package sqlancer.clickhouse.ast; -import java.math.BigInteger; +import com.clickhouse.client.ClickHouseDataType; -import ru.yandex.clickhouse.domain.ClickHouseDataType; -import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; public abstract class ClickHouseConstant extends ClickHouseExpression { - public static class ClickHouseNullConstant extends ClickHouseConstant { - - @Override - public String toString() { - return "NULL"; - } - - @Override - public boolean isNull() { - return true; - } - - @Override - public boolean asBooleanNotNull() { - throw new AssertionError(); - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Nothing; - } - - @Override - public boolean compareInternal(Object value) { - return false; - } - - @Override - public ClickHouseConstant applyEquals(ClickHouseConstant right) { - return ClickHouseConstant.createNullConstant(); - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - return ClickHouseConstant.createNullConstant(); - } - - @Override - public Object getValue() { - return null; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - return null; - } - } - - public static class ClickHouseUInt8Constant extends ClickHouseConstant { - - private final int value; - - public ClickHouseUInt8Constant(int value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt8; - } - - @Override - public boolean compareInternal(Object val) { - return value == (int) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case UInt128: - return ClickHouseConstant.createUInt128Constant(BigInteger.valueOf(value)); - case Int128: - return ClickHouseConstant.createInt128Constant(BigInteger.valueOf(value)); - case UInt256: - return ClickHouseConstant.createUInt256Constant(BigInteger.valueOf(value)); - case Int256: - return ClickHouseConstant.createInt256Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant((double) value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt8Constant extends ClickHouseConstant { - - private final int value; - - public ClickHouseInt8Constant(int value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int8; - } - - @Override - public boolean compareInternal(Object val) { - return value == (int) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt16Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseUInt16Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt16; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt16Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseInt16Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int16; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt32Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseUInt32Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt32; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt32Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseInt32Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int32; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt64Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseUInt64Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt64; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt64Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseInt64Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int64; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt128Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseUInt128Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt128; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case UInt128: - return ClickHouseConstant.createUInt128Constant(value); - case Int128: - return ClickHouseConstant.createInt128Constant(value); - case UInt256: - return ClickHouseConstant.createUInt256Constant(value); - case Int256: - return ClickHouseConstant.createInt256Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt128Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseInt128Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int128; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case UInt128: - return ClickHouseConstant.createUInt128Constant(value); - case Int128: - return ClickHouseConstant.createInt128Constant(value); - case UInt256: - return ClickHouseConstant.createUInt256Constant(value); - case Int256: - return ClickHouseConstant.createInt256Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt256Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseUInt256Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt256; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case UInt128: - return ClickHouseConstant.createUInt128Constant(value); - case Int128: - return ClickHouseConstant.createInt128Constant(value); - case UInt256: - return ClickHouseConstant.createUInt256Constant(value); - case Int256: - return ClickHouseConstant.createInt256Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt256Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseInt256Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int256; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case UInt128: - return ClickHouseConstant.createUInt128Constant(value); - case Int128: - return ClickHouseConstant.createInt128Constant(value); - case UInt256: - return ClickHouseConstant.createUInt256Constant(value); - case Int256: - return ClickHouseConstant.createInt256Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseFloat32Constant extends ClickHouseConstant { - - private final float value; - - public ClickHouseFloat32Constant(float value) { - this.value = value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public String toString() { - if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; - } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; - } - return String.valueOf(value); - } - - @Override - public boolean compareInternal(Object val) { - return Float.compare(value, (float) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asDouble() < right.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - ClickHouseConstant converted = right.cast(ClickHouseDataType.Float32); - return this.asDouble() < converted.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - - @Override - public boolean asBooleanNotNull() { - return Float.compare(value, (float) 0) == 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Float32; - } - - @Override - public double asDouble() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant((long) value); - case Int8: - return ClickHouseConstant.createInt8Constant((long) value); - case UInt16: - return ClickHouseConstant.createUInt16Constant((long) value); - case Int16: - return ClickHouseConstant.createInt16Constant((long) value); - case UInt32: - return ClickHouseConstant.createUInt32Constant((long) value); - case Int32: - return ClickHouseConstant.createInt32Constant((long) value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf((long) value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf((long) value)); - case Float32: - return ClickHouseConstant.createFloat32Constant(value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseFloat64Constant extends ClickHouseConstant { - - private final double value; - - public ClickHouseFloat64Constant(double value) { - this.value = value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public String toString() { - if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; - } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; - } - return String.valueOf(value); - } - - @Override - public boolean compareInternal(Object val) { - return Double.compare(value, (double) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asDouble() < right.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - ClickHouseConstant converted = right.cast(ClickHouseDataType.Float64); - return this.asDouble() < converted.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - - @Override - public boolean asBooleanNotNull() { - return Double.compare(value, 0.0) == 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Float64; - } - - @Override - public double asDouble() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant((long) value); - case Int8: - return ClickHouseConstant.createInt8Constant((long) value); - case UInt16: - return ClickHouseConstant.createUInt16Constant((long) value); - case Int16: - return ClickHouseConstant.createInt16Constant((long) value); - case UInt32: - return ClickHouseConstant.createUInt32Constant((long) value); - case Int32: - return ClickHouseConstant.createInt32Constant((long) value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf((long) value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf((long) value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseStringConstant extends ClickHouseConstant { - - private final String value; - - public ClickHouseStringConstant(String value) { - this.value = value; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public String toString() { - return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; - } - - @Override - public boolean asBooleanNotNull() { - return value.length() > 0; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((String) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asString().compareTo(right.asString()) <= 0 ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.String; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(Integer.parseInt(value)); - case Int8: - return ClickHouseConstant.createInt8Constant(Integer.parseInt(value)); - case UInt16: - return ClickHouseConstant.createUInt16Constant(Integer.parseInt(value)); - case Int16: - return ClickHouseConstant.createInt16Constant(Integer.parseInt(value)); - case UInt32: - return ClickHouseConstant.createUInt32Constant(Integer.parseInt(value)); - case Int32: - return ClickHouseConstant.createInt32Constant(Integer.parseInt(value)); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) Float.parseFloat(value)); - case Float64: - return ClickHouseConstant.createFloat64Constant((double) Double.parseDouble(value)); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static ClickHouseConstant createStringConstant(String text) { - return new ClickHouseStringConstant(text); - } - - public static ClickHouseConstant createFloat64Constant(double val) { - return new ClickHouseFloat64Constant(val); - } - - public static ClickHouseConstant createFloat32Constant(float val) { - return new ClickHouseFloat32Constant(val); - } - - public static ClickHouseConstant createIntConstant(ClickHouseDataType type, long val) { - switch (type) { - case IntervalYear: - break; - case IntervalQuarter: - break; - case IntervalMonth: - break; - case IntervalWeek: - break; - case IntervalDay: - break; - case IntervalHour: - break; - case IntervalMinute: - break; - case IntervalSecond: - break; - case UInt256: - return createUInt256Constant(BigInteger.valueOf(val)); - case UInt128: - return createUInt128Constant(BigInteger.valueOf(val)); - case UInt64: - return createUInt64Constant(BigInteger.valueOf(val)); - case UInt32: - return createUInt32Constant(val); - case UInt16: - return createUInt16Constant(val); - case UInt8: - return createUInt8Constant(val); - case Int256: - return createInt256Constant(BigInteger.valueOf(val)); - case Int128: - return createInt256Constant(BigInteger.valueOf(val)); - case Int64: - return createInt64Constant(BigInteger.valueOf(val)); - case Int32: - return createInt32Constant(val); - case Int16: - return createInt16Constant(val); - case Int8: - return createInt8Constant(val); - case Date: - break; - case DateTime: - break; - case Enum8: - break; - case Enum16: - break; - case Float32: - break; - case Float64: - break; - case Decimal32: - break; - case Decimal64: - break; - case Decimal128: - break; - case Decimal: - break; - case UUID: - break; - case String: - break; - case FixedString: - break; - case Nothing: - break; - case Nested: - break; - case Tuple: - break; - case Array: - break; - case AggregateFunction: - break; - case Unknown: - break; - default: - break; - } - throw new AssertionError(type); - } - - public static ClickHouseConstant createInt256Constant(BigInteger val) { - return new ClickHouseInt256Constant(val); - } - - public static ClickHouseConstant createUInt256Constant(BigInteger val) { - return new ClickHouseUInt256Constant(val); - } - - public static ClickHouseConstant createInt128Constant(BigInteger val) { - return new ClickHouseInt128Constant(val); - } - - public static ClickHouseConstant createUInt128Constant(BigInteger val) { - return new ClickHouseUInt128Constant(val); - } - - public static ClickHouseConstant createInt64Constant(BigInteger val) { - return new ClickHouseInt64Constant(val); - } - - public static ClickHouseConstant createUInt64Constant(BigInteger val) { - return new ClickHouseUInt64Constant(val); - } - - public static ClickHouseConstant createInt32Constant(long val) { - return new ClickHouseInt32Constant(val); - } - - public static ClickHouseConstant createUInt32Constant(long val) { - return new ClickHouseUInt32Constant(val); - } - - public static ClickHouseConstant createUInt16Constant(long val) { - return new ClickHouseUInt16Constant(val); - } - - public static ClickHouseConstant createInt16Constant(long val) { - return new ClickHouseInt16Constant(val); - } - - public static ClickHouseConstant createUInt8Constant(long val) { - return new ClickHouseUInt8Constant((int) val); - } - - public static ClickHouseConstant createInt8Constant(long val) { - return new ClickHouseInt8Constant((int) val); - } - public abstract boolean isNull(); - public static ClickHouseConstant createNullConstant() { - return new ClickHouseNullConstant(); - } - - public static ClickHouseConstant createTrue() { - return new ClickHouseUInt8Constant(1); - } - - public static ClickHouseConstant createFalse() { - return new ClickHouseUInt8Constant(0); - } - - public static ClickHouseConstant createBoolean(boolean val) { - return val ? createTrue() : createFalse(); - } - public abstract ClickHouseConstant cast(ClickHouseDataType type); public abstract boolean asBooleanNotNull(); @@ -1904,8 +18,8 @@ public static ClickHouseConstant createBoolean(boolean val) { public ClickHouseConstant applyEquals(ClickHouseConstant right) { if (this.getDataType() == right.getDataType()) { - return this.compareInternal(right.getValue()) ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); + return this.compareInternal(right.getValue()) ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); } else { ClickHouseConstant converted = right.cast(this.getDataType()); return this.applyEquals(converted); diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index 161fd0206..34eda72a5 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -1,6 +1,6 @@ package sqlancer.clickhouse.ast; -import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.common.visitor.BinaryOperation; import sqlancer.common.visitor.UnaryOperation; public abstract class ClickHouseExpression { @@ -35,41 +35,67 @@ public ClickHouseExpression getExpression() { } } + public static class ClickHouseJoinOnClause extends ClickHouseExpression + implements BinaryOperation { + private final ClickHouseExpression left; + private final ClickHouseExpression right; + + public ClickHouseJoinOnClause(ClickHouseExpression left, ClickHouseExpression right) { + this.left = left; + this.right = right; + } + + @Override + public final ClickHouseExpression getLeft() { + return this.left; + } + + @Override + public final ClickHouseExpression getRight() { + return this.right; + } + + @Override + public String getOperatorRepresentation() { + return "="; + } + } + public static class ClickHouseJoin extends ClickHouseExpression { // TODO: support ANY, ALL, ASOF modifiers public enum JoinType { - INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, NATURAL, LEFT_SEMI, RIGHT_SEMI, LEFT_ANTI, RIGHT_ANTI; + INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, RIGHT_SEMI, LEFT_ANTI, RIGHT_ANTI; } - private final ClickHouseSchema.ClickHouseTable leftTable; - private final ClickHouseSchema.ClickHouseTable rightTable; - private ClickHouseExpression onClause; + private final ClickHouseTableReference leftTable; + private final ClickHouseTableReference rightTable; + private ClickHouseJoinOnClause onClause; private final ClickHouseJoin.JoinType type; - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable leftTable, ClickHouseSchema.ClickHouseTable rightTable, - ClickHouseJoin.JoinType type, ClickHouseExpression onClause) { + public ClickHouseJoin(ClickHouseTableReference leftTable, ClickHouseTableReference rightTable, + ClickHouseJoin.JoinType type, ClickHouseJoinOnClause onClause) { this.leftTable = leftTable; this.rightTable = rightTable; this.onClause = onClause; this.type = type; } - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable leftTable, ClickHouseSchema.ClickHouseTable rightTable, + public ClickHouseJoin(ClickHouseTableReference leftTable, ClickHouseTableReference rightTable, ClickHouseJoin.JoinType type) { this.leftTable = leftTable; this.rightTable = rightTable; - if (type != ClickHouseJoin.JoinType.NATURAL) { + if (type != ClickHouseJoin.JoinType.CROSS) { throw new AssertionError(); } this.onClause = null; this.type = type; } - public ClickHouseSchema.ClickHouseTable getLeftTable() { + public ClickHouseTableReference getLeftTable() { return leftTable; } - public ClickHouseSchema.ClickHouseTable getTable() { + public ClickHouseTableReference getRightTable() { return rightTable; } @@ -81,7 +107,7 @@ public ClickHouseJoin.JoinType getType() { return type; } - public void setOnClause(ClickHouseExpression onClause) { + public void setOnClause(ClickHouseJoinOnClause onClause) { this.onClause = onClause; } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java index 7a72f2919..24d489982 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java @@ -6,7 +6,7 @@ public class ClickHouseSelect extends ClickHouseExpression { private ClickHouseSelect.SelectType fromOptions = ClickHouseSelect.SelectType.ALL; - private List fromList = Collections.emptyList(); + private ClickHouseExpression fromClause; private ClickHouseExpression whereClause; private List groupByClause = Collections.emptyList(); private ClickHouseExpression limitClause; @@ -24,8 +24,12 @@ public void setSelectType(ClickHouseSelect.SelectType fromOptions) { this.setFromOptions(fromOptions); } - public void setFromTables(List fromTables) { - this.setFromList(fromTables); + public void setFromClause(ClickHouseExpression fromClause) { + this.fromClause = fromClause; + } + + public ClickHouseExpression getFromClause() { + return fromClause; } public ClickHouseSelect.SelectType getFromOptions() { @@ -36,14 +40,6 @@ public void setFromOptions(ClickHouseSelect.SelectType fromOptions) { this.fromOptions = fromOptions; } - public List getFromList() { - return fromList; - } - - public void setFromList(List fromList) { - this.fromList = fromList; - } - public ClickHouseExpression getWhereClause() { return whereClause; } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java b/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java index 415dfca22..406ffb15c 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java @@ -1,17 +1,34 @@ package sqlancer.clickhouse.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; public class ClickHouseTableReference extends ClickHouseExpression { private final ClickHouseTable table; + private final String alias; - public ClickHouseTableReference(ClickHouseTable table) { + public ClickHouseTableReference(ClickHouseTable table, String alias) { this.table = table; + this.alias = alias; } public ClickHouseTable getTable() { return table; } + public String getTableName() { + return (alias == null) ? table.getName() : alias; + } + + public String getAlias() { + return alias; + } + + public List getColumnReferences() { + return this.table.getColumns().stream().map(c -> c.asColumnReference(this.alias)).collect(Collectors.toList()); + } + } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java new file mode 100644 index 000000000..30707354f --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java @@ -0,0 +1,52 @@ +package sqlancer.clickhouse.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.visitor.UnaryOperation; + +public class ClickHouseUnaryFunctionOperation extends ClickHouseExpression + implements UnaryOperation { + private final ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator operator; + private final ClickHouseExpression expression; + + public ClickHouseUnaryFunctionOperation(ClickHouseExpression expression, ClickHouseUnaryFunctionOperator operator) { + this.operator = operator; + this.expression = expression; + } + + public enum ClickHouseUnaryFunctionOperator implements Operator { + EXP("exp"), SQRT("sqrt"), ERF("erf"), SIN("sin"), COS("cos"), TAN("tan"), SIGN("sign"), RADIANS("radians"), + LOG("log"), ABS("abs"); + + private String textRepresentation; + + ClickHouseUnaryFunctionOperator(String text) { + this.textRepresentation = text; + } + + public static ClickHouseUnaryFunctionOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + + @Override + public ClickHouseExpression getExpression() { + return expression; + } + + @Override + public String getOperatorRepresentation() { + return operator.getTextRepresentation(); + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.PREFIX; + } + +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java index 553f85070..444818413 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.ast; import sqlancer.Randomly; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.visitor.UnaryOperation; @@ -86,7 +87,7 @@ public ClickHouseConstant getExpectedValue() { if (negate) { val = !val; } - return ClickHouseConstant.createInt32Constant(val ? 1 : 0); + return ClickHouseCreateConstant.createInt32Constant(val ? 1 : 0); } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java index 972136e64..15ab8fb36 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java @@ -1,7 +1,9 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.visitor.UnaryOperation; @@ -20,10 +22,10 @@ public enum ClickHouseUnaryPrefixOperator implements Operator { @Override public ClickHouseConstant apply(ClickHouseConstant constant) { if (constant.getDataType() == ClickHouseDataType.Nothing) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return constant.asBooleanNotNull() ? ClickHouseConstant.createFalse() - : ClickHouseConstant.createTrue(); + return constant.asBooleanNotNull() ? ClickHouseCreateConstant.createFalse() + : ClickHouseCreateConstant.createTrue(); } } }, @@ -31,7 +33,7 @@ public ClickHouseConstant apply(ClickHouseConstant constant) { @Override public ClickHouseConstant apply(ClickHouseConstant constant) { if (constant.getDataType() == ClickHouseDataType.Int32) { - return ClickHouseConstant.createInt32Constant(-constant.asInt()); + return ClickHouseCreateConstant.createInt32Constant(-constant.asInt()); } throw new AssertionError(constant); } diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java new file mode 100644 index 000000000..d5164162c --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java @@ -0,0 +1,142 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseBooleanConstant extends ClickHouseConstant { + + private final boolean value; + + public ClickHouseBooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String toString() { + return value ? "true" : "false"; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Bool; + } + + @Override + public boolean compareInternal(Object val) { + if (val instanceof Boolean) { + return value == ((Boolean) val).booleanValue(); + } else { + return value == ((int) val != 0); + } + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value ? 1 : 0; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(asInt()); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(asInt()); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(asInt()); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(asInt()); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(asInt()); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(asInt()); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(asInt())); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(asInt())); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(BigInteger.valueOf(asInt())); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(BigInteger.valueOf(asInt())); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(BigInteger.valueOf(asInt())); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(BigInteger.valueOf(asInt())); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) asInt()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant((double) asInt()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(asInt() != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java new file mode 100644 index 000000000..dc87619a8 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java @@ -0,0 +1,162 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; +import sqlancer.clickhouse.ast.ClickHouseExpression; + +public abstract class ClickHouseCreateConstant extends ClickHouseExpression { + + public static ClickHouseConstant createStringConstant(String text) { + return new ClickHouseStringConstant(text); + } + + public static ClickHouseConstant createFloat64Constant(double val) { + return new ClickHouseFloat64Constant(val); + } + + public static ClickHouseConstant createFloat32Constant(float val) { + return new ClickHouseFloat32Constant(val); + } + + public static ClickHouseConstant createInt256Constant(BigInteger val) { + return new ClickHouseInt256Constant(val); + } + + public static ClickHouseConstant createUInt256Constant(BigInteger val) { + return new ClickHouseUInt256Constant(val); + } + + public static ClickHouseConstant createInt128Constant(BigInteger val) { + return new ClickHouseInt128Constant(val); + } + + public static ClickHouseConstant createUInt128Constant(BigInteger val) { + return new ClickHouseUInt128Constant(val); + } + + public static ClickHouseConstant createInt64Constant(BigInteger val) { + return new ClickHouseInt64Constant(val); + } + + public static ClickHouseConstant createUInt64Constant(BigInteger val) { + return new ClickHouseUInt64Constant(val); + } + + public static ClickHouseConstant createInt32Constant(long val) { + return new ClickHouseInt32Constant(val); + } + + public static ClickHouseConstant createUInt32Constant(long val) { + return new ClickHouseUInt32Constant(val); + } + + public static ClickHouseConstant createUInt16Constant(long val) { + return new ClickHouseUInt16Constant(val); + } + + public static ClickHouseConstant createInt16Constant(long val) { + return new ClickHouseInt16Constant(val); + } + + public static ClickHouseConstant createUInt8Constant(long val) { + return new ClickHouseUInt8Constant((int) val); + } + + public static ClickHouseConstant createInt8Constant(long val) { + return new ClickHouseInt8Constant((int) val); + } + + public static ClickHouseConstant createBooleanConstant(Boolean b) { + return new ClickHouseBooleanConstant(b); + } + + public static ClickHouseConstant createNullConstant() { + return new ClickHouseNullConstant(); + } + + public static ClickHouseConstant createTrue() { + return new ClickHouseBooleanConstant(true); + } + + public static ClickHouseConstant createFalse() { + return new ClickHouseBooleanConstant(false); + } + + public static ClickHouseConstant createBoolean(boolean val) { + return val ? createTrue() : createFalse(); + } + + public static ClickHouseConstant createIntConstant(ClickHouseDataType type, long val) { + switch (type) { + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(BigInteger.valueOf(val)); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(BigInteger.valueOf(val)); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(val)); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(BigInteger.valueOf(val)); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(BigInteger.valueOf(val)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(val)); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + case String: + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } + +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java new file mode 100644 index 000000000..1385790cb --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java @@ -0,0 +1,136 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseFloat32Constant extends ClickHouseConstant { + + private final float value; + + public ClickHouseFloat32Constant(float value) { + this.value = value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + @Override + public boolean compareInternal(Object val) { + return Float.compare(value, (float) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asDouble() < right.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + ClickHouseConstant converted = right.cast(ClickHouseDataType.Float32); + return this.asDouble() < converted.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + + @Override + public boolean asBooleanNotNull() { + return Float.compare(value, (float) 0) == 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Float32; + } + + @Override + public double asDouble() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant((long) value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant((long) value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant((long) value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant((long) value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant((long) value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant((long) value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf((long) value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf((long) value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java new file mode 100644 index 000000000..9146faa39 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java @@ -0,0 +1,136 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseFloat64Constant extends ClickHouseConstant { + + private final double value; + + public ClickHouseFloat64Constant(double value) { + this.value = value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + @Override + public boolean compareInternal(Object val) { + return Double.compare(value, (double) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asDouble() < right.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + ClickHouseConstant converted = right.cast(ClickHouseDataType.Float64); + return this.asDouble() < converted.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + + @Override + public boolean asBooleanNotNull() { + return Double.compare(value, 0.0) == 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Float64; + } + + @Override + public double asDouble() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant((long) value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant((long) value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant((long) value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant((long) value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant((long) value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant((long) value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf((long) value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf((long) value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java new file mode 100644 index 000000000..c388cb638 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt128Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt128Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int128; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java new file mode 100644 index 000000000..1fde4dfb1 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt16Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseInt16Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int16; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java new file mode 100644 index 000000000..f40ea5297 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt256Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt256Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int256; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java new file mode 100644 index 000000000..408df18fc --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt32Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseInt32Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int32; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java new file mode 100644 index 000000000..cb5c4b5f8 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java @@ -0,0 +1,131 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt64Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt64Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int64; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java new file mode 100644 index 000000000..38ad4878c --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt8Constant extends ClickHouseConstant { + + private final int value; + + public ClickHouseInt8Constant(int value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int8; + } + + @Override + public boolean compareInternal(Object val) { + return value == (int) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java new file mode 100644 index 000000000..08370d560 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java @@ -0,0 +1,53 @@ +package sqlancer.clickhouse.ast.constant; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseNullConstant extends ClickHouseConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public boolean asBooleanNotNull() { + throw new AssertionError(); + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Nothing; + } + + @Override + public boolean compareInternal(Object value) { + return false; + } + + @Override + public ClickHouseConstant applyEquals(ClickHouseConstant right) { + return ClickHouseCreateConstant.createNullConstant(); + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + return ClickHouseCreateConstant.createNullConstant(); + } + + @Override + public Object getValue() { + return null; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + return null; + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java new file mode 100644 index 000000000..bde359f92 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java @@ -0,0 +1,125 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseStringConstant extends ClickHouseConstant { + + private final String value; + + public ClickHouseStringConstant(String value) { + this.value = value; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; + } + + @Override + public boolean asBooleanNotNull() { + return value.length() > 0; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((String) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asString().compareTo(right.asString()) <= 0 ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.String; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(Integer.parseInt(value)); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(Integer.parseInt(value)); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(Integer.parseInt(value)); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(Integer.parseInt(value)); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(Integer.parseInt(value)); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(Integer.parseInt(value)); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) Float.parseFloat(value)); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant((double) Double.parseDouble(value)); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value == "true"); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java new file mode 100644 index 000000000..5952c967c --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt128Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt128Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt128; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java new file mode 100644 index 000000000..c7ae84fdf --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt16Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseUInt16Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt16; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java new file mode 100644 index 000000000..95b333323 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt256Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt256Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt256; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java new file mode 100644 index 000000000..8e4f5fd09 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt32Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseUInt32Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt32; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java new file mode 100644 index 000000000..cd3363850 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java @@ -0,0 +1,131 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt64Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt64Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt64; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java new file mode 100644 index 000000000..de0d308c3 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java @@ -0,0 +1,138 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt8Constant extends ClickHouseConstant { + + private final int value; + + public ClickHouseUInt8Constant(int value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt8; + } + + @Override + public boolean compareInternal(Object val) { + return value == (int) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(BigInteger.valueOf(value)); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(BigInteger.valueOf(value)); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(BigInteger.valueOf(value)); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant((double) value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java index 612e807d5..deae2a9aa 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java @@ -29,7 +29,7 @@ public static List getTableRefs(List tableRefs = new ArrayList<>(); for (ClickHouseSchema.ClickHouseTable t : tables) { ClickHouseTableReference tableRef; - tableRef = new ClickHouseTableReference(t); + tableRef = new ClickHouseTableReference(t, null); tableRefs.add(tableRef); } return tableRefs; diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 6e86b28ec..cd8ad8ff3 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -4,22 +4,27 @@ import java.util.List; import java.util.stream.Collectors; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseBinaryArithmeticOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; -import sqlancer.clickhouse.ast.ClickHouseConstant; import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseTableReference; +import sqlancer.clickhouse.ast.ClickHouseUnaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.gen.TypedExpressionGenerator; public class ClickHouseExpressionGenerator @@ -28,12 +33,54 @@ public class ClickHouseExpressionGenerator private final ClickHouseGlobalState globalState; public boolean allowAggregateFunctions; + private final List columnRefs; + public ClickHouseExpressionGenerator(ClickHouseGlobalState globalState) { this.globalState = globalState; + this.columnRefs = new ArrayList<>(); + } + + public final void addColumns(List col) { + this.columnRefs.addAll(col); + } + + private enum ColumnLike { + UNARY_PREFIX, BINARY_ARITHMETIC, UNARY_FUNCTION, BINARY_FUNCTION } private enum Expression { - UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL + UNARY_PREFIX, BINARY_ARITHMETIC, UNARY_FUNCTION, BINARY_FUNCTION, BINARY_LOGICAL, BINARY_COMPARISON, + UNARY_POSTFIX + } + + public ClickHouseExpression generateExpressionWithColumns(List columns, int depth) { + if (columns.isEmpty() || depth > 0 && Randomly.getBoolean()) { + return generateConstant(null); + } + + if (depth >= 2 || Randomly.getBoolean()) { + return columns.get((int) Randomly.getNotCachedInteger(0, columns.size() - 1)); + } + + ColumnLike expr = Randomly.fromOptions(ColumnLike.values()); + switch (expr) { + case UNARY_PREFIX: + return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, depth + 1), + ClickHouseUnaryPrefixOperator.MINUS); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, depth + 1), + generateExpressionWithColumns(columns, depth + 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, depth + 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, depth + 1), + generateExpressionWithColumns(columns, depth + 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); + default: + throw new AssertionError(expr); + } } @Override @@ -66,42 +113,52 @@ protected ClickHouseExpression generateExpression(ClickHouseLancerDataType type, return new ClickHouseBinaryLogicalOperation(generateExpression(leftLeafType, depth + 1), generateExpression(rightLeafType, depth + 1), ClickHouseBinaryLogicalOperation.ClickHouseBinaryLogicalOperator.getRandom()); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(leftLeafType, depth + 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation(generateExpression(leftLeafType, depth + 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(leftLeafType, depth + 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); default: throw new AssertionError(expr); } } - protected ClickHouseExpression generateJoinClause(ClickHouseSchema.ClickHouseTable leftTable, - ClickHouseSchema.ClickHouseTable rightTable) { - ClickHouseExpression leftExpr = getColumnNameFromTable(leftTable); - ClickHouseExpression rightExpr = getColumnNameFromTable(rightTable); - return new ClickHouseBinaryComparisonOperation(leftExpr, rightExpr, - ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.EQUALS); + protected ClickHouseExpression.ClickHouseJoinOnClause generateJoinClause(ClickHouseTableReference leftTable, + ClickHouseTableReference rightTable) { + List leftColumns = leftTable.getColumnReferences(); + List rightColumns = rightTable.getColumnReferences(); + ClickHouseExpression leftExpr = generateExpressionWithColumns(leftColumns, 0); + ClickHouseExpression rightExpr = generateExpressionWithColumns(rightColumns, 0); + return new ClickHouseExpression.ClickHouseJoinOnClause(leftExpr, rightExpr); } @Override protected ClickHouseExpression generateColumn(ClickHouseLancerDataType type) { - if (columns.isEmpty()) { + if (columnRefs.isEmpty()) { return generateConstant(type); } - List filteredColumns = columns.stream() - .filter(c -> c.getType().getType().name().equals(type.getType().name())).collect(Collectors.toList()); - ClickHouseColumn column = filteredColumns.isEmpty() ? Randomly.fromList(columns) - : Randomly.fromList(filteredColumns); - return new ClickHouseColumnReference(column); + List filteredColumns = columnRefs.stream() + .filter(c -> c.getColumn().getType().getType().name().equals(type.getType().name())) + .collect(Collectors.toList()); + return filteredColumns.isEmpty() ? Randomly.fromList(columnRefs) : Randomly.fromList(filteredColumns); } protected ClickHouseExpression getColumnNameFromTable(ClickHouseSchema.ClickHouseTable table) { - if (columns.isEmpty()) { + if (columnRefs.isEmpty()) { return generateConstant(ClickHouseLancerDataType.getRandom()); } - List filteredColumns = columns.stream().filter(c -> c.getTable() == table) - .collect(Collectors.toList()); + List filteredColumns = columnRefs.stream() + .filter(c -> c.getColumn().getTable() == table).collect(Collectors.toList()); if (filteredColumns.isEmpty()) { return generateConstant(ClickHouseLancerDataType.getRandom()); } - ClickHouseColumn column = Randomly.fromList(filteredColumns); - return new ClickHouseColumnReference(column); + return Randomly.fromList(filteredColumns); } @Override @@ -109,29 +166,29 @@ protected ClickHouseLancerDataType getRandomType() { return ClickHouseLancerDataType.getRandom(); } - public List getRandomJoinClauses( + public List getRandomJoinClauses(ClickHouseTableReference left, List tables) { List joinStatements = new ArrayList<>(); if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } - if (Randomly.getBoolean() && tables.size() > 1) { + List leftTables = new ArrayList<>(); + leftTables.add(left); + if (Randomly.getBoolean() && !tables.isEmpty()) { int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); for (int i = 0; i < nrJoinClauses; i++) { - ClickHouseSchema.ClickHouseTable leftTable = Randomly.fromList(tables); - ClickHouseSchema.ClickHouseTable rightTable = Randomly.fromList(tables); - ClickHouseExpression joinClause = generateJoinClause(leftTable, rightTable); - ClickHouseExpression.ClickHouseJoin.JoinType options; - options = Randomly.fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); - if (options == ClickHouseExpression.ClickHouseJoin.JoinType.NATURAL) { - // NATURAL joins do not have an ON clause - joinClause = null; - } + ClickHouseTableReference leftTable = leftTables + .get((int) Randomly.getNotCachedInteger(0, leftTables.size() - 1)); + ClickHouseTableReference rightTable = new ClickHouseTableReference(Randomly.fromList(tables), + "right_" + i); + ClickHouseExpression.ClickHouseJoinOnClause joinClause = generateJoinClause(leftTable, rightTable); + ClickHouseExpression.ClickHouseJoin.JoinType options = Randomly + .fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(leftTable, rightTable, options, joinClause); joinStatements.add(j); + leftTables.add(rightTable); } - } return joinStatements; } @@ -142,7 +199,8 @@ protected boolean canGenerateColumnOfType(ClickHouseLancerDataType type) { } @Override - public ClickHouseExpression generateConstant(ClickHouseLancerDataType type) { + public ClickHouseExpression generateConstant(ClickHouseLancerDataType genType) { + ClickHouseLancerDataType type = (genType == null) ? ClickHouseLancerDataType.getRandom() : genType; switch (type.getType()) { case Int8: case UInt8: @@ -152,13 +210,13 @@ public ClickHouseExpression generateConstant(ClickHouseLancerDataType type) { case UInt32: case Int64: case UInt64: - return ClickHouseConstant.createIntConstant(type.getType(), globalState.getRandomly().getInteger()); + return ClickHouseCreateConstant.createIntConstant(type.getType(), globalState.getRandomly().getInteger()); case Float32: - return ClickHouseConstant.createFloat32Constant((float) globalState.getRandomly().getDouble()); + return ClickHouseCreateConstant.createFloat32Constant((float) globalState.getRandomly().getDouble()); case Float64: - return ClickHouseConstant.createFloat64Constant(globalState.getRandomly().getDouble()); + return ClickHouseCreateConstant.createFloat64Constant(globalState.getRandomly().getDouble()); case String: - return ClickHouseConstant.createStringConstant(globalState.getRandomly().getString()); + return ClickHouseCreateConstant.createStringConstant(globalState.getRandomly().getString()); default: throw new AssertionError(); } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index 5b62e88bc..f3e5baecc 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -2,8 +2,10 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; + +import com.clickhouse.client.ClickHouseDataType; -import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; @@ -79,8 +81,8 @@ public void start() { if (engine == ClickHouseEngine.MergeTree) { if (Randomly.getBoolean()) { sb.append(" ORDER BY "); - ClickHouseExpression expr = gen - .generateExpression(ClickHouseSchema.ClickHouseLancerDataType.getRandom()); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); sb.append(ClickHouseToStringVisitor.asString(expr)); } else { sb.append(" ORDER BY tuple() "); @@ -88,14 +90,14 @@ public void start() { if (Randomly.getBoolean()) { sb.append(" PARTITION BY "); - ClickHouseExpression expr = gen - .generateExpression(ClickHouseSchema.ClickHouseLancerDataType.getRandom()); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); sb.append(ClickHouseToStringVisitor.asString(expr)); } if (Randomly.getBoolean()) { sb.append(" SAMPLE BY "); - ClickHouseExpression expr = gen - .generateExpression(ClickHouseSchema.ClickHouseLancerDataType.getRandom()); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); sb.append(ClickHouseToStringVisitor.asString(expr)); } // TODO: PRIMARY KEY diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java index b8da925e5..c7d56ab99 100644 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -19,7 +19,6 @@ import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; import sqlancer.clickhouse.ast.ClickHouseTableReference; @@ -41,17 +40,17 @@ public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { @Override public void check() throws SQLException { + ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); + ClickHouseTable leftTable = randomTables.getTables().remove(0); + ClickHouseTableReference table = new ClickHouseTableReference(leftTable, "left"); List columns = randomTables.getColumns(); + columns.addAll(leftTable.getColumns()); ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); ClickHouseExpression randomWhereCondition = gen.generateExpression(ClickHouseLancerDataType.getRandom()); - List tables = randomTables.getTables(); - List tableList = tables.stream().map(t -> new ClickHouseTableReference(t)) - .collect(Collectors.toList()); - List joins = gen.getRandomJoinClauses(tables); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); - int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, - randomWhereCondition, joins); + List joins = gen.getRandomJoinClauses(table, randomTables.getTables()); + int secondCount = getSecondQuery(table, randomWhereCondition, joins); + int firstCount = getFirstQueryCount(con, table, columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { throw new IgnoreMeException(); } @@ -61,14 +60,14 @@ public void check() throws SQLException { } } - private int getSecondQuery(List tableList, ClickHouseExpression whereClause, + private int getSecondQuery(ClickHouseExpression table, ClickHouseExpression whereClause, List joins) throws SQLException { ClickHouseSelect select = new ClickHouseSelect(); ClickHouseExpression inner = new ClickHouseAliasOperation(whereClause, "check"); select.setFetchColumns(Arrays.asList(inner)); - select.setFromList(tableList); + select.setFromClause(table); select.setJoinClauses(joins); int secondCount = 0; unoptimizedQueryString = "SELECT SUM(check <> 0) FROM (" + ClickHouseToStringVisitor.asString(select) @@ -91,14 +90,14 @@ private int getSecondQuery(List tableList, ClickHouseExpre return secondCount; } - private int getFirstQueryCount(SQLConnection con, List tableList, - List columns, ClickHouseExpression randomWhereCondition, - List joins) throws SQLException { + private int getFirstQueryCount(SQLConnection con, ClickHouseExpression tableList, List columns, + ClickHouseExpression randomWhereCondition, List joins) + throws SQLException { ClickHouseSelect select = new ClickHouseSelect(); - List allColumns = columns.stream().map((c) -> new ClickHouseColumnReference(c)) + List allColumns = columns.stream().map(c -> c.asColumnReference(null)) .collect(Collectors.toList()); select.setFetchColumns(allColumns); - select.setFromList(tableList); + select.setFromClause(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { select.setOrderByExpressions( diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 6275b6a87..2646878a2 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -4,7 +4,8 @@ import java.util.Arrays; import java.util.List; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; @@ -12,17 +13,16 @@ import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; public class ClickHouseTLPAggregateOracle extends ClickHouseTLPBase { - private ClickHouseExpressionGenerator gen; - public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState state) { super(state); ClickHouseErrors.addExpectedExpressionErrors(errors); @@ -31,8 +31,15 @@ public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState sta @Override public void check() throws SQLException { ClickHouseSchema s = state.getSchema(); - ClickHouseSchema.ClickHouseTables targetTables = s.getRandomTableNonEmptyTables(); - gen = new ClickHouseExpressionGenerator(state).setColumns(targetTables.getColumns()); + ClickHouseSchema.ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); + ClickHouseSchema.ClickHouseTable table = randomTables.getTables().remove(0); + ClickHouseTableReference tableRef = new ClickHouseTableReference(table, table.getName()); + List columns = randomTables.getColumns(); + columns.addAll(table.getColumns()); + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); + List joins = gen.getRandomJoinClauses(tableRef, randomTables.getTables()); + + gen = new ClickHouseExpressionGenerator(state).setColumns(randomTables.getColumns()); ClickHouseSelect select = new ClickHouseSelect(); ClickHouseAggregate.ClickHouseAggregateFunction windowFunction = Randomly.fromOptions( ClickHouseAggregate.ClickHouseAggregateFunction.MIN, @@ -41,8 +48,9 @@ public void check() throws SQLException { ClickHouseAggregate aggregate = new ClickHouseAggregate( gen.generateExpressions(ClickHouseSchema.ClickHouseLancerDataType.getRandom(), 1), windowFunction); select.setFetchColumns(Arrays.asList(aggregate)); - List from = ClickHouseCommon.getTableRefs(targetTables.getTables(), s); - select.setFromList(from); + ClickHouseExpression from = tableRef; + select.setFromClause(from); + select.setJoinClauses(joins); if (Randomly.getBoolean()) { select.setOrderByExpressions(gen.generateOrderBys()); } @@ -56,12 +64,22 @@ public void check() throws SQLException { ClickHouseUnaryPostfixOperation notNullClause = new ClickHouseUnaryPostfixOperation(whereClause, ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.IS_NULL, false); - ClickHouseSelect leftSelect = getSelect(aggregate, from, whereClause); - ClickHouseSelect middleSelect = getSelect(aggregate, from, negatedClause); - ClickHouseSelect rightSelect = getSelect(aggregate, from, notNullClause); + select.setFetchColumns(Arrays.asList(new ClickHouseAliasOperation(aggregate, "aggr"))); + select.setFromClause(from); + select.setWhereClause(whereClause); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setGroupByClause(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + String metamorphicText = "SELECT " + aggregate.getFunc().toString() + "(aggr) FROM ("; - metamorphicText += ClickHouseVisitor.asString(leftSelect) + " UNION ALL " - + ClickHouseVisitor.asString(middleSelect) + " UNION ALL " + ClickHouseVisitor.asString(rightSelect); + metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; + select.setWhereClause(negatedClause); + metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; + select.setWhereClause(notNullClause); + metamorphicText += ClickHouseVisitor.asString(select); metamorphicText += ")"; metamorphicText += " SETTINGS aggregate_functions_null_for_empty = 1"; List firstResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQuery, errors, state); @@ -87,20 +105,4 @@ public void check() throws SQLException { } } - private ClickHouseSelect getSelect(ClickHouseAggregate aggregate, List from, - ClickHouseExpression whereClause) { - ClickHouseSelect leftSelect = new ClickHouseSelect(); - leftSelect.setFetchColumns( - Arrays.asList(new ClickHouseExpression.ClickHousePostfixText(aggregate, " as aggr", null))); - leftSelect.setFromList(from); - leftSelect.setWhereClause(whereClause); - if (Randomly.getBooleanWithRatherLowProbability()) { - leftSelect.setGroupByClause(gen.generateExpressions(Randomly.smallNumber() + 1)); - } - if (Randomly.getBoolean()) { - leftSelect.setOrderByExpressions(gen.generateOrderBys()); - } - return leftSelect; - } - } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 245343830..a9b09fd92 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -1,20 +1,25 @@ package sqlancer.clickhouse.oracle.tlp; +import static java.lang.Math.min; +import static java.util.stream.IntStream.range; + import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; +import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; +import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.gen.ClickHouseCommon; +import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; @@ -23,7 +28,7 @@ public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { - ClickHouseSchema s; + ClickHouseSchema schema; ClickHouseTables targetTables; ClickHouseExpressionGenerator gen; ClickHouseSelect select; @@ -35,23 +40,34 @@ public ClickHouseTLPBase(ClickHouseGlobalState state) { @Override public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new ClickHouseExpressionGenerator(state).setColumns(targetTables.getColumns()); + gen = new ClickHouseExpressionGenerator(state); + schema = state.getSchema(); initializeTernaryPredicateVariants(); select = new ClickHouseSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List joinStatements = gen.getRandomJoinClauses(tables); - List tableRefs = ClickHouseCommon.getTableRefs(tables, s); - select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); - select.setFromTables(tableRefs); + List tables = schema.getRandomTableNonEmptyTables().getTables(); + ClickHouseTableReference table = new ClickHouseTableReference( + tables.get((int) Randomly.getNotCachedInteger(0, tables.size())), + Randomly.getBoolean() ? "left" : null); + select.setFromClause(table); + List columns = table.getColumnReferences(); + + if (state.getClickHouseOptions().testJoins && Randomly.getBoolean()) { + List joinStatements = gen.getRandomJoinClauses(table, tables); + select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); + } + gen.addColumns(columns); + range(0, Randomly.smallNumber()).mapToObj(i -> gen.generateExpressionWithColumns(columns, 0)) + .collect(Collectors.toList()); + select.setFetchColumns(generateFetchColumns(columns)); select.setWhereClause(null); + // Smoke check + ComparatorHelper.getResultSetFirstColumnAsString(ClickHouseVisitor.asString(select), errors, state); } - List generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new ClickHouseColumnReference(c)) - .collect(Collectors.toList()); + List generateFetchColumns(List columns) { + List list = Randomly.extractNrRandomColumns(columns, + min(1 + Randomly.smallNumber(), columns.size())); + return list.stream().map(c -> (ClickHouseExpression) c).collect(Collectors.toList()); } @Override diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index 0cdee3864..8aff8fd77 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -3,14 +3,10 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import sqlancer.ComparatorHelper; -import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseVisitor; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; -import sqlancer.clickhouse.ast.ClickHouseExpression; public class ClickHouseTLPGroupByOracle extends ClickHouseTLPBase { @@ -39,13 +35,4 @@ public void check() throws SQLException { ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } - - @Override - List generateFetchColumns() { - List columns; - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new ClickHouseColumnReference(c)) - .collect(Collectors.toList()); - return columns; - } - } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index c7bde4b70..d71e17624 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -4,8 +4,10 @@ import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; @@ -14,9 +16,9 @@ import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; public class ClickHouseTLPHavingOracle extends ClickHouseTLPBase { @@ -29,25 +31,36 @@ public ClickHouseTLPHavingOracle(ClickHouseProvider.ClickHouseGlobalState state) @Override public void check() throws SQLException { ClickHouseSchema s = state.getSchema(); - ClickHouseSchema.ClickHouseTables targetTables = s.getRandomTableNonEmptyTables(); - List groupByColumns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c)).collect(Collectors.toList()); - List columns = targetTables.getColumns(); + ClickHouseSchema.ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); + ClickHouseSchema.ClickHouseTable table = randomTables.getTables().remove(0); + ClickHouseTableReference tableRef = new ClickHouseTableReference(table, table.getName()); + List columns = randomTables.getColumns(); + columns.addAll(table.getColumns()); ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); + List joins = gen.getRandomJoinClauses(tableRef, randomTables.getTables()); + List colRefs = joins.stream() + .flatMap(j -> j.getRightTable().getColumnReferences().stream()).collect(Collectors.toList()); + ClickHouseExpressionGenerator aggrGen = new ClickHouseExpressionGenerator(state).allowAggregates(true) .setColumns(columns); ClickHouseSelect select = new ClickHouseSelect(); select.setFetchColumns(aggrGen.generateExpressions(Randomly.smallNumber() + 1)); - List tables = targetTables.getTables(); - List joinStatements = gen.getRandomJoinClauses(tables); - List from = ClickHouseCommon.getTableRefs(tables, state.getSchema()); - select.setJoinClauses(joinStatements); + ClickHouseTableReference from = new ClickHouseTableReference(table, table.getName()); + select.setJoinClauses(joins); select.setSelectType(ClickHouseSelect.SelectType.ALL); - select.setFromTables(from); + select.setFromClause(from); // TODO order by? + + List groupByColumns = IntStream.range(0, Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(colRefs, 0)).collect(Collectors.toList()); + + if (groupByColumns.isEmpty()) { + throw new IgnoreMeException(); + } select.setGroupByClause(groupByColumns); select.setHavingClause(null); String originalQueryString = ClickHouseVisitor.asString(select); + originalQueryString += " SETTINGS aggregate_functions_null_for_empty=1, enable_optimize_predicate_expression=0"; // https://github.com/ClickHouse/ClickHouse/issues/12264 List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); diff --git a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java index d9dd89a5f..97ca1e282 100644 --- a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java +++ b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java @@ -5,7 +5,8 @@ import java.util.Arrays; import java.util.stream.Collectors; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -13,117 +14,117 @@ class ClickHouseBinaryComparisonOperationTest { @Test void getExpectedValueTrueEqualsTrue() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant equals = trueConst.applyEquals(ClickHouseConstant.createTrue()); - assertEquals(equals.asInt(), 1); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant equals = trueConst.applyEquals(ClickHouseCreateConstant.createTrue()); + assertEquals(true, equals.asBooleanNotNull()); } @Test void getExpectedValueTrueNotEqualsFalse() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant falseConst = ClickHouseConstant.createFalse(); - ClickHouseConstant equals = trueConst.applyEquals(ClickHouseConstant.createFalse()); - ClickHouseConstant equalsFalse = falseConst.applyEquals(ClickHouseConstant.createTrue()); - assertEquals(equals.asInt(), 0); - assertEquals(equalsFalse.asInt(), 0); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant falseConst = ClickHouseCreateConstant.createFalse(); + ClickHouseConstant equals = trueConst.applyEquals(ClickHouseCreateConstant.createFalse()); + ClickHouseConstant equalsFalse = falseConst.applyEquals(ClickHouseCreateConstant.createTrue()); + assertEquals(false, equals.asBooleanNotNull()); + assertEquals(false, equalsFalse.asBooleanNotNull()); } @Test void getExpectedValueFloat64EqualsFloat64() { - ClickHouseConstant oneConst = ClickHouseConstant.createFloat64Constant(1); - ClickHouseConstant oneFConst = ClickHouseConstant.createFloat64Constant(1.0); - ClickHouseConstant zeroConst = ClickHouseConstant.createFloat64Constant(0); - ClickHouseConstant zeroFConst = ClickHouseConstant.createFloat64Constant(0.0); - - assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(oneFConst.applyEquals(oneFConst).asInt(), 1); - assertEquals(oneConst.applyEquals(oneFConst).asInt(), 1); - - assertEquals(oneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(oneFConst.applyEquals(zeroFConst).asInt(), 0); - assertEquals(zeroConst.applyEquals(zeroFConst).asInt(), 1); - assertEquals(zeroFConst.applyEquals(zeroConst).asInt(), 1); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createFloat64Constant(1); + ClickHouseConstant oneFConst = ClickHouseCreateConstant.createFloat64Constant(1.0); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createFloat64Constant(0); + ClickHouseConstant zeroFConst = ClickHouseCreateConstant.createFloat64Constant(0.0); + + assertEquals(true, oneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(true, oneFConst.applyEquals(oneFConst).asBooleanNotNull()); + assertEquals(true, oneConst.applyEquals(oneFConst).asBooleanNotNull()); + + assertEquals(false, oneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, oneFConst.applyEquals(zeroFConst).asBooleanNotNull()); + assertEquals(true, zeroConst.applyEquals(zeroFConst).asBooleanNotNull()); + assertEquals(true, zeroFConst.applyEquals(zeroConst).asBooleanNotNull()); } @Test void getExpectedValueInt32EqualsBool() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant falseConst = ClickHouseConstant.createFalse(); - ClickHouseConstant oneConst = ClickHouseConstant.createInt32Constant(1); - ClickHouseConstant zeroConst = ClickHouseConstant.createInt32Constant(0); - ClickHouseConstant negativeConst = ClickHouseConstant.createInt32Constant(-100); - ClickHouseConstant positiveConst = ClickHouseConstant.createInt32Constant(10000); - - assertEquals(trueConst.applyEquals(oneConst).asInt(), 1); - assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(falseConst.applyEquals(oneConst).asInt(), 0); - - assertEquals(trueConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(oneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(falseConst.applyEquals(zeroConst).asInt(), 1); - - assertEquals(negativeConst.applyEquals(oneConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(trueConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(falseConst).asInt(), 0); - - assertEquals(positiveConst.applyEquals(oneConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(trueConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(falseConst).asInt(), 0); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant falseConst = ClickHouseCreateConstant.createFalse(); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createInt32Constant(1); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createInt32Constant(0); + ClickHouseConstant negativeConst = ClickHouseCreateConstant.createInt32Constant(-100); + ClickHouseConstant positiveConst = ClickHouseCreateConstant.createInt32Constant(10000); + + assertEquals(true, trueConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(true, oneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, falseConst.applyEquals(oneConst).asBooleanNotNull()); + + assertEquals(false, trueConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, oneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(true, falseConst.applyEquals(zeroConst).asBooleanNotNull()); + + assertEquals(false, negativeConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(falseConst).asBooleanNotNull()); + + assertEquals(false, positiveConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(falseConst).asBooleanNotNull()); } @Test void getExpectedValueIntEqualsInt() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant falseConst = ClickHouseConstant.createFalse(); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant falseConst = ClickHouseCreateConstant.createFalse(); for (ClickHouseDataType type : Arrays. stream(ClickHouseDataType.values()) .filter((dt) -> dt.name().contains("Int") && !dt.name().contains("Interval")) .collect(Collectors.toList())) { - ClickHouseConstant oneConst = ClickHouseConstant.createIntConstant(type, 1); - ClickHouseConstant zeroConst = ClickHouseConstant.createIntConstant(type, 0); - ClickHouseConstant negativeConst = ClickHouseConstant.createIntConstant(type, -100); - ClickHouseConstant positiveConst = ClickHouseConstant.createIntConstant(type, 10000); - - assertEquals(trueConst.applyEquals(oneConst).asInt(), 1); - assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(falseConst.applyEquals(oneConst).asInt(), 0); - - assertEquals(trueConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(oneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(falseConst.applyEquals(zeroConst).asInt(), 1); - - assertEquals(negativeConst.applyEquals(oneConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(trueConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(falseConst).asInt(), 0); - - assertEquals(positiveConst.applyEquals(oneConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(trueConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(falseConst).asInt(), 0); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createIntConstant(type, 1); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createIntConstant(type, 0); + ClickHouseConstant negativeConst = ClickHouseCreateConstant.createIntConstant(type, -100); + ClickHouseConstant positiveConst = ClickHouseCreateConstant.createIntConstant(type, 10000); + + assertEquals(true, trueConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(true, oneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, falseConst.applyEquals(oneConst).asBooleanNotNull()); + + assertEquals(false, trueConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, oneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(true, falseConst.applyEquals(zeroConst).asBooleanNotNull()); + + assertEquals(false, negativeConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(falseConst).asBooleanNotNull()); + + assertEquals(false, positiveConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(falseConst).asBooleanNotNull()); } } @Test void getExpectedValueInt32EqualsFloat64() { - ClickHouseConstant float64OneConst = ClickHouseConstant.createFloat64Constant(1.0); - ClickHouseConstant float64ZeroConst = ClickHouseConstant.createFloat64Constant(0.0); - ClickHouseConstant oneConst = ClickHouseConstant.createInt32Constant(1); - ClickHouseConstant zeroConst = ClickHouseConstant.createInt32Constant(0); - ClickHouseConstant negativeConst = ClickHouseConstant.createInt32Constant(-100); - ClickHouseConstant positiveConst = ClickHouseConstant.createInt32Constant(10000); + ClickHouseConstant float64OneConst = ClickHouseCreateConstant.createFloat64Constant(1.0); + ClickHouseConstant float64ZeroConst = ClickHouseCreateConstant.createFloat64Constant(0.0); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createInt32Constant(1); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createInt32Constant(0); + ClickHouseConstant negativeConst = ClickHouseCreateConstant.createInt32Constant(-100); + ClickHouseConstant positiveConst = ClickHouseCreateConstant.createInt32Constant(10000); - assertEquals(float64OneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(float64ZeroConst.applyEquals(oneConst).asInt(), 0); + assertEquals(true, float64OneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, float64ZeroConst.applyEquals(oneConst).asBooleanNotNull()); - assertEquals(float64OneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(float64ZeroConst.applyEquals(zeroConst).asInt(), 1); + assertEquals(false, float64OneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(true, float64ZeroConst.applyEquals(zeroConst).asBooleanNotNull()); - assertEquals(negativeConst.applyEquals(float64OneConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(float64ZeroConst).asInt(), 0); + assertEquals(false, negativeConst.applyEquals(float64OneConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(float64ZeroConst).asBooleanNotNull()); - assertEquals(positiveConst.applyEquals(float64OneConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(float64ZeroConst).asInt(), 0); + assertEquals(false, positiveConst.applyEquals(float64OneConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(float64ZeroConst).asBooleanNotNull()); } } diff --git a/test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java b/test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java new file mode 100644 index 000000000..fc786a625 --- /dev/null +++ b/test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java @@ -0,0 +1,117 @@ +package sqlancer.clickhouse.ast; + +import org.junit.jupiter.api.Test; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ClickHouseOperatorsVisitorTest { + + @Test + void selectUnaryNot() { + ClickHouseConstant trueConst = ClickHouseCreateConstant.createBoolean(true); + ClickHouseExpression notTrue = new ClickHouseUnaryPrefixOperation(trueConst, + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.NOT); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(notTrue)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT (NOT (true))"; + assertEquals(answer, result); + } + + @Test + void selectUnaryMinus() { + ClickHouseConstant fiveConst = ClickHouseCreateConstant.createUInt32Constant(5); + ClickHouseExpression minusFive = new ClickHouseUnaryPrefixOperation(fiveConst, + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.MINUS); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(minusFive)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT (- (5))"; + assertEquals(answer, result); + } + + @Test + void selectUnaryExp() { + ClickHouseConstant tenConst = ClickHouseCreateConstant.createInt32Constant(10); + ClickHouseExpression minusFive = new ClickHouseUnaryFunctionOperation(tenConst, + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.EXP); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(minusFive)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT (exp (10))"; + assertEquals(answer, result); + } + + @Test + void selectBinaryPlus() { + ClickHouseConstant dConst = ClickHouseCreateConstant.createFloat32Constant((float) -1.1); + ClickHouseConstant tenConst = ClickHouseCreateConstant.createInt32Constant(10); + ClickHouseExpression expr = new ClickHouseBinaryArithmeticOperation(dConst, tenConst, + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.ADD); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT ((-1.1)+(10))"; + assertEquals(answer, result); + } + + @Test + void selectBinaryPow() { + ClickHouseConstant threeConst = ClickHouseCreateConstant.createInt8Constant(3); + ClickHouseConstant tenConst = ClickHouseCreateConstant.createInt32Constant(10); + ClickHouseExpression expr = new ClickHouseBinaryFunctionOperation(threeConst, tenConst, + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.POW); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT pow(3,10)"; + assertEquals(answer, result); + } + + @Test + void selectBinaryLCM() { + ClickHouseConstant aConst = ClickHouseCreateConstant.createInt8Constant(100); + ClickHouseConstant bConst = ClickHouseCreateConstant.createInt32Constant(-100); + ClickHouseExpression expr = new ClickHouseBinaryFunctionOperation(aConst, bConst, + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.LCM); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT lcm(100,-100)"; + assertEquals(answer, result); + } + + @Test + void selectBinaryDivCol() { + ClickHouseColumnReference a = new ClickHouseColumnReference(new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null), null, null); + ClickHouseColumnReference b = new ClickHouseColumnReference(new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null), null, null); + ClickHouseExpression expr = new ClickHouseBinaryFunctionOperation(a, b, + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.INT_DIV); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT intDiv(a,b)"; + assertEquals(answer, result); + } + + @Test + void selectBinaryComp() { + ClickHouseConstant aConst = ClickHouseCreateConstant.createInt8Constant(10); + ClickHouseConstant bConst = ClickHouseCreateConstant.createInt32Constant(100); + ClickHouseExpression expr = new ClickHouseBinaryComparisonOperation(aConst, bConst, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.GREATER); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT ((10)>(100))"; + assertEquals(answer, result); + } + +} diff --git a/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java new file mode 100644 index 000000000..7231aced8 --- /dev/null +++ b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java @@ -0,0 +1,345 @@ +package sqlancer.clickhouse.ast; + +import org.junit.jupiter.api.Test; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.constant.ClickHouseInt8Constant; +import sqlancer.common.schema.TableIndex; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ClickHouseToStringVisitorTest { + + @Test + void select1Test() { + ClickHouseConstant oneConst = new ClickHouseInt8Constant(1); + ClickHouseSelect selectOne = new ClickHouseSelect(); + selectOne.setFetchColumns(Arrays.asList(oneConst)); + String result = ClickHouseVisitor.asString(selectOne); + String answer = "SELECT 1"; + assertEquals(answer, result); + } + + @Test + void select1asATest() { + ClickHouseAliasOperation oneConstAsA = new ClickHouseAliasOperation(new ClickHouseInt8Constant(1), "a"); + ClickHouseSelect selectOne = new ClickHouseSelect(); + selectOne.setFetchColumns(Arrays.asList(oneConstAsA)); + String result = ClickHouseVisitor.asString(selectOne); + String answer = "SELECT 1 AS `a`"; + assertEquals(answer, result); + } + + @Test + void selectATest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseSelect selectA = new ClickHouseSelect(); + selectA.setFetchColumns(Arrays.asList(a_ref)); + selectA.setFromClause(table_ref); + String result = ClickHouseVisitor.asString(selectA); + String answer = "SELECT t.a FROM t"; + assertEquals(answer, result); + } + + @Test + void selectAasBTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseAliasOperation b = new ClickHouseAliasOperation(a_ref, "b"); + ClickHouseColumnReference b_ref = new ClickHouseColumnReference(b); + ClickHouseSelect selectA = new ClickHouseSelect(); + selectA.setFetchColumns(Arrays.asList(b, b_ref)); + selectA.setFromClause(table_ref); + String result = ClickHouseVisitor.asString(selectA); + String answer = "SELECT t.a AS `b`, b FROM t"; + assertEquals(answer, result); + } + + @Test + void selectABTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + selectAB.setFetchColumns(Arrays.asList(a_ref, b_ref)); + selectAB.setFromClause(table_ref); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT t.a, t.b FROM t"; + assertEquals(answer, result); + } + + @Test + void selectWhereAGreaterBTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + selectAB.setFetchColumns(Arrays.asList(a_ref, b_ref)); + selectAB.setFromClause(table_ref); + selectAB.setWhereClause(new ClickHouseBinaryComparisonOperation(a_ref, b_ref, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.GREATER)); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT t.a, t.b FROM t WHERE ((t.a)>(t.b))"; + assertEquals(answer, result); + } + + @Test + void selectWhereAGreaterConstTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + selectAB.setFetchColumns(Arrays.asList(a_ref, b_ref)); + selectAB.setFromClause(table_ref); + ClickHouseConstant c_const = new ClickHouseInt8Constant(1); + selectAB.setWhereClause(new ClickHouseBinaryComparisonOperation(a_ref, c_const, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.GREATER)); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT t.a, t.b FROM t WHERE ((t.a)>(1))"; + assertEquals(answer, result); + } + + @Test + void selectSumAGroupByBTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + ClickHouseAggregate sum_a = new ClickHouseAggregate(Arrays.asList(a_ref), + ClickHouseAggregate.ClickHouseAggregateFunction.SUM); + selectAB.setFetchColumns(Arrays.asList(sum_a)); + selectAB.setFromClause(table_ref); + selectAB.setGroupByClause(Arrays.asList(b_ref)); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT SUM(t.a) FROM t GROUP BY t.b"; + assertEquals(answer, result); + } + + @Test + void selectCrossJoinTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", empty_col_list, indexes, + false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", empty_col_list, indexes, + false); + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, null); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, null); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table1); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table1); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table2); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table2); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseColumnReference a1_ref = a1_col.asColumnReference(null); + ClickHouseColumnReference b1_ref = b1_col.asColumnReference(null); + ClickHouseColumnReference a2_ref = a2_col.asColumnReference(null); + ClickHouseColumnReference b2_ref = b2_col.asColumnReference(null); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.CROSS); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT t1.a1, t2.a2, t1.b1, t2.b2 FROM t1 JOIN t2"; + assertEquals(answer, result); + } + + @Test + void selectCrossJoinAliasedTest() { + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", + Arrays.asList(a1_col, b1_col), indexes, false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", + Arrays.asList(a2_col, b2_col), indexes, false); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, "left"); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, "right"); + + List t1_col_ref = table1_ref.getColumnReferences(); + ClickHouseColumnReference a1_ref = t1_col_ref.get(0); + ClickHouseColumnReference a2_ref = t1_col_ref.get(1); + + List t2_col_ref = table2_ref.getColumnReferences(); + ClickHouseColumnReference b1_ref = t2_col_ref.get(0); + ClickHouseColumnReference b2_ref = t2_col_ref.get(1); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.CROSS); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT left.a1, left.b1, right.a2, right.b2 FROM t1 AS left JOIN t2 AS right"; + assertEquals(answer, result); + } + + @Test + void selectJoinONTest() { + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", + Arrays.asList(a1_col, b1_col), indexes, false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", + Arrays.asList(a2_col, b2_col), indexes, false); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, null); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, null); + + List t1_col_ref = table1_ref.getColumnReferences(); + ClickHouseColumnReference a1_ref = t1_col_ref.get(0); + ClickHouseColumnReference b1_ref = t1_col_ref.get(1); + + List t2_col_ref = table2_ref.getColumnReferences(); + ClickHouseColumnReference a2_ref = t2_col_ref.get(0); + ClickHouseColumnReference b2_ref = t2_col_ref.get(1); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoinOnClause on = new ClickHouseExpression.ClickHouseJoinOnClause(a1_ref, + a2_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.INNER, on); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT t1.a1, t2.a2, t1.b1, t2.b2 FROM t1 INNER JOIN t2 ON ((t1.a1)=(t2.a2))"; + assertEquals(answer, result); + } + + @Test + void selectJoinONAliasedTest() { + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", + Arrays.asList(a1_col, b1_col), indexes, false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", + Arrays.asList(a2_col, b2_col), indexes, false); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, "left"); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, "right"); + + List t1_col_ref = table1_ref.getColumnReferences(); + ClickHouseColumnReference a1_ref = t1_col_ref.get(0); + ClickHouseColumnReference b1_ref = t1_col_ref.get(1); + + List t2_col_ref = table2_ref.getColumnReferences(); + ClickHouseColumnReference a2_ref = t2_col_ref.get(0); + ClickHouseColumnReference b2_ref = t2_col_ref.get(1); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoinOnClause on = new ClickHouseExpression.ClickHouseJoinOnClause(a1_ref, + a2_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.INNER, on); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT left.a1, right.a2, left.b1, right.b2 FROM t1 AS left INNER JOIN t2 AS right ON ((left.a1)=(right.a2))"; + assertEquals(answer, result); + } +} From 894b358b08a8059ad05eb51d0769205503ebf5bb Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 19 Nov 2022 12:28:00 +0800 Subject: [PATCH 0665/1284] tidb: ignore Empty pattern is invalid error (#616) Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index f37bdb685..0586846fc 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -31,6 +31,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { // regex errors.add("error parsing regexp"); errors.add("from regexp"); + errors.add("Empty pattern is invalid"); // To avoid bugs errors.add("Unknown column"); // https://github.com/pingcap/tidb/issues/35522 From 032a4460bf3c1d1da5861ae5dfb4195d27bc4e70 Mon Sep 17 00:00:00 2001 From: taichong Date: Sun, 20 Nov 2022 10:39:23 +0800 Subject: [PATCH 0666/1284] fix: add databend expr errors --- src/sqlancer/databend/DatabendErrors.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 7e1f80c4e..bea8c9558 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -9,6 +9,7 @@ private DatabendErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE @@ -21,11 +22,13 @@ public static void addExpressionErrors(ExpectedErrors errors) { public static void addInsertErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); } public static void addGroupByErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); } From 422b41ad185305d3b0ec318c059ad5f4d4fbdf36 Mon Sep 17 00:00:00 2001 From: Nishita Date: Wed, 23 Nov 2022 19:28:41 +0530 Subject: [PATCH 0667/1284] First commit for HSQLDB --- pom.xml | 6 + src/sqlancer/hsqldb/HSQLDBErrors.java | 24 ++ src/sqlancer/hsqldb/HSQLDBOptions.java | 31 +++ src/sqlancer/hsqldb/HSQLDBProvider.java | 104 ++++++++ src/sqlancer/hsqldb/HSQLDBSchema.java | 153 ++++++++++++ .../hsqldb/HSQLDBToStringVisitor.java | 91 +++++++ src/sqlancer/hsqldb/ast/HSQLDBConstant.java | 229 ++++++++++++++++++ src/sqlancer/hsqldb/ast/HSQLDBExpression.java | 4 + src/sqlancer/hsqldb/ast/HSQLDBJoin.java | 124 ++++++++++ src/sqlancer/hsqldb/ast/HSQLDBSelect.java | 18 ++ .../hsqldb/gen/HSQLDBExpressionGenerator.java | 88 +++++++ .../hsqldb/gen/HSQLDBInsertGenerator.java | 49 ++++ .../hsqldb/gen/HSQLDBTableGenerator.java | 57 +++++ .../hsqldb/gen/HSQLDBUpdateGenerator.java | 42 ++++ .../test/HSQLDBQueryPartitioningBase.java | 25 ++ 15 files changed, 1045 insertions(+) create mode 100644 src/sqlancer/hsqldb/HSQLDBErrors.java create mode 100644 src/sqlancer/hsqldb/HSQLDBOptions.java create mode 100644 src/sqlancer/hsqldb/HSQLDBProvider.java create mode 100644 src/sqlancer/hsqldb/HSQLDBSchema.java create mode 100644 src/sqlancer/hsqldb/HSQLDBToStringVisitor.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBConstant.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBExpression.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBJoin.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBSelect.java create mode 100644 src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java create mode 100644 src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java create mode 100644 src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java create mode 100644 src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java create mode 100644 src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java diff --git a/pom.xml b/pom.xml index 6dae29f62..ffee5dfd9 100644 --- a/pom.xml +++ b/pom.xml @@ -352,6 +352,12 @@ questdb 6.5.3 + + org.hsqldb + hsqldb + 2.7.1 + runtime + diff --git a/src/sqlancer/hsqldb/HSQLDBErrors.java b/src/sqlancer/hsqldb/HSQLDBErrors.java new file mode 100644 index 000000000..5fa77051a --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBErrors.java @@ -0,0 +1,24 @@ +package sqlancer.hsqldb; + +import sqlancer.common.query.ExpectedErrors; + +public final class HSQLDBErrors { + + private HSQLDBErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + } + + private static void addRegexErrors(ExpectedErrors errors) { + } + + private static void addFunctionErrors(ExpectedErrors errors) { + } + + public static void addInsertErrors(ExpectedErrors errors) { + addRegexErrors(errors); + addFunctionErrors(errors); + } + +} diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java new file mode 100644 index 000000000..b2418fe8d --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -0,0 +1,31 @@ +package sqlancer.hsqldb; + +import com.beust.jcommander.Parameter; +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.hsqldb.test.HSQLDBQueryPartitioningBase; + +import java.sql.SQLException; +import java.util.List; + +public class HSQLDBOptions implements DBMSSpecificOptions { + + @Parameter(names = "--oracle") + public List oracle = List.of(HSQLDBOracleFactory.WHERE); + + public enum HSQLDBOracleFactory implements OracleFactory { + WHERE { + @Override + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) throws SQLException { + return new HSQLDBQueryPartitioningBase(globalState); + } + } + } + + @Override + public List getTestOracleFactory() { + return oracle; + } + +} diff --git a/src/sqlancer/hsqldb/HSQLDBProvider.java b/src/sqlancer/hsqldb/HSQLDBProvider.java new file mode 100644 index 000000000..cd238df23 --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBProvider.java @@ -0,0 +1,104 @@ +package sqlancer.hsqldb; + +import com.google.auto.service.AutoService; +import sqlancer.*; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.hsqldb.gen.HSQLDBInsertGenerator; +import sqlancer.hsqldb.gen.HSQLDBTableGenerator; +import sqlancer.hsqldb.gen.HSQLDBUpdateGenerator; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +@AutoService(DatabaseProvider.class) +public class HSQLDBProvider extends SQLProviderAdapter { + + private static final String HSQLDB = "hsqldb"; + + public HSQLDBProvider() { + super(HSQLDBGlobalState.class, HSQLDBOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(HSQLDBInsertGenerator::getQuery), + UPDATE(HSQLDBUpdateGenerator::getQuery); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + @Override + public SQLConnection createDatabase(HSQLDBGlobalState globalState) throws Exception { + String databaseName = globalState.getDatabaseName(); + String url = "jdbc:hsqldb:file:" + databaseName; + MainOptions options = globalState.getOptions(); + Connection connection = DriverManager.getConnection(url, options.getUserName(), options.getPassword()); + //When a server instance is started, or when a connection is made to an in-process database, + // a new, empty database is created if no database exists at the given path. + try (Statement s = connection.createStatement()) { + s.execute("DROP SCHEMA PUBLIC CASCADE"); + s.execute("SET DATABASE SQL DOUBLE NAN FALSE"); + } + return new SQLConnection(connection); + } + + @Override + public String getDBMSName() { + return HSQLDB; + } + + @Override + public void generateDatabase(HSQLDBGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new HSQLDBTableGenerator().getQuery(globalState, null); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + StatementExecutor se = new StatementExecutor<>(globalState, HSQLDBProvider.Action.values(), + HSQLDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + private static int mapActions(HSQLDBProvider.HSQLDBGlobalState globalState, HSQLDBProvider.Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case UPDATE: + return r.getInteger(0, 1); + default: + throw new AssertionError(a); + } + } + + public static class HSQLDBGlobalState extends SQLGlobalState { + + @Override + protected HSQLDBSchema readSchema() throws SQLException { + return HSQLDBSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } +} + + diff --git a/src/sqlancer/hsqldb/HSQLDBSchema.java b/src/sqlancer/hsqldb/HSQLDBSchema.java new file mode 100644 index 000000000..13059ef3e --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBSchema.java @@ -0,0 +1,153 @@ +package sqlancer.hsqldb; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class HSQLDBSchema extends AbstractSchema { + + public HSQLDBSchema(List databaseTables) { + super(databaseTables); + } + + public static HSQLDBSchema fromConnection(SQLConnection connection, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(connection); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(connection, tableName); + boolean isView = tableName.startsWith("v"); + HSQLDBSchema.HSQLDBTable t = new HSQLDBSchema.HSQLDBTable(tableName, databaseColumns, isView); + for (HSQLDBSchema.HSQLDBColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new HSQLDBSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'PUBLIC';")) { + while (rs.next()) { + tableNames.add(rs.getString("TABLE_NAME")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + String sql = "SELECT COLUMN_NAME, DATA_TYPE, TYPE_NAME, COLUMN_SIZE FROM INFORMATION_SCHEMA.SYSTEM_COLUMNS WHERE TABLE_NAME = '%s';"; + try (ResultSet rs = s.executeQuery(String.format(sql, tableName))) { + while (rs.next()) { + HSQLDBDataType dataType = HSQLDBDataType.from(rs.getString("TYPE_NAME")); + HSQLDBCompositeDataType compositeDataType = new HSQLDBCompositeDataType(dataType, rs.getInt("COLUMN_SIZE")); + HSQLDBColumn column = new HSQLDBColumn(rs.getString("COLUMN_NAME"), null, compositeDataType ); + tableNames.add(column); + } + } + } + return tableNames; + } + + public static class HSQLDBTable extends AbstractRelationalTable { + + public HSQLDBTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static class HSQLDBColumn extends AbstractTableColumn + { + + public HSQLDBColumn(String name, HSQLDBTable table, HSQLDBCompositeDataType type) { + super(name, table, type); + } + } + + public enum HSQLDBDataType { + + INTEGER, DOUBLE, BOOLEAN, CHAR, VARCHAR, BINARY, TIME, DATE, TIMESTAMP, NULL; + + public static HSQLDBSchema.HSQLDBDataType getRandomWithoutNull() { + HSQLDBSchema.HSQLDBDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == HSQLDBSchema.HSQLDBDataType.NULL); + return dt; + } + + public static HSQLDBDataType from(String type_name) { + for( HSQLDBDataType value : HSQLDBDataType.values()) { + if(value.name().equals(type_name)) { + return value; + } + } + return NULL; + } + } + + public static class HSQLDBCompositeDataType{ + private final int size; + private final HSQLDBDataType type; + + public HSQLDBCompositeDataType(HSQLDBDataType type, int size) { + this.type = type; + this.size = size; + } + + + public static HSQLDBCompositeDataType getRandomWithoutNull() { + HSQLDBSchema.HSQLDBDataType type = HSQLDBSchema.HSQLDBDataType.getRandomWithoutNull(); + int size; + switch (type) { + case VARCHAR: + case CHAR: + case TIME: + case BINARY: + case TIMESTAMP: + size = Randomly.fromOptions(4,6,8); + break; + case BOOLEAN: + case INTEGER: + case DOUBLE: + //case UUID: + //case OTHER: + case DATE: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new HSQLDBSchema.HSQLDBCompositeDataType(type, size); + } + + public HSQLDBDataType getType() { + return type; + } + + public int getSize() { + return size; + } + } +} diff --git a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java new file mode 100644 index 000000000..66ccbfd12 --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java @@ -0,0 +1,91 @@ +package sqlancer.hsqldb; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.hsqldb.ast.HSQLDBConstant; +import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; + +public class HSQLDBToStringVisitor extends NewToStringVisitor { + + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof HSQLDBConstant) { + visit((HSQLDBConstant) expr); + } else if (expr instanceof HSQLDBSelect) { + visit((HSQLDBSelect) expr); + } else if (expr instanceof HSQLDBJoin) { + visit((HSQLDBJoin) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + public static String asString(Node expr) { + HSQLDBToStringVisitor visitor = new HSQLDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + private void visit(HSQLDBJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getOuterType() != null) { + sb.append(join.getOuterType()); + } + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(HSQLDBConstant constant) { + sb.append(constant.toString()); + } + + private void visit(HSQLDBSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java new file mode 100644 index 000000000..d4126bd6e --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java @@ -0,0 +1,229 @@ +package sqlancer.hsqldb.ast; +import sqlancer.common.ast.newast.Node; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public class HSQLDBConstant implements Node { + + private HSQLDBConstant() { + } + public static class HSQLDBNullConstant extends HSQLDBConstant { + + @Override + public String toString() { + return "Null"; + } + + } + + public static class HSQLDBIntConstant extends HSQLDBConstant { + + private final int value; + + public HSQLDBIntConstant(long value) { + this.value = (int)value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class HSQLDBDoubleConstant extends HSQLDBConstant { + + private final double value; + + public HSQLDBDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "1.0e1/0.0e1"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-1.0e1/0.0e1"; + } + return String.valueOf(value); + } + + } + + public static class HSQLDBTextConstant extends HSQLDBConstant { + + private final String value; + + public HSQLDBTextConstant(String value) { + this.value = value; + } + + public HSQLDBTextConstant(String value, int size) { + this.value = value.substring(0, Math.min(value.length(), size)); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + + public static class HSQLDBBinaryConstant extends HSQLDBConstant { + + private final String value; + + public HSQLDBBinaryConstant(long value, int size) { + StringBuilder hex = new StringBuilder(Long.toHexString(value)); + if(hex.length() < 2) { + hex.append('0'); + } + this.value = hex.substring(0, Math.min(hex.length(), size)); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "X'" + value + "'"; + } + + } + + public static class HSQLDBDateConstant extends HSQLDBConstant { + + public String textRepr; + + public HSQLDBDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + } + + public static class HSQLDBTimestampConstant extends HSQLDBConstant { + + public String textRepr; + + public HSQLDBTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + } + + public static class HSQLDBTimeConstant extends HSQLDBConstant { + + public String textRepr; + + public HSQLDBTimeConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIME '%s'", textRepr); + } + + } + + public static class HSQLDBBooleanConstant extends HSQLDBConstant { + + private final boolean value; + + public HSQLDBBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static Node createStringConstant(String text, int size) { + return new HSQLDBTextConstant(text, size); + } + + public static Node createFloatConstant(double val) { + return new HSQLDBDoubleConstant(val); + } + + public static Node createIntConstant(long val) { + return new HSQLDBIntConstant(val); + } + + public static Node createNullConstant() { + return new HSQLDBNullConstant(); + } + + public static Node createBooleanConstant(boolean val) { + return new HSQLDBBooleanConstant(val); + } + + public static Node createDateConstant(long integer) { + return new HSQLDBDateConstant(integer); + } + + public static Node createTimeConstant(long integer, int size) { + return new HSQLDBTimeConstant(integer); + } + + public static Node createTimestampConstant(long integer, int size) { + return new HSQLDBTimestampConstant(integer); + } + + public static Node createBinaryConstant(long nonCachedInteger, int size) { + return new HSQLDBBinaryConstant(nonCachedInteger, size); + } + + +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBExpression.java b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java new file mode 100644 index 000000000..b5359c11c --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java @@ -0,0 +1,4 @@ +package sqlancer.hsqldb.ast; + +public interface HSQLDBExpression { +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java new file mode 100644 index 000000000..524702839 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java @@ -0,0 +1,124 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; + +import java.util.ArrayList; +import java.util.List; + +public class HSQLDBJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + private OuterType outerType; + + public enum JoinType { + INNER, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum OuterType { + FULL, LEFT, RIGHT; + + public static OuterType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public HSQLDBJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + private void setOuterType(OuterType outerType) { + this.outerType = outerType; + } + + public OuterType getOuterType() { + return outerType; + } + + public static List> getJoins( + List> tableList, HSQLDBGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(globalState).setColumns(columns); + switch (HSQLDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case NATURAL: + joinExpressions.add(HSQLDBJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); + break; + case LEFT: + joinExpressions + .add(HSQLDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case RIGHT: + joinExpressions + .add(HSQLDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static HSQLDBJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new HSQLDBJoin(left, right, JoinType.RIGHT, predicate); + } + + public static HSQLDBJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new HSQLDBJoin(left, right, JoinType.LEFT, predicate); + } + + public static HSQLDBJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new HSQLDBJoin(left, right, JoinType.INNER, predicate); + } + + public static Node createNaturalJoin(TableReferenceNode left, + TableReferenceNode right, OuterType naturalJoinType) { + HSQLDBJoin join = new HSQLDBJoin(left, right, JoinType.NATURAL, null); + join.setOuterType(naturalJoinType); + return join; + } + +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java new file mode 100644 index 000000000..05cd390f2 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java @@ -0,0 +1,18 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class HSQLDBSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java new file mode 100644 index 000000000..d01454a7f --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -0,0 +1,88 @@ +package sqlancer.hsqldb.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.ast.HSQLDBConstant; +import sqlancer.hsqldb.ast.HSQLDBExpression; + + +public final class HSQLDBExpressionGenerator extends TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { + + HSQLDBProvider.HSQLDBGlobalState hsqldbGlobalState; + public HSQLDBExpressionGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { + this.hsqldbGlobalState = globalState; + } + + @Override + public Node generatePredicate() { + return null; + } + + @Override + public Node negatePredicate(Node predicate) { + return null; + } + + @Override + public Node isNull(Node expr) { + return null; + } + + @Override + public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataType type) { + if (type.getType() == HSQLDBSchema.HSQLDBDataType.NULL || Randomly.getBooleanWithSmallProbability()) { + return HSQLDBConstant.createNullConstant(); + } + switch (type.getType()) { + case CHAR: + return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getAlphabeticChar(), type.getSize()); + case VARCHAR: + return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getString(), type.getSize()); + case TIME: + return HSQLDBConstant.createTimeConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); + case TIMESTAMP: + return HSQLDBConstant.createTimestampConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize() ); + + case INTEGER: + return HSQLDBConstant.HSQLDBIntConstant.createIntConstant(Randomly.getNonCachedInteger()); + case DOUBLE: + return HSQLDBConstant.HSQLDBDoubleConstant.createFloatConstant(hsqldbGlobalState.getRandomly().getDouble()); + case BOOLEAN: + return HSQLDBConstant.HSQLDBBooleanConstant.createBooleanConstant(Randomly.getBoolean()); + case DATE: + return HSQLDBConstant.createDateConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis())); + case BINARY: + return HSQLDBConstant.createBinaryConstant(Randomly.getNonCachedInteger(), type.getSize()); +// case OTHER: +// break; +// case NULL: +// break; + default: + throw new AssertionError("Unknown type: " + type); + } + } + + @Override + protected Node generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { + return null; + } + + @Override + protected Node generateColumn(HSQLDBSchema.HSQLDBCompositeDataType type) { + return null; + } + + @Override + protected HSQLDBSchema.HSQLDBCompositeDataType getRandomType() { + return HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull(); + } + + @Override + protected boolean canGenerateColumnOfType(HSQLDBSchema.HSQLDBCompositeDataType type) { + return columns.stream().anyMatch(c -> c.getType() == type); + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java new file mode 100644 index 000000000..0ce3fcea9 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.hsqldb.gen; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hsqldb.HSQLDBErrors; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBExpression; + +import java.util.List; +import java.util.stream.Collectors; + +public class HSQLDBInsertGenerator extends AbstractInsertGenerator { + + private final HSQLDBProvider.HSQLDBGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public HSQLDBInsertGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { + return new HSQLDBInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + HSQLDBErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertValue(HSQLDBSchema.HSQLDBColumn column) { + Node expression = new HSQLDBExpressionGenerator(globalState).generateConstant(column.getType()); + String s = HSQLDBToStringVisitor.asString(expression); + sb.append(s); + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java new file mode 100644 index 000000000..58b1cc426 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java @@ -0,0 +1,57 @@ +package sqlancer.hsqldb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; + +public class HSQLDBTableGenerator { + + public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @Nullable String tableName) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String name = tableName; + if (tableName == null) { + name = globalState.getSchema().getFreeTableName(); + } + sb.append("CREATE TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(name); + sb.append("("); + List columns = getNewColumns(name); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + sb.append(columns.get(i).getType().getType().name()); + if(columns.get(i).getType().getSize() > 0) { + //Cannot specify size for non composite data types + sb.append("("); + sb.append(columns.get(i).getType().getSize()); + sb.append(")"); + } + } + sb.append(")"); + sb.append(";"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private static List getNewColumns(String tableName) { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + HSQLDBSchema.HSQLDBCompositeDataType columnType = HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull(); + columns.add(new HSQLDBSchema.HSQLDBColumn(columnName, null, columnType)); + } + return columns; + } +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java new file mode 100644 index 000000000..92e751300 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -0,0 +1,42 @@ +package sqlancer.hsqldb.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hsqldb.HSQLDBErrors; +import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBSchema; + +import java.util.List; + +public final class HSQLDBUpdateGenerator { + + private HSQLDBUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(HSQLDBGlobalState globalState) { + StringBuilder sb = new StringBuilder("UPDATE "); + ExpectedErrors errors = new ExpectedErrors(); + HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + //HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(" SET "); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append("="); +// Node expr; +// if (Randomly.getBooleanWithSmallProbability()) { +// //expr = gen.generateExpression(); +// HSQLDBErrors.addExpressionErrors(errors); +// } else { +// //expr = gen.generateConstant(); +// } + } + HSQLDBErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java new file mode 100644 index 000000000..69216dca7 --- /dev/null +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -0,0 +1,25 @@ +package sqlancer.hsqldb.test; + +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.ast.HSQLDBExpression; + +public class HSQLDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> implements TestOracle { + + public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { + super(state); + } + + @Override + protected ExpressionGenerator> getGen() { + return null; + } + + @Override + public void check() throws Exception { + + } +} From 94693f7e92cf5bf7128897c1b174b7bd39d7d88d Mon Sep 17 00:00:00 2001 From: Nishita Date: Fri, 25 Nov 2022 18:46:29 +0530 Subject: [PATCH 0668/1284] Added Update statements --- src/sqlancer/hsqldb/HSQLDBProvider.java | 2 +- .../hsqldb/gen/HSQLDBInsertGenerator.java | 3 +- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 30 ++++++++++++------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/hsqldb/HSQLDBProvider.java b/src/sqlancer/hsqldb/HSQLDBProvider.java index cd238df23..351c8ab8c 100644 --- a/src/sqlancer/hsqldb/HSQLDBProvider.java +++ b/src/sqlancer/hsqldb/HSQLDBProvider.java @@ -85,7 +85,7 @@ private static int mapActions(HSQLDBProvider.HSQLDBGlobalState globalState, HSQL case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case UPDATE: - return r.getInteger(0, 1); + return r.getInteger(0, 10); default: throw new AssertionError(a); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index 0ce3fcea9..e53b67d44 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -3,7 +3,6 @@ import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.hsqldb.HSQLDBErrors; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.HSQLDBToStringVisitor; @@ -35,7 +34,7 @@ private SQLQueryAdapter generate() { sb.append(")"); sb.append(" VALUES "); insertColumns(columns); - HSQLDBErrors.addInsertErrors(errors); + //HSQLDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index 92e751300..97b8f6e18 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -1,24 +1,32 @@ package sqlancer.hsqldb.gen; +import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.hsqldb.HSQLDBErrors; -import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBExpression; import java.util.List; public final class HSQLDBUpdateGenerator { - private HSQLDBUpdateGenerator() { + private final static ExpectedErrors errors = new ExpectedErrors(); + private final HSQLDBProvider.HSQLDBGlobalState globalState; + private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { + this.globalState = globalState; } - public static SQLQueryAdapter getQuery(HSQLDBGlobalState globalState) { + public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { + return new HSQLDBUpdateGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); - //HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); List columns = table.getRandomNonEmptyColumnSubset(); for (int i = 0; i < columns.size(); i++) { @@ -27,15 +35,15 @@ public static SQLQueryAdapter getQuery(HSQLDBGlobalState globalState) { } sb.append(columns.get(i).getName()); sb.append("="); -// Node expr; + Node expr; // if (Randomly.getBooleanWithSmallProbability()) { -// //expr = gen.generateExpression(); +// expr = gen.generateExpression(columns.get(i).getType()); // HSQLDBErrors.addExpressionErrors(errors); // } else { -// //expr = gen.generateConstant(); -// } + expr = gen.generateConstant(columns.get(i).getType()); + //} + sb.append(HSQLDBToStringVisitor.asString(expr)); } - HSQLDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } From 58508fed82d17b9de0cf1012056dc870069d86c0 Mon Sep 17 00:00:00 2001 From: Nishita Date: Mon, 28 Nov 2022 12:00:56 +0530 Subject: [PATCH 0669/1284] Fixes Checkstyle violations and adds support for expression --- src/sqlancer/hsqldb/HSQLDBErrors.java | 3 + src/sqlancer/hsqldb/HSQLDBOptions.java | 11 +- src/sqlancer/hsqldb/HSQLDBProvider.java | 54 +++++---- src/sqlancer/hsqldb/HSQLDBSchema.java | 78 +++++++------ .../hsqldb/HSQLDBToStringVisitor.java | 3 +- src/sqlancer/hsqldb/ast/HSQLDBConstant.java | 10 +- src/sqlancer/hsqldb/ast/HSQLDBJoin.java | 51 ++++----- .../ast/HSQLDBUnaryPostfixOperation.java | 78 +++++++++++++ .../ast/HSQLDBUnaryPrefixOperation.java | 83 ++++++++++++++ .../hsqldb/gen/HSQLDBExpressionGenerator.java | 103 +++++++++++++----- .../hsqldb/gen/HSQLDBInsertGenerator.java | 12 +- .../hsqldb/gen/HSQLDBTableGenerator.java | 19 ++-- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 21 ++-- .../test/HSQLDBQueryPartitioningBase.java | 49 ++++++++- .../HSQLDBQueryPartitioningWhereTester.java | 43 ++++++++ 15 files changed, 465 insertions(+), 153 deletions(-) create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java create mode 100644 src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java diff --git a/src/sqlancer/hsqldb/HSQLDBErrors.java b/src/sqlancer/hsqldb/HSQLDBErrors.java index 5fa77051a..8e2a8cd2f 100644 --- a/src/sqlancer/hsqldb/HSQLDBErrors.java +++ b/src/sqlancer/hsqldb/HSQLDBErrors.java @@ -8,12 +8,15 @@ private HSQLDBErrors() { } public static void addExpressionErrors(ExpectedErrors errors) { + errors.add(""); } private static void addRegexErrors(ExpectedErrors errors) { + errors.add(""); } private static void addFunctionErrors(ExpectedErrors errors) { + errors.add(""); } public static void addInsertErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index b2418fe8d..54954b797 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -1,13 +1,14 @@ package sqlancer.hsqldb; +import java.sql.SQLException; +import java.util.List; + import com.beust.jcommander.Parameter; + import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; -import sqlancer.hsqldb.test.HSQLDBQueryPartitioningBase; - -import java.sql.SQLException; -import java.util.List; +import sqlancer.hsqldb.test.HSQLDBQueryPartitioningWhereTester; public class HSQLDBOptions implements DBMSSpecificOptions { @@ -18,7 +19,7 @@ public enum HSQLDBOracleFactory implements OracleFactory { @@ -22,9 +31,8 @@ public HSQLDBProvider() { super(HSQLDBGlobalState.class, HSQLDBOptions.class); } - public enum Action implements AbstractAction { - INSERT(HSQLDBInsertGenerator::getQuery), - UPDATE(HSQLDBUpdateGenerator::getQuery); + public enum Action implements AbstractAction { + INSERT(HSQLDBInsertGenerator::getQuery), UPDATE(HSQLDBUpdateGenerator::getQuery); private final SQLQueryProvider sqlQueryProvider; @@ -44,7 +52,7 @@ public SQLConnection createDatabase(HSQLDBGlobalState globalState) throws Except String url = "jdbc:hsqldb:file:" + databaseName; MainOptions options = globalState.getOptions(); Connection connection = DriverManager.getConnection(url, options.getUserName(), options.getPassword()); - //When a server instance is started, or when a connection is made to an in-process database, + // When a server instance is started, or when a connection is made to an in-process database, // a new, empty database is created if no database exists at the given path. try (Statement s = connection.createStatement()) { s.execute("DROP SCHEMA PUBLIC CASCADE"); @@ -70,24 +78,24 @@ public void generateDatabase(HSQLDBGlobalState globalState) throws Exception { if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } - StatementExecutor se = new StatementExecutor<>(globalState, HSQLDBProvider.Action.values(), - HSQLDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); + StatementExecutor se = new StatementExecutor<>(globalState, + HSQLDBProvider.Action.values(), HSQLDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); se.executeStatements(); } private static int mapActions(HSQLDBProvider.HSQLDBGlobalState globalState, HSQLDBProvider.Action a) { Randomly r = globalState.getRandomly(); switch (a) { - case INSERT: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case UPDATE: - return r.getInteger(0, 10); - default: - throw new AssertionError(a); + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case UPDATE: + return r.getInteger(0, 10); + default: + throw new AssertionError(a); } } @@ -100,5 +108,3 @@ protected HSQLDBSchema readSchema() throws SQLException { } } - - diff --git a/src/sqlancer/hsqldb/HSQLDBSchema.java b/src/sqlancer/hsqldb/HSQLDBSchema.java index 13059ef3e..e1e1cb94b 100644 --- a/src/sqlancer/hsqldb/HSQLDBSchema.java +++ b/src/sqlancer/hsqldb/HSQLDBSchema.java @@ -1,5 +1,12 @@ package sqlancer.hsqldb; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.DBMSCommon; @@ -8,20 +15,13 @@ import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.TableIndex; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - public class HSQLDBSchema extends AbstractSchema { public HSQLDBSchema(List databaseTables) { super(databaseTables); } - public static HSQLDBSchema fromConnection(SQLConnection connection, String databaseName) throws SQLException { + public static HSQLDBSchema fromConnection(SQLConnection connection, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(connection); for (String tableName : tableNames) { @@ -43,7 +43,8 @@ public static HSQLDBSchema fromConnection(SQLConnection connection, String datab private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'PUBLIC';")) { + try (ResultSet rs = s + .executeQuery("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'PUBLIC';")) { while (rs.next()) { tableNames.add(rs.getString("TABLE_NAME")); } @@ -59,8 +60,9 @@ private static List getTableColumns(SQLConnection con, String tabl try (ResultSet rs = s.executeQuery(String.format(sql, tableName))) { while (rs.next()) { HSQLDBDataType dataType = HSQLDBDataType.from(rs.getString("TYPE_NAME")); - HSQLDBCompositeDataType compositeDataType = new HSQLDBCompositeDataType(dataType, rs.getInt("COLUMN_SIZE")); - HSQLDBColumn column = new HSQLDBColumn(rs.getString("COLUMN_NAME"), null, compositeDataType ); + HSQLDBCompositeDataType compositeDataType = new HSQLDBCompositeDataType(dataType, + rs.getInt("COLUMN_SIZE")); + HSQLDBColumn column = new HSQLDBColumn(rs.getString("COLUMN_NAME"), null, compositeDataType); tableNames.add(column); } } @@ -68,7 +70,8 @@ private static List getTableColumns(SQLConnection con, String tabl return tableNames; } - public static class HSQLDBTable extends AbstractRelationalTable { + public static class HSQLDBTable + extends AbstractRelationalTable { public HSQLDBTable(String tableName, List columns, boolean isView) { super(tableName, columns, Collections.emptyList(), isView); @@ -76,8 +79,8 @@ public HSQLDBTable(String tableName, List columns, bo } - public static class HSQLDBColumn extends AbstractTableColumn - { + public static class HSQLDBColumn + extends AbstractTableColumn { public HSQLDBColumn(String name, HSQLDBTable table, HSQLDBCompositeDataType type) { super(name, table, type); @@ -96,9 +99,9 @@ public static HSQLDBSchema.HSQLDBDataType getRandomWithoutNull() { return dt; } - public static HSQLDBDataType from(String type_name) { - for( HSQLDBDataType value : HSQLDBDataType.values()) { - if(value.name().equals(type_name)) { + public static HSQLDBDataType from(String typeName) { + for (HSQLDBDataType value : HSQLDBDataType.values()) { + if (value.name().equals(typeName)) { return value; } } @@ -106,7 +109,7 @@ public static HSQLDBDataType from(String type_name) { } } - public static class HSQLDBCompositeDataType{ + public static class HSQLDBCompositeDataType { private final int size; private final HSQLDBDataType type; @@ -115,28 +118,31 @@ public HSQLDBCompositeDataType(HSQLDBDataType type, int size) { this.size = size; } - public static HSQLDBCompositeDataType getRandomWithoutNull() { HSQLDBSchema.HSQLDBDataType type = HSQLDBSchema.HSQLDBDataType.getRandomWithoutNull(); + return getRandomWithType(type); + } + + public static HSQLDBCompositeDataType getRandomWithType(HSQLDBSchema.HSQLDBDataType type) { int size; switch (type) { - case VARCHAR: - case CHAR: - case TIME: - case BINARY: - case TIMESTAMP: - size = Randomly.fromOptions(4,6,8); - break; - case BOOLEAN: - case INTEGER: - case DOUBLE: - //case UUID: - //case OTHER: - case DATE: - size = 0; - break; - default: - throw new AssertionError(type); + case VARCHAR: + case CHAR: + case TIME: + case BINARY: + case TIMESTAMP: + size = Randomly.fromOptions(4, 6, 8); + break; + case BOOLEAN: + case INTEGER: + case DOUBLE: + // case UUID: + // case OTHER: + case DATE: + size = 0; + break; + default: + throw new AssertionError(type); } return new HSQLDBSchema.HSQLDBCompositeDataType(type, size); diff --git a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java index 66ccbfd12..722e241fa 100644 --- a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java +++ b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java @@ -9,7 +9,6 @@ public class HSQLDBToStringVisitor extends NewToStringVisitor { - @Override public void visitSpecific(Node expr) { if (expr instanceof HSQLDBConstant) { @@ -44,7 +43,7 @@ private void visit(HSQLDBJoin join) { visit(join.getOnCondition()); } } - + private void visit(HSQLDBConstant constant) { sb.append(constant.toString()); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java index d4126bd6e..f6eed5e13 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java @@ -1,13 +1,15 @@ package sqlancer.hsqldb.ast; -import sqlancer.common.ast.newast.Node; import java.sql.Timestamp; import java.text.SimpleDateFormat; +import sqlancer.common.ast.newast.Node; + public class HSQLDBConstant implements Node { private HSQLDBConstant() { } + public static class HSQLDBNullConstant extends HSQLDBConstant { @Override @@ -22,7 +24,7 @@ public static class HSQLDBIntConstant extends HSQLDBConstant { private final int value; public HSQLDBIntConstant(long value) { - this.value = (int)value; + this.value = (int) value; } @Override @@ -83,14 +85,13 @@ public String toString() { } - public static class HSQLDBBinaryConstant extends HSQLDBConstant { private final String value; public HSQLDBBinaryConstant(long value, int size) { StringBuilder hex = new StringBuilder(Long.toHexString(value)); - if(hex.length() < 2) { + if (hex.length() < 2) { hex.append('0'); } this.value = hex.substring(0, Math.min(hex.length(), size)); @@ -225,5 +226,4 @@ public static Node createBinaryConstant(long nonCachedInteger, return new HSQLDBBinaryConstant(nonCachedInteger, size); } - } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java index 524702839..ef715052d 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java @@ -1,5 +1,8 @@ package sqlancer.hsqldb.ast; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; @@ -8,9 +11,6 @@ import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -import java.util.ArrayList; -import java.util.List; - public class HSQLDBJoin implements Node { private final TableReferenceNode leftTable; @@ -36,8 +36,8 @@ public static OuterType getRandom() { } public HSQLDBJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; @@ -78,39 +78,40 @@ public static List> getJoins( columns.addAll(rightTable.getTable().getColumns()); HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(globalState).setColumns(columns); switch (HSQLDBJoin.JoinType.getRandom()) { - case INNER: - joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - case NATURAL: - joinExpressions.add(HSQLDBJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); - break; - case LEFT: - joinExpressions - .add(HSQLDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - case RIGHT: - joinExpressions - .add(HSQLDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - default: - throw new AssertionError(); - } + case INNER: + joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTable, rightTable, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case NATURAL: + joinExpressions.add(HSQLDBJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); + break; + case LEFT: + joinExpressions.add(HSQLDBJoin.createLeftOuterJoin(leftTable, rightTable, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case RIGHT: + joinExpressions.add(HSQLDBJoin.createRightOuterJoin(leftTable, rightTable, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + default: + throw new AssertionError(); + } } return joinExpressions; } public static HSQLDBJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new HSQLDBJoin(left, right, JoinType.RIGHT, predicate); } public static HSQLDBJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new HSQLDBJoin(left, right, JoinType.LEFT, predicate); } public static HSQLDBJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new HSQLDBJoin(left, right, JoinType.INNER, predicate); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java new file mode 100644 index 000000000..12f8137e0 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java @@ -0,0 +1,78 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { + + // private final Node expr; + // private final HSQLDBUnaryPostfixOperator op; + private boolean negate; + + public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op, boolean negate) { + super(expr, op); + // this.expr = expr; + // this.op = op; + setNegate(negate); + } + + public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op) { + super(expr, op); + // this.expr = expr; + // this.op = op; + } + + public enum HSQLDBUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public HSQLDBSchema.HSQLDBDataType[] getInputDataTypes() { + return HSQLDBSchema.HSQLDBDataType.values(); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public HSQLDBSchema.HSQLDBDataType[] getInputDataTypes() { + return HSQLDBSchema.HSQLDBDataType.values(); + } + }; + + private final String textRepresentations; + + HSQLDBUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static HSQLDBUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract HSQLDBSchema.HSQLDBDataType[] getInputDataTypes(); + + } + + public boolean isNegated() { + return negate; + } + + public void setNegate(boolean negate) { + this.negate = negate; + } + + public Node getExpression() { + return getExpr(); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java new file mode 100644 index 000000000..233b9e943 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java @@ -0,0 +1,83 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { + + // private final HSQLDBUnaryPrefixOperation.HSQLDBUnaryPrefixOperator operation; + // private final Node expression; + + public HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperator operation, Node expression) { + super(expression, operation); + // this.operation = operation; + // this.expression = expression; + } + + @Override + public String getOperatorRepresentation() { + return null; + } + + public enum HSQLDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", HSQLDBSchema.HSQLDBDataType.BOOLEAN, HSQLDBSchema.HSQLDBDataType.INTEGER) { + @Override + public HSQLDBSchema.HSQLDBDataType getExpressionType() { + return HSQLDBSchema.HSQLDBDataType.BOOLEAN; + } + + @Override + protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { + return null; // TODO + } + }, + + UNARY_PLUS("+", HSQLDBSchema.HSQLDBDataType.INTEGER) { + @Override + public HSQLDBSchema.HSQLDBDataType getExpressionType() { + return HSQLDBSchema.HSQLDBDataType.INTEGER; + } + + @Override + protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { + return expectedValue; + } + }, + UNARY_MINUS("-", HSQLDBSchema.HSQLDBDataType.INTEGER) { + @Override + public HSQLDBSchema.HSQLDBDataType getExpressionType() { + return HSQLDBSchema.HSQLDBDataType.INTEGER; + } + + @Override + protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { + return null; + } + }; + + private String textRepresentation; + private HSQLDBSchema.HSQLDBDataType[] dataTypes; + + HSQLDBUnaryPrefixOperator(String textRepresentation, HSQLDBSchema.HSQLDBDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract HSQLDBSchema.HSQLDBDataType getExpressionType(); + + public HSQLDBSchema.HSQLDBDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); + } + + protected abstract HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index d01454a7f..6e9be4fcf 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -1,34 +1,48 @@ package sqlancer.hsqldb.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBUnaryPostfixOperation; +import sqlancer.hsqldb.ast.HSQLDBUnaryPrefixOperation; +public final class HSQLDBExpressionGenerator extends + TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { -public final class HSQLDBExpressionGenerator extends TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { + private enum Expression { + BINARY_LOGICAL; + } HSQLDBProvider.HSQLDBGlobalState hsqldbGlobalState; + public HSQLDBExpressionGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { this.hsqldbGlobalState = globalState; } @Override public Node generatePredicate() { - return null; + return generateExpression( + HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithType(HSQLDBSchema.HSQLDBDataType.BOOLEAN)); } @Override public Node negatePredicate(Node predicate) { - return null; + return new HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperation.HSQLDBUnaryPrefixOperator.NOT, predicate); } @Override public Node isNull(Node expr) { - return null; + return new HSQLDBUnaryPostfixOperation(expr, HSQLDBUnaryPostfixOperation.HSQLDBUnaryPostfixOperator.IS_NULL); } @Override @@ -37,37 +51,52 @@ public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataT return HSQLDBConstant.createNullConstant(); } switch (type.getType()) { - case CHAR: - return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getAlphabeticChar(), type.getSize()); - case VARCHAR: - return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getString(), type.getSize()); - case TIME: - return HSQLDBConstant.createTimeConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); - case TIMESTAMP: - return HSQLDBConstant.createTimestampConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize() ); - - case INTEGER: - return HSQLDBConstant.HSQLDBIntConstant.createIntConstant(Randomly.getNonCachedInteger()); - case DOUBLE: - return HSQLDBConstant.HSQLDBDoubleConstant.createFloatConstant(hsqldbGlobalState.getRandomly().getDouble()); - case BOOLEAN: - return HSQLDBConstant.HSQLDBBooleanConstant.createBooleanConstant(Randomly.getBoolean()); - case DATE: - return HSQLDBConstant.createDateConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis())); - case BINARY: - return HSQLDBConstant.createBinaryConstant(Randomly.getNonCachedInteger(), type.getSize()); -// case OTHER: -// break; -// case NULL: -// break; - default: - throw new AssertionError("Unknown type: " + type); + case CHAR: + return HSQLDBConstant.HSQLDBTextConstant + .createStringConstant(hsqldbGlobalState.getRandomly().getAlphabeticChar(), type.getSize()); + case VARCHAR: + return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getString(), + type.getSize()); + case TIME: + return HSQLDBConstant.createTimeConstant( + hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); + case TIMESTAMP: + return HSQLDBConstant.createTimestampConstant( + hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); + + case INTEGER: + return HSQLDBConstant.HSQLDBIntConstant.createIntConstant(Randomly.getNonCachedInteger()); + case DOUBLE: + return HSQLDBConstant.HSQLDBDoubleConstant.createFloatConstant(hsqldbGlobalState.getRandomly().getDouble()); + case BOOLEAN: + return HSQLDBConstant.HSQLDBBooleanConstant.createBooleanConstant(Randomly.getBoolean()); + case DATE: + return HSQLDBConstant + .createDateConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis())); + case BINARY: + return HSQLDBConstant.createBinaryConstant(Randomly.getNonCachedInteger(), type.getSize()); + default: + throw new AssertionError("Unknown type: " + type); } } @Override protected Node generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { - return null; + if (depth >= hsqldbGlobalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } + + List possibleOptions = new ArrayList<>( + Arrays.asList(HSQLDBExpressionGenerator.Expression.values())); + + HSQLDBExpressionGenerator.Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case BINARY_LOGICAL: + BinaryOperatorNode.Operator op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(type), generateExpression(type), op); + default: + throw new AssertionError(); + } } @Override @@ -85,4 +114,18 @@ protected boolean canGenerateColumnOfType(HSQLDBSchema.HSQLDBCompositeDataType t return columns.stream().anyMatch(c -> c.getType() == type); } + public enum HSQLDBBinaryLogicalOperator implements BinaryOperatorNode.Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index e53b67d44..c21d7c786 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -1,4 +1,8 @@ package sqlancer.hsqldb.gen; + +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; @@ -8,9 +12,6 @@ import sqlancer.hsqldb.HSQLDBToStringVisitor; import sqlancer.hsqldb.ast.HSQLDBExpression; -import java.util.List; -import java.util.stream.Collectors; - public class HSQLDBInsertGenerator extends AbstractInsertGenerator { private final HSQLDBProvider.HSQLDBGlobalState globalState; @@ -34,13 +35,14 @@ private SQLQueryAdapter generate() { sb.append(")"); sb.append(" VALUES "); insertColumns(columns); - //HSQLDBErrors.addInsertErrors(errors); + // HSQLDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } @Override protected void insertValue(HSQLDBSchema.HSQLDBColumn column) { - Node expression = new HSQLDBExpressionGenerator(globalState).generateConstant(column.getType()); + Node expression = new HSQLDBExpressionGenerator(globalState) + .generateConstant(column.getType()); String s = HSQLDBToStringVisitor.asString(expression); sb.append(s); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java index 58b1cc426..48606e9bf 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java @@ -1,15 +1,15 @@ package sqlancer.hsqldb.gen; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; + import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; - public class HSQLDBTableGenerator { public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @Nullable String tableName) { @@ -25,7 +25,7 @@ public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @N } sb.append(name); sb.append("("); - List columns = getNewColumns(name); + List columns = getNewColumns(); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); @@ -33,8 +33,8 @@ public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @N sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType().getType().name()); - if(columns.get(i).getType().getSize() > 0) { - //Cannot specify size for non composite data types + if (columns.get(i).getType().getSize() > 0) { + // Cannot specify size for non composite data types sb.append("("); sb.append(columns.get(i).getType().getSize()); sb.append(")"); @@ -45,11 +45,12 @@ public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @N return new SQLQueryAdapter(sb.toString(), errors, true); } - private static List getNewColumns(String tableName) { + private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { String columnName = String.format("c%d", i); - HSQLDBSchema.HSQLDBCompositeDataType columnType = HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull(); + HSQLDBSchema.HSQLDBCompositeDataType columnType = HSQLDBSchema.HSQLDBCompositeDataType + .getRandomWithoutNull(); columns.add(new HSQLDBSchema.HSQLDBColumn(columnName, null, columnType)); } return columns; diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index 97b8f6e18..7963aca90 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -1,5 +1,7 @@ package sqlancer.hsqldb.gen; +import java.util.List; + import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -8,12 +10,11 @@ import sqlancer.hsqldb.HSQLDBToStringVisitor; import sqlancer.hsqldb.ast.HSQLDBExpression; -import java.util.List; - public final class HSQLDBUpdateGenerator { - private final static ExpectedErrors errors = new ExpectedErrors(); + private static final ExpectedErrors EXPECTED_ERRORS = new ExpectedErrors(); private final HSQLDBProvider.HSQLDBGlobalState globalState; + private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { this.globalState = globalState; } @@ -36,15 +37,15 @@ private SQLQueryAdapter getQuery() { sb.append(columns.get(i).getName()); sb.append("="); Node expr; -// if (Randomly.getBooleanWithSmallProbability()) { -// expr = gen.generateExpression(columns.get(i).getType()); -// HSQLDBErrors.addExpressionErrors(errors); -// } else { - expr = gen.generateConstant(columns.get(i).getType()); - //} + // if (Randomly.getBooleanWithSmallProbability()) { + // expr = gen.generateExpression(columns.get(i).getType()); + // HSQLDBErrors.addExpressionErrors(errors); + // } else { + expr = gen.generateConstant(columns.get(i).getType()); + // } sb.append(HSQLDBToStringVisitor.asString(expr)); } - return new SQLQueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), EXPECTED_ERRORS); } } diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java index 69216dca7..d6d3d553e 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -1,13 +1,33 @@ package sqlancer.hsqldb.test; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; + +public class HSQLDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> + implements TestOracle { + + HSQLDBSelect select; + HSQLDBExpressionGenerator expressionGenerator; -public class HSQLDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> implements TestOracle { + HSQLDBSchema schema; + + List targetTables; public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { super(state); @@ -15,11 +35,36 @@ public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { @Override protected ExpressionGenerator> getGen() { - return null; + return expressionGenerator; } @Override public void check() throws Exception { + schema = state.getSchema(); + targetTables = schema.getDatabaseTablesRandomSubsetNotEmpty(); + expressionGenerator = new HSQLDBExpressionGenerator(state) + .setColumns(targetTables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList())); + initializeTernaryPredicateVariants(); + select = new HSQLDBSelect(); + select.setFetchColumns(generateFetchColumns()); + List> tableList = targetTables.stream() + .map(t -> new TableReferenceNode(t)) + .collect(Collectors.toList()); + List> joins = HSQLDBJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new HSQLDBSchema.HSQLDBColumn("*", null, null))); + } else { + columns = Randomly.nonEmptySubset(targetTables.stream().flatMap(t -> t.getColumns().stream()) + .map(c -> new ColumnReferenceNode(c)) + .collect(Collectors.toList())); + } + return columns; } } diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java new file mode 100644 index 000000000..09f51051d --- /dev/null +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java @@ -0,0 +1,43 @@ +package sqlancer.hsqldb.test; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.duckdb.test.DuckDBQueryPartitioningBase; +import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBToStringVisitor; + +public class HSQLDBQueryPartitioningWhereTester extends HSQLDBQueryPartitioningBase { + + public HSQLDBQueryPartitioningWhereTester(HSQLDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + select.setWhereClause(null); + String originalQueryString = HSQLDBToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByExpressions(expressionGenerator.generateOrderBys()); + } + select.setWhereClause(predicate); + String firstQueryString = HSQLDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = HSQLDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = HSQLDBToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + } + +} From e68539d42b68e400cfc5bfc4d972e8d948aee531 Mon Sep 17 00:00:00 2001 From: Nishita Date: Mon, 28 Nov 2022 18:30:53 +0530 Subject: [PATCH 0670/1284] Adds Binary Comparsion and Binary arithmetic expressions --- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 58 ++++++++++++++++++- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index 6e9be4fcf..ec8a26b76 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -20,7 +20,7 @@ public final class HSQLDBExpressionGenerator extends TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { private enum Expression { - BINARY_LOGICAL; + BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC; } HSQLDBProvider.HSQLDBGlobalState hsqldbGlobalState; @@ -90,13 +90,23 @@ protected Node generateExpression(HSQLDBSchema.HSQLDBComposite Arrays.asList(HSQLDBExpressionGenerator.Expression.values())); HSQLDBExpressionGenerator.Expression expr = Randomly.fromList(possibleOptions); + BinaryOperatorNode.Operator op; switch (expr) { case BINARY_LOGICAL: - BinaryOperatorNode.Operator op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(type), generateExpression(type), op); + op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); + break; + case BINARY_COMPARISON: + op = HSQLDBDBBinaryComparisonOperator.getRandom(); + break; + case BINARY_ARITHMETIC: + op = HSQLDBDBBinaryArithmeticOperator.getRandom(); + break; default: throw new AssertionError(); } + + return new NewBinaryOperatorNode<>(generateExpression(type), generateExpression(type), op); + } @Override @@ -128,4 +138,46 @@ public static BinaryOperatorNode.Operator getRandom() { } } + + public enum HSQLDBDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), + REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); + + private String textRepr; + + HSQLDBDBBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum HSQLDBDBBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); + + private String textRepr; + + HSQLDBDBBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } } From 0c037dda2be1150c1ae19e72d40cee92dea51517 Mon Sep 17 00:00:00 2001 From: Nishita Date: Mon, 28 Nov 2022 18:43:02 +0530 Subject: [PATCH 0671/1284] Fixes failing test --- src/sqlancer/hsqldb/HSQLDBOptions.java | 2 ++ src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index 54954b797..d025d4576 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -4,12 +4,14 @@ import java.util.List; import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; import sqlancer.hsqldb.test.HSQLDBQueryPartitioningWhereTester; +@Parameters(commandDescription = "hsqldb") public class HSQLDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index ec8a26b76..1db82eca9 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -94,8 +94,8 @@ protected Node generateExpression(HSQLDBSchema.HSQLDBComposite switch (expr) { case BINARY_LOGICAL: op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); - break; - case BINARY_COMPARISON: + break; + case BINARY_COMPARISON: op = HSQLDBDBBinaryComparisonOperator.getRandom(); break; case BINARY_ARITHMETIC: From b423e07335cf5d35a427b92508f7228890d5c507 Mon Sep 17 00:00:00 2001 From: Nishita Dutta Date: Tue, 29 Nov 2022 23:21:45 +0530 Subject: [PATCH 0672/1284] Expressions for SELECT (#1) Implements expressions for select clause in HSQLDB --- .../hsqldb/ast/HSQLDBColumnReference.java | 11 ++++++ .../ast/HSQLDBUnaryPostfixOperation.java | 21 ---------- .../ast/HSQLDBUnaryPrefixOperation.java | 31 +-------------- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 39 +++++++++++++------ .../hsqldb/gen/HSQLDBUpdateGenerator.java | 14 ++++--- .../test/HSQLDBQueryPartitioningBase.java | 11 ++++-- .../HSQLDBQueryPartitioningWhereTester.java | 4 +- 7 files changed, 56 insertions(+), 75 deletions(-) create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java diff --git a/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java new file mode 100644 index 000000000..b2179eca3 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBColumnReference extends ColumnReferenceNode { + + public HSQLDBColumnReference(HSQLDBSchema.HSQLDBColumn column) { + super(column); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java index 12f8137e0..9e63efee5 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java @@ -8,21 +8,8 @@ public class HSQLDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { - // private final Node expr; - // private final HSQLDBUnaryPostfixOperator op; - private boolean negate; - - public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op, boolean negate) { - super(expr, op); - // this.expr = expr; - // this.op = op; - setNegate(negate); - } - public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op) { super(expr, op); - // this.expr = expr; - // this.op = op; } public enum HSQLDBUnaryPostfixOperator implements BinaryOperatorNode.Operator { @@ -58,14 +45,6 @@ public String getTextRepresentation() { } - public boolean isNegated() { - return negate; - } - - public void setNegate(boolean negate) { - this.negate = negate; - } - public Node getExpression() { return getExpr(); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java index 233b9e943..b1327d0be 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java @@ -1,6 +1,5 @@ package sqlancer.hsqldb.ast; -import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; @@ -8,18 +7,13 @@ public class HSQLDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { - // private final HSQLDBUnaryPrefixOperation.HSQLDBUnaryPrefixOperator operation; - // private final Node expression; - public HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperator operation, Node expression) { super(expression, operation); - // this.operation = operation; - // this.expression = expression; } @Override public String getOperatorRepresentation() { - return null; + return this.op.getTextRepresentation(); } public enum HSQLDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { @@ -28,11 +22,6 @@ public enum HSQLDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { public HSQLDBSchema.HSQLDBDataType getExpressionType() { return HSQLDBSchema.HSQLDBDataType.BOOLEAN; } - - @Override - protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { - return null; // TODO - } }, UNARY_PLUS("+", HSQLDBSchema.HSQLDBDataType.INTEGER) { @@ -40,40 +29,22 @@ protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { public HSQLDBSchema.HSQLDBDataType getExpressionType() { return HSQLDBSchema.HSQLDBDataType.INTEGER; } - - @Override - protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { - return expectedValue; - } }, UNARY_MINUS("-", HSQLDBSchema.HSQLDBDataType.INTEGER) { @Override public HSQLDBSchema.HSQLDBDataType getExpressionType() { return HSQLDBSchema.HSQLDBDataType.INTEGER; } - - @Override - protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { - return null; - } }; private String textRepresentation; - private HSQLDBSchema.HSQLDBDataType[] dataTypes; HSQLDBUnaryPrefixOperator(String textRepresentation, HSQLDBSchema.HSQLDBDataType... dataTypes) { this.textRepresentation = textRepresentation; - this.dataTypes = dataTypes.clone(); } public abstract HSQLDBSchema.HSQLDBDataType getExpressionType(); - public HSQLDBSchema.HSQLDBDataType getRandomInputDataTypes() { - return Randomly.fromOptions(dataTypes); - } - - protected abstract HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue); - @Override public String getTextRepresentation() { return this.textRepresentation; diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index 1db82eca9..a978dfdc3 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; @@ -11,6 +12,7 @@ import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; import sqlancer.hsqldb.ast.HSQLDBUnaryPostfixOperation; @@ -47,10 +49,9 @@ public Node isNull(Node expr) { @Override public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataType type) { - if (type.getType() == HSQLDBSchema.HSQLDBDataType.NULL || Randomly.getBooleanWithSmallProbability()) { - return HSQLDBConstant.createNullConstant(); - } switch (type.getType()) { + case NULL: + return HSQLDBConstant.createNullConstant(); case CHAR: return HSQLDBConstant.HSQLDBTextConstant .createStringConstant(hsqldbGlobalState.getRandomly().getAlphabeticChar(), type.getSize()); @@ -82,7 +83,8 @@ public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataT @Override protected Node generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { - if (depth >= hsqldbGlobalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + if (depth >= hsqldbGlobalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithSmallProbability()) { return generateLeafNode(type); } @@ -93,25 +95,26 @@ protected Node generateExpression(HSQLDBSchema.HSQLDBComposite BinaryOperatorNode.Operator op; switch (expr) { case BINARY_LOGICAL: + case BINARY_ARITHMETIC: op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); break; case BINARY_COMPARISON: op = HSQLDBDBBinaryComparisonOperator.getRandom(); break; - case BINARY_ARITHMETIC: - op = HSQLDBDBBinaryArithmeticOperator.getRandom(); - break; default: throw new AssertionError(); } - return new NewBinaryOperatorNode<>(generateExpression(type), generateExpression(type), op); + return new NewBinaryOperatorNode<>(generateExpression(type, depth + 1), generateExpression(type, depth + 1), + op); } @Override protected Node generateColumn(HSQLDBSchema.HSQLDBCompositeDataType type) { - return null; + HSQLDBSchema.HSQLDBColumn column = Randomly + .fromList(columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList())); + return new HSQLDBColumnReference(column); } @Override @@ -140,9 +143,7 @@ public static BinaryOperatorNode.Operator getRandom() { } public enum HSQLDBDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), - LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), - REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); private String textRepr; @@ -180,4 +181,18 @@ public String getTextRepresentation() { } } + + @Override + public List> generateOrderBys() { + List> expressions = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + ArrayList hsqldbColumns = new ArrayList<>(columns); + for (int i = 0; i < nr && !hsqldbColumns.isEmpty(); i++) { + HSQLDBSchema.HSQLDBColumn randomColumn = Randomly.fromList(hsqldbColumns); + HSQLDBColumnReference columnReference = new HSQLDBColumnReference(randomColumn); + hsqldbColumns.remove(randomColumn); + expressions.add(columnReference); + } + return expressions; + } } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index 7963aca90..b4b419b45 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -2,6 +2,7 @@ import java.util.List; +import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -37,12 +38,13 @@ private SQLQueryAdapter getQuery() { sb.append(columns.get(i).getName()); sb.append("="); Node expr; - // if (Randomly.getBooleanWithSmallProbability()) { - // expr = gen.generateExpression(columns.get(i).getType()); - // HSQLDBErrors.addExpressionErrors(errors); - // } else { - expr = gen.generateConstant(columns.get(i).getType()); - // } + if (Randomly.getBooleanWithSmallProbability()) { + sb.append(" WHERE "); + expr = gen.generateExpression(columns.get(i).getType()); + // HSQLDBErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(columns.get(i).getType()); + } sb.append(HSQLDBToStringVisitor.asString(expr)); } return new SQLQueryAdapter(sb.toString(), EXPECTED_ERRORS); diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java index d6d3d553e..ec501ab5e 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; @@ -13,8 +14,9 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBToStringVisitor; import sqlancer.hsqldb.ast.HSQLDBExpression; -import sqlancer.hsqldb.ast.HSQLDBJoin; +//import sqlancer.hsqldb.ast.HSQLDBJoin; import sqlancer.hsqldb.ast.HSQLDBSelect; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; @@ -50,10 +52,13 @@ public void check() throws Exception { List> tableList = targetTables.stream() .map(t -> new TableReferenceNode(t)) .collect(Collectors.toList()); - List> joins = HSQLDBJoin.getJoins(tableList, state); - select.setJoinList(joins.stream().collect(Collectors.toList())); + // List> joins = HSQLDBJoin.getJoins(tableList, state); + // select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); + + ComparatorHelper.getResultSetFirstColumnAsString(HSQLDBToStringVisitor.asString(select), errors, state); + } List> generateFetchColumns() { diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java index 09f51051d..0b1706583 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java @@ -5,7 +5,6 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.duckdb.test.DuckDBQueryPartitioningBase; import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; import sqlancer.hsqldb.HSQLDBToStringVisitor; @@ -18,7 +17,6 @@ public HSQLDBQueryPartitioningWhereTester(HSQLDBGlobalState state) { @Override public void check() throws Exception { super.check(); - select.setWhereClause(null); String originalQueryString = HSQLDBToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); @@ -37,7 +35,7 @@ public void check() throws Exception { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state); } } From a601c52aa58e452e2ec31e62d31c9cf26a0438ce Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 30 Nov 2022 09:08:38 +0800 Subject: [PATCH 0673/1284] fix: test databend with github actions service image --- .github/workflows/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 67ade8e58..6ae3bd74e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -120,6 +120,15 @@ jobs: databend: name: DBMS Tests (Databend) runs-on: ubuntu-latest + services: + databend: + image: datafuselabs/databend + env: + DATABEND_DEFAULT_USER: sqlancer + DATABEND_DEFAULT_PASSWORD: sqlancer + ports: + - 8000:8000 + - 3307:3307 steps: - uses: actions/checkout@v2 with: @@ -130,14 +139,6 @@ jobs: java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up Databend - run: | - sudo apt update - sudo apt install mysql-client - docker pull datafuselabs/databend-query - docker run --net=host datafuselabs/databend-query & - - name: Create SQLancer user - run: mysql -uroot -h127.0.0.1 -P3307 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" - name: Run Tests run: | DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test From 0c2ed8c09c7fad29c6e125a49770e74a47e8f954 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 05:57:11 +0000 Subject: [PATCH 0674/1284] build(deps): bump checkstyle from 10.3.4 to 10.5.0 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3.4 to 10.5.0. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3.4...checkstyle-10.5.0) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ffee5dfd9..950ea4243 100644 --- a/pom.xml +++ b/pom.xml @@ -209,7 +209,7 @@ com.puppycrawl.tools checkstyle - 10.3.4 + 10.5.0 From f09e98f4dad567a61ec2dca97062bb9d2a13a3ff Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Thu, 1 Dec 2022 20:37:07 +0800 Subject: [PATCH 0675/1284] chore(databend):code tidying and local optimization --- src/sqlancer/ProviderAdapter.java | 8 +--- src/sqlancer/databend/DatabendErrors.java | 11 +++-- src/sqlancer/databend/DatabendProvider.java | 8 ++-- src/sqlancer/databend/DatabendSchema.java | 10 +---- .../gen/DatabendRandomQuerySynthesizer.java | 43 +++++++++++-------- .../databend/gen/DatabendTableGenerator.java | 30 ++----------- .../databend/gen/DatabendViewGenerator.java | 9 ---- .../databend/test/DatabendNoRECOracle.java | 24 ++++------- test/sqlancer/dbms/TestDatabend.java | 4 +- 9 files changed, 53 insertions(+), 94 deletions(-) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index a16f8388c..46475f260 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -45,12 +45,8 @@ public void generateAndTestDatabase(G globalState) throws Exception { for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { assert localState != null; - try { - oracle.check(); - globalState.getManager().incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - - } + oracle.check(); + globalState.getManager().incrementSelectQueryCount(); assert localState != null; localState.executedWithoutError(); } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 7e1f80c4e..ab9724a5f 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -9,24 +9,29 @@ private DatabendErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Division by zero"); - + errors.add("/ by zero"); + errors.add("ORDER BY position"); + errors.add("GROUP BY position"); /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE * databend2; CREATE TABLE t0(c0VARCHAR VARCHAR NULL, c1VARCHAR VARCHAR NULL, c2FLOAT FLOAT NOT NULL - * DEFAULT(NULL)); CREATE TABLE t1(c0INT BIGINT NULL); INSERT INTO t0(c1varchar, c0varchar) VALUES ('067596', - * '19'), ('', '87'); + * DEFAULT(NULL)); CREATE TABLE t1(c0INT BIGINT NULL); INSERT INTO t0(c1varchar, c0varchar) VALUES + * ('067596','19'), ('', '87'); */ errors.add("Can't cast column from null into non-nullable type"); } public static void addInsertErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); } public static void addGroupByErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); + errors.add("GROUP BY position"); } } diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 9a94241d0..0ce3b7695 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -23,6 +23,7 @@ import sqlancer.databend.gen.DatabendInsertGenerator; import sqlancer.databend.gen.DatabendRandomQuerySynthesizer; import sqlancer.databend.gen.DatabendTableGenerator; +import sqlancer.databend.gen.DatabendViewGenerator; @AutoService(DatabaseProvider.class) public class DatabendProvider extends SQLProviderAdapter { @@ -38,8 +39,7 @@ public enum Action implements AbstractAction { // DELETE(DatabendDeleteGenerator::generate), // // UPDATE(DatabendUpdateGenerator::getQuery), // - // CREATE_VIEW(DatabendViewGenerator::generate), //TODO 等待databend的create view语法 更加贴近mysql - EXPLAIN((g) -> { + CREATE_VIEW(DatabendViewGenerator::generate), EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); DatabendErrors.addGroupByErrors(errors); @@ -73,8 +73,8 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); // case DELETE: // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); - // case CREATE_VIEW: //TODO 暂时关闭create view - // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + case CREATE_VIEW: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 98d7acafd..0e12689ee 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -10,7 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.DBMSCommon; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; @@ -237,9 +236,6 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con, databaseName); for (String tableName : tableNames) { - if (DBMSCommon.matchesIndexName(tableName)) { - continue; // TODO: unexpected? - } List databaseColumns = getTableColumns(con, tableName, databaseName); boolean isView = tableName.startsWith("v"); DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); @@ -292,11 +288,7 @@ private static List getTableColumns(SQLConnection con, String ta } } } - // if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { - // TODO: implement an option to enable/disable rowids - // columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, - // false)); - // } + return columns; } diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 286764c6e..70422b324 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -5,10 +5,12 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.ast.DatabendConstant; @@ -23,38 +25,42 @@ private DatabendRandomQuerySynthesizer() { public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); - DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState) - .setColumns(targetTables.getColumns()); + List targetColumns = targetTables.getColumns(); + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(targetColumns); DatabendSelect select = new DatabendSelect(); // TODO distinct - // select.setDistinct(Randomly.getBoolean()); + select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); List> columns = new ArrayList<>(); + int freeColumns = targetColumns.size(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); + Node expression = null; + if (freeColumns > 0 && Randomly.getBoolean()) { + expression = new ColumnReferenceNode<>(targetColumns.get(freeColumns - 1)); + freeColumns -= 1; + } else { + expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); + } columns.add(expression); - // } else { - // columns.add(gen()); - // } } select.setFetchColumns(columns); List tables = targetTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); List> joins = DatabendJoin.getJoins(tableList, globalState); - select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setJoinList(joins); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - // if (Randomly.getBoolean()) {//TODO order by超过实际行数 - // select.setOrderByExpressions(gen.generateOrderBys()); - // } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } - // if (Randomly.getBoolean()) { //TODO group by超过实际行数 - // select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - // } + if (Randomly.getBoolean()) { // TODO 该表达式生成的类型较多，需要对其限制 + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } if (Randomly.getBoolean()) { select.setLimitClause( @@ -64,10 +70,11 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setOffsetClause( DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } - // TODO 待添加HavingClause - // if (Randomly.getBoolean()) { - // select.setHavingClause(gen.generateHavingClause()); - // } + + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + return select; } diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index c83cd93e9..32505f98e 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -34,27 +34,14 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType()); - // if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() - // && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { - // sb.append(" COLLATE "); - // sb.append(getRandomCollate()); - // } - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" UNIQUE"); - // } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); } else { sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 } - // if (globalState.getDbmsSpecificOptions().testCheckConstraints //databend 无check约束 - // && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" CHECK("); - // sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); - // DatabendErrors.addExpressionErrors(errors); - // sb.append(")"); - // } + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 @@ -62,22 +49,11 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(")"); } } - // databend并没有索引 - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { - // errors.add("Invalid type for index"); - // List primaryKeyColumns = Randomly.nonEmptySubset(columns); - // sb.append(", PRIMARY KEY("); - // sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - // sb.append(")"); - // } + sb.append(")"); return new SQLQueryAdapter(sb.toString(), errors, true); } - public static String getRandomCollate() { - return Randomly.fromOptions("NOCASE", "NOACCENT", "NOACCENT.NOCASE", "C", "POSIX"); - } - private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { diff --git a/src/sqlancer/databend/gen/DatabendViewGenerator.java b/src/sqlancer/databend/gen/DatabendViewGenerator.java index 55cc6ffc5..75ad57239 100644 --- a/src/sqlancer/databend/gen/DatabendViewGenerator.java +++ b/src/sqlancer/databend/gen/DatabendViewGenerator.java @@ -17,15 +17,6 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { StringBuilder sb = new StringBuilder("CREATE "); sb.append("VIEW "); sb.append(globalState.getSchema().getFreeViewName()); - // sb.append("("); - // for (int i = 0; i < nrColumns; i++) { - // if (i != 0) { - // sb.append(", "); - // } - // sb.append("c"); - // sb.append(i); - // } - // sb.append(") AS "); sb.append(" AS "); sb.append(DatabendToStringVisitor .asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index cb27fa78b..c6d9c9229 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -3,7 +3,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -48,15 +47,14 @@ public void check() throws SQLException { DatabendTables randomTables = s.getRandomTableNonEmptyTables(); // 随机获得nr张表 List columns = randomTables.getColumns(); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - - Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件，形式为ast - + Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件 List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); List> joins = DatabendJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); // 禁用优化 - int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, + int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), + randomWhereCondition, joins); + int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { throw new IgnoreMeException(); @@ -67,23 +65,20 @@ public void check() throws SQLException { } } - private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + private int getUnoptimizedQueryCount(List> tableList, + Node randomWhereCondition, List> joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); - // DatabendExpression isTrue = DatabendPostfixOperation.create(randomWhereCondition, - // PostfixOperator.IS_TRUE); Node asText = new NewPostfixTextNode<>(new DatabendCastOperation( new NewPostfixTextNode(randomWhereCondition, " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); - select.setFetchColumns(Arrays.asList(asText)); // ? + select.setFetchColumns(List.of(asText)); select.setFromList(tableList); select.setJoinList(joins); int secondCount = 0; unoptimizedQueryString = "SELECT SUM(count) FROM (" + DatabendToStringVisitor.asString(select) + ") as res"; - errors.add("canceling statement due to statement timeout"); SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { @@ -101,17 +96,14 @@ private int getSecondQuery(List> tableList, Node> tableList, + private int getOptimizedQueryCount(SQLConnection con, List> tableList, List columns, Node randomWhereCondition, List> joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); - // DatabendAggregate aggr = new DatabendAggregate( List> allColumns = columns.stream() .map((c) -> new ColumnReferenceNode(c)) .collect(Collectors.toList()); - // DatabendAggregateFunction.COUNT); - // select.setFetchColumns(Arrays.asList(aggr)); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index 326a43283..0c1755a54 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -17,7 +17,7 @@ public void testDatabendNoREC() { assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); } @@ -29,7 +29,7 @@ public void testDatabendTLPQueryPartitioning() { assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); } From 4d9abd417db61a14f03f584b9e0d82255624faac Mon Sep 17 00:00:00 2001 From: Nishita Dutta Date: Fri, 2 Dec 2022 17:00:15 +0530 Subject: [PATCH 0676/1284] Implement NoREC Oracle and fixed UPDATE statements for HSQLDB (#2) * First commit for HSQLDB * Added Update statements * Fixes Checkstyle violations and adds support for expression * Adds Binary Comparsion and Binary arithmetic expressions * Fixes failing test * Fixes where and select expressions as per hsqldb * Fixes Checkstyle violations and adds support for expression * Formatting fixes * Implement HSQLDB Select Expressions * Fix UPDATE statements and implement NoREC --- src/sqlancer/hsqldb/HSQLDBOptions.java | 10 +- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 12 +- .../hsqldb/test/HSQLDBNoRECOracle.java | 124 ++++++++++++++++++ 3 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index d025d4576..9cf6dc228 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -9,13 +9,15 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; +import sqlancer.hsqldb.test.HSQLDBNoRECOracle; import sqlancer.hsqldb.test.HSQLDBQueryPartitioningWhereTester; @Parameters(commandDescription = "hsqldb") public class HSQLDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") - public List oracle = List.of(HSQLDBOracleFactory.WHERE); + public List oracle = List.of(HSQLDBOracleFactory.WHERE, + HSQLDBOracleFactory.NOREC); public enum HSQLDBOracleFactory implements OracleFactory { WHERE { @@ -23,6 +25,12 @@ public enum HSQLDBOracleFactory implements OracleFactory columns = table.getRandomNonEmptyColumnSubset(); for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -38,14 +40,16 @@ private SQLQueryAdapter getQuery() { sb.append(columns.get(i).getName()); sb.append("="); Node expr; + expr = gen.generateConstant(columns.get(i).getType()); + sb.append(HSQLDBToStringVisitor.asString(expr)); if (Randomly.getBooleanWithSmallProbability()) { sb.append(" WHERE "); expr = gen.generateExpression(columns.get(i).getType()); - // HSQLDBErrors.addExpressionErrors(errors); - } else { - expr = gen.generateConstant(columns.get(i).getType()); + sb.append(HSQLDBToStringVisitor.asString(expr)); + errors.add("data type of expression is not boolean"); + HSQLDBErrors.addExpressionErrors(errors); } - sb.append(HSQLDBToStringVisitor.asString(expr)); + } return new SQLQueryAdapter(sb.toString(), EXPECTED_ERRORS); } diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java new file mode 100644 index 000000000..7de1918b4 --- /dev/null +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -0,0 +1,124 @@ +package sqlancer.hsqldb.test; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.hsqldb.HSQLDBErrors; +import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBCompositeDataType; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBDataType; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBColumnReference; +import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; + +public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { + + private final HSQLDBSchema s; + + public HSQLDBNoRECOracle(HSQLDBGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + HSQLDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + List tables = s.getDatabaseTablesRandomSubsetNotEmpty(); + List columns = tables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList()); + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state).setColumns(columns); + + Node randomWhereCondition = gen + .generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)); + + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = HSQLDBJoin.getJoins(tableList, state); + int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); // 禁用优化 + int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getSecondQuery(List> tableList, Node randomWhereCondition, + List> joins) throws SQLException { + HSQLDBSelect select = new HSQLDBSelect(); + HSQLDBColumn c = new HSQLDBColumn("COUNT(*)", null, null); + select.setFetchColumns(List.of(new HSQLDBColumnReference(c))); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + select.setJoinList(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + HSQLDBToStringVisitor.asString(select) + ") as res"; + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, List> joins) + throws SQLException { + HSQLDBSelect select = new HSQLDBSelect(); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new HSQLDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + select.setJoinList(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = HSQLDBToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} From 7560fb632dd07473a21225ef63d0138fe5dbac47 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 4 Dec 2022 16:07:58 +0800 Subject: [PATCH 0677/1284] feature(databend): add delete generator Signed-off-by: Yisong Han --- src/sqlancer/databend/DatabendProvider.java | 23 +++++++-------- .../databend/gen/DatabendDeleteGenerator.java | 29 +++++++++++++++++++ .../gen/DatabendNewExpressionGenerator.java | 2 +- ...abendQueryPartitioningAggregateTester.java | 21 +++++++------- 4 files changed, 50 insertions(+), 25 deletions(-) create mode 100644 src/sqlancer/databend/gen/DatabendDeleteGenerator.java diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 0ce3b7695..f22a7cec9 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -20,6 +20,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.gen.DatabendDeleteGenerator; import sqlancer.databend.gen.DatabendInsertGenerator; import sqlancer.databend.gen.DatabendRandomQuerySynthesizer; import sqlancer.databend.gen.DatabendTableGenerator; @@ -34,12 +35,12 @@ public DatabendProvider() { public enum Action implements AbstractAction { - INSERT(DatabendInsertGenerator::getQuery), // - // TODO 等待databend实现update && delete - // DELETE(DatabendDeleteGenerator::generate), // + INSERT(DatabendInsertGenerator::getQuery), + DELETE(DatabendDeleteGenerator::generate), + // TODO 等待databend实现update // UPDATE(DatabendUpdateGenerator::getQuery), // - - CREATE_VIEW(DatabendViewGenerator::generate), EXPLAIN((g) -> { + CREATE_VIEW(DatabendViewGenerator::generate), + EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); DatabendErrors.addGroupByErrors(errors); @@ -71,8 +72,8 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { // TODO 等待databend实现update && delete // case UPDATE: // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); - // case DELETE: - // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); case CREATE_VIEW: return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: @@ -91,7 +92,7 @@ protected DatabendSchema readSchema() throws SQLException { @Override public void generateDatabase(DatabendGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + for (int i = 0; i < Randomly.fromOptions(3, 4); i++) { boolean success; do { SQLQueryAdapter qt = new DatabendTableGenerator().getQuery(globalState); @@ -107,7 +108,7 @@ public void generateDatabase(DatabendGlobalState globalState) throws Exception { throw new IgnoreMeException(); } }); - se.executeStatements(); // 在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 + se.executeStatements(); // 增删改一些数据（按权重随机选取算法） } @Override @@ -129,12 +130,8 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - } - try (Statement s = con.createStatement()) { s.execute("CREATE DATABASE " + databaseName); globalState.getState().logStatement("CREATE DATABASE " + databaseName); - } - try (Statement s = con.createStatement()) { s.execute("USE " + databaseName); globalState.getState().logStatement("USE " + databaseName); } diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java new file mode 100644 index 000000000..dcef8d7a8 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -0,0 +1,29 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendToStringVisitor; + +public class DatabendDeleteGenerator { + + public DatabendDeleteGenerator() { + } + + public static SQLQueryAdapter generate(DatabendGlobalState globalState){ + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + if(Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) + .generateExpression(DatabendDataType.BOOLEAN))); + DatabendErrors.addExpressionErrors(errors); + } + return new SQLQueryAdapter(sb.toString(),errors); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 43d38939f..a67a5d4ff 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -251,7 +251,7 @@ protected boolean canGenerateColumnOfType(DatabendDataType type) { public enum DatabendAggregateFunction { MAX(1), MIN(1), AVG(1, DatabendDataType.INT, DatabendDataType.FLOAT), COUNT(1), SUM(1, DatabendDataType.INT, DatabendDataType.FLOAT), STDDEV_POP(1), COVAR_POP(1), COVAR_SAMP(2); - // , STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) + //, *_IF, *_DISTINCT private int nrArgs; private DatabendDataType[] dataTypes; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index 59d6695ca..56f311f2b 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -53,7 +53,7 @@ public void check() throws SQLException { List> fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { - fetchColumns.add(gen.generateAggregate()); + fetchColumns.add(gen.generateAggregate()); // TODO 更换成非聚合函数 } select.setFetchColumns(Arrays.asList(aggregate)); // if (Randomly.getBooleanWithRatherLowProbability()) { @@ -181,16 +181,15 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, Node whereClause, List> joinList) { - DatabendSelect leftSelect = new DatabendSelect(); - leftSelect.setFetchColumns(aggregates); - leftSelect.setFromList(from); - leftSelect.setWhereClause(whereClause); - leftSelect.setJoinList(joinList); - // if (Randomly.getBooleanWithSmallProbability()) { - // leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); //TODO group by超过实际行数 - // leftSelect.setGroupByExpressions(select.getFetchColumns());// TODO group by不能放入聚合函数 - // } - return leftSelect; + DatabendSelect select = new DatabendSelect(); + select.setFetchColumns(aggregates); + select.setFromList(from); + select.setWhereClause(whereClause); + select.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + select.setGroupByExpressions(List.of(gen.generateConstant(DatabendDataType.INT))); // TODO 仍可加强 + } + return select; } } From 625204e01fb4a02e8de3e715cdee2a37515b9018 Mon Sep 17 00:00:00 2001 From: Kareem Shehata Date: Sun, 4 Dec 2022 16:36:52 +0800 Subject: [PATCH 0678/1284] Add Reducer capabilities by introducing a Reproducer that can be retrieved from TestOracle (#624) * Made TestOracle generic taking a type parameter for global state. Updated all usages of TestOracle to add this type info. * Add Reproducer class and enable getting the reproducer from a TestOracle. * Added use of Reproducer in an example Reducer and SQLite3's NoREC Oracle. * ProviderAdapter returns a Reproducer if it's available, and Main uses it to Reduce queries. Currently experimental and only outputs to stdout. * Fixing formatting * Fixing linter problems * Fixing HSLDB errors from rebase * Fixing HSQLDBNoRECOracle3 Co-authored-by: Manuel Rigger --- src/sqlancer/DatabaseProvider.java | 4 +- src/sqlancer/Main.java | 19 +++- src/sqlancer/MainOptions.java | 6 ++ src/sqlancer/OracleFactory.java | 2 +- src/sqlancer/ProviderAdapter.java | 15 +++- src/sqlancer/Reducer.java | 7 ++ src/sqlancer/Reproducer.java | 5 ++ src/sqlancer/StateToReproduce.java | 6 +- src/sqlancer/StatementReducer.java | 87 +++++++++++++++++++ src/sqlancer/arangodb/ArangoDBOptions.java | 6 +- .../test/ArangoDBQueryPartitioningBase.java | 2 +- src/sqlancer/citus/CitusOptions.java | 12 +-- src/sqlancer/citus/CitusProvider.java | 19 ++-- .../clickhouse/ClickHouseOptions.java | 12 +-- .../oracle/norec/ClickHouseNoRECOracle.java | 3 +- .../oracle/tlp/ClickHouseTLPBase.java | 2 +- .../cockroachdb/CockroachDBOptions.java | 20 ++--- .../oracle/CockroachDBNoRECOracle.java | 3 +- .../tlp/CockroachDBTLPAggregateOracle.java | 2 +- .../oracle/tlp/CockroachDBTLPBase.java | 5 +- .../common/oracle/CompositeTestOracle.java | 16 ++-- .../oracle/DocumentRemovalOracleBase.java | 2 +- src/sqlancer/common/oracle/NoRECBase.java | 2 +- .../oracle/PivotedQuerySynthesisBase.java | 2 +- .../TernaryLogicPartitioningOracleBase.java | 2 +- src/sqlancer/common/oracle/TestOracle.java | 8 +- src/sqlancer/databend/DatabendOptions.java | 18 ++-- .../databend/test/DatabendNoRECOracle.java | 2 +- ...abendQueryPartitioningAggregateTester.java | 3 +- .../test/DatabendQueryPartitioningBase.java | 5 +- ...DatabendQueryPartitioningHavingTester.java | 3 +- src/sqlancer/duckdb/DuckDBOptions.java | 18 ++-- .../duckdb/test/DuckDBNoRECOracle.java | 2 +- ...uckDBQueryPartitioningAggregateTester.java | 3 +- .../test/DuckDBQueryPartitioningBase.java | 3 +- .../DuckDBQueryPartitioningHavingTester.java | 3 +- src/sqlancer/h2/H2Options.java | 2 +- src/sqlancer/h2/H2QueryPartitioningBase.java | 2 +- src/sqlancer/hsqldb/HSQLDBOptions.java | 6 +- .../hsqldb/test/HSQLDBNoRECOracle.java | 2 +- .../test/HSQLDBQueryPartitioningBase.java | 2 +- src/sqlancer/mariadb/MariaDBOptions.java | 2 +- .../mariadb/oracle/MariaDBNoRECOracle.java | 2 +- src/sqlancer/mongodb/MongoDBOptions.java | 12 +-- .../test/MongoDBDocumentRemovalBase.java | 5 +- .../test/MongoDBQueryPartitioningBase.java | 3 +- src/sqlancer/mysql/MySQLOptions.java | 4 +- .../oracle/MySQLQueryPartitioningBase.java | 4 +- src/sqlancer/oceanbase/OceanBaseOptions.java | 6 +- .../oracle/OceanBaseNoRECOracle.java | 2 +- .../oceanbase/oracle/OceanBaseTLPBase.java | 3 +- src/sqlancer/postgres/PostgresOptions.java | 12 +-- .../postgres/oracle/PostgresNoRECOracle.java | 2 +- .../tlp/PostgresTLPAggregateOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPBase.java | 2 +- src/sqlancer/questdb/QuestDBOptions.java | 2 +- .../test/QuestDBQueryPartitioningBase.java | 3 +- src/sqlancer/sqlite3/SQLite3Options.java | 22 ++--- .../sqlite3/oracle/SQLite3Fuzzer.java | 2 +- .../sqlite3/oracle/SQLite3NoRECOracle.java | 65 +++++++++++--- .../oracle/tlp/SQLite3TLPAggregateOracle.java | 2 +- .../sqlite3/oracle/tlp/SQLite3TLPBase.java | 2 +- .../oracle/tlp/SQLite3TLPHavingOracle.java | 2 +- src/sqlancer/tidb/TiDBOptions.java | 10 +-- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 2 +- .../tidb/oracle/TiDBTLPHavingOracle.java | 2 +- src/sqlancer/yugabyte/ycql/YCQLOptions.java | 2 +- .../yugabyte/ycql/test/YCQLFuzzer.java | 2 +- src/sqlancer/yugabyte/ysql/YSQLOptions.java | 16 ++-- .../yugabyte/ysql/oracle/YSQLCatalog.java | 2 +- .../yugabyte/ysql/oracle/YSQLFuzzer.java | 2 +- .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 2 +- .../oracle/tlp/YSQLTLPAggregateOracle.java | 2 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 2 +- 74 files changed, 374 insertions(+), 174 deletions(-) create mode 100644 src/sqlancer/Reducer.java create mode 100644 src/sqlancer/Reproducer.java create mode 100644 src/sqlancer/StatementReducer.java diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 72acb30d8..48afb00f5 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -24,11 +24,13 @@ public interface DatabaseProvider, O extends DBMS * @param globalState * the state created and is valid for this method call. * + * @return Reproducer if a bug is found and a reproducer is available. + * * @throws Exception * if creating the database fails. * */ - void generateAndTestDatabase(G globalState) throws Exception; + Reproducer generateAndTestDatabase(G globalState) throws Exception; C createDatabase(G globalState) throws Exception; diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 7953386b7..dfb2bb89b 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -314,13 +314,30 @@ public void run() throws Exception { if (options.logEachSelect()) { logger.writeCurrent(state.getState()); } - provider.generateAndTestDatabase(state); + Reproducer reproducer = provider.generateAndTestDatabase(state); try { logger.getCurrentFileWriter().close(); logger.currentFileWriter = null; } catch (IOException e) { throw new AssertionError(e); } + if (reproducer != null && options.useReducer()) { + System.out.println("EXPERIMENTAL: Trying to reduce queries using a simple reducer."); + System.out.println("Reduced query will be output to stdout but not logs."); + G newGlobalState = createGlobalState(); + newGlobalState.setState(stateToRepro); + newGlobalState.setRandomly(r); + newGlobalState.setDatabaseName(databaseName); + newGlobalState.setMainOptions(options); + newGlobalState.setDbmsSpecificOptions(command); + QueryManager newManager = new QueryManager<>(newGlobalState); + newGlobalState.setStateLogger(new StateLogger(databaseName, provider, options)); + newGlobalState.setManager(newManager); + + Reducer reducer = new StatementReducer<>(provider); + reducer.reduce(state, reproducer, newGlobalState); + throw new AssertionError("Found a potential bug"); + } } } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 2916dbdd2..329010b39 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -107,6 +107,9 @@ public class MainOptions { @Parameter(names = "--database-prefix", description = "The prefix used for each database created") private String databasePrefix = "database"; // NOPMD + @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") + private boolean useReducer = false; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -244,4 +247,7 @@ public boolean performConnectionTest() { return useConnectionTest; } + public boolean useReducer() { + return useReducer; + } } diff --git a/src/sqlancer/OracleFactory.java b/src/sqlancer/OracleFactory.java index 897293c6b..9d6e1704b 100644 --- a/src/sqlancer/OracleFactory.java +++ b/src/sqlancer/OracleFactory.java @@ -4,7 +4,7 @@ public interface OracleFactory> { - TestOracle create(G globalState) throws Exception; + TestOracle create(G globalState) throws Exception; /** * Indicates whether the test oracle requires that all tables (including views) contain at least one row. diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index a16f8388c..1a530e93e 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -35,13 +35,13 @@ public Class getOptionClass() { } @Override - public void generateAndTestDatabase(G globalState) throws Exception { + public Reproducer generateAndTestDatabase(G globalState) throws Exception { try { generateDatabase(globalState); checkViewsAreValid(globalState); globalState.getManager().incrementCreateDatabase(); - TestOracle oracle = getTestOracle(globalState); + TestOracle oracle = getTestOracle(globalState); for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { assert localState != null; @@ -50,6 +50,12 @@ public void generateAndTestDatabase(G globalState) throws Exception { globalState.getManager().incrementSelectQueryCount(); } catch (IgnoreMeException e) { + } catch (AssertionError e) { + Reproducer reproducer = oracle.getLastReproducer(); + if (reproducer != null) { + return reproducer; + } + throw e; } assert localState != null; localState.executedWithoutError(); @@ -58,11 +64,12 @@ public void generateAndTestDatabase(G globalState) throws Exception { } finally { globalState.getConnection().close(); } + return null; } protected abstract void checkViewsAreValid(G globalState); - protected TestOracle getTestOracle(G globalState) throws Exception { + protected TestOracle getTestOracle(G globalState) throws Exception { List> testOracleFactory = globalState.getDbmsSpecificOptions() .getTestOracleFactory(); boolean testOracleRequiresMoreThanZeroRows = testOracleFactory.stream() @@ -75,7 +82,7 @@ protected TestOracle getTestOracle(G globalState) throws Exception { if (testOracleFactory.size() == 1) { return testOracleFactory.get(0).create(globalState); } else { - return new CompositeTestOracle(testOracleFactory.stream().map(o -> { + return new CompositeTestOracle(testOracleFactory.stream().map(o -> { try { return o.create(globalState); } catch (Exception e1) { diff --git a/src/sqlancer/Reducer.java b/src/sqlancer/Reducer.java new file mode 100644 index 000000000..0e6589262 --- /dev/null +++ b/src/sqlancer/Reducer.java @@ -0,0 +1,7 @@ +package sqlancer; + +public interface Reducer> { + + void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception; + +} diff --git a/src/sqlancer/Reproducer.java b/src/sqlancer/Reproducer.java new file mode 100644 index 000000000..ef64bd0fe --- /dev/null +++ b/src/sqlancer/Reproducer.java @@ -0,0 +1,5 @@ +package sqlancer; + +public interface Reproducer> { + boolean bugStillTriggers(G globalState); +} diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index f6408ce58..7ac3e9ccf 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -9,7 +9,7 @@ public class StateToReproduce { - private final List> statements = new ArrayList<>(); + private List> statements = new ArrayList<>(); private final String databaseName; @@ -128,4 +128,8 @@ public OracleRunReproductionState createLocalState() { return new OracleRunReproductionState(); } + public void setStatements(List> statements) { + this.statements = statements; + } + } diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java new file mode 100644 index 000000000..14db4ba5b --- /dev/null +++ b/src/sqlancer/StatementReducer.java @@ -0,0 +1,87 @@ +package sqlancer; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import sqlancer.common.query.Query; + +public class StatementReducer, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> + implements Reducer { + private final DatabaseProvider provider; + private boolean observedChange; + + public StatementReducer(DatabaseProvider provider) { + this.provider = provider; + } + + @SuppressWarnings("unchecked") + @Override + public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + + List> knownToReproduceBugStatements = new ArrayList<>(); + for (Query stat : state.getState().getStatements()) { + knownToReproduceBugStatements.add((Query) stat); + } + System.out.println("Starting query:"); + printQueries(knownToReproduceBugStatements); + System.out.println(); + + do { + observedChange = false; + knownToReproduceBugStatements = tryReduction(state, reproducer, newGlobalState, + knownToReproduceBugStatements, (candidateStatements, i) -> { + candidateStatements.remove((int) i); + return true; + }); + } while (observedChange); + + System.out.println("Reduced query:"); + printQueries(knownToReproduceBugStatements); + } + + private List> tryReduction(G state, // NOPMD + Reproducer reproducer, G newGlobalState, List> knownToReproduceBugStatements, + BiFunction>, Integer, Boolean> reductionOperation) throws Exception { + + List> statements = knownToReproduceBugStatements; + for (int i = 0; i < statements.size(); i++) { + try (C con2 = provider.createDatabase(newGlobalState)) { + newGlobalState.setConnection(con2); + List> candidateStatements = new ArrayList<>(statements); + if (!reductionOperation.apply(candidateStatements, i)) { + continue; + } + newGlobalState.getState().setStatements(candidateStatements.stream().collect(Collectors.toList())); + for (Query s : candidateStatements) { + try { + s.execute(newGlobalState); + } catch (Throwable ignoredException) { + // ignore + } + } + try { + if (reproducer.bugStillTriggers(newGlobalState)) { + observedChange = true; + statements = candidateStatements; + // reproducer.outputHook((SQLite3GlobalState) newGlobalState); + // state.getLogger().logReduced(newGlobalState.getState()); + } + } catch (Throwable ignoredException) { + + } + } + } + return statements; + } + + private void printQueries(List> statements) { + System.out.println("==============================="); + for (Query q : statements) { + System.out.println(q.getLogString()); + } + System.out.println("==============================="); + } + +} diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index b821fe8c3..7e20883d5 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -40,10 +40,10 @@ public List getTestOracleFactory() { public enum ArangoDBOracleFactory implements OracleFactory { QUERY_PARTITIONING { @Override - public TestOracle create(ArangoDBGlobalState globalState) throws Exception { - List oracles = new ArrayList<>(); + public TestOracle create(ArangoDBGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); oracles.add(new ArangoDBQueryPartitioningWhereTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } } } diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java index f583ed04f..859ffc74e 100644 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java @@ -17,7 +17,7 @@ public class ArangoDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, ArangoDBProvider.ArangoDBGlobalState> - implements TestOracle { + implements TestOracle { protected ArangoDBSchema schema; protected List targetColumns; diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index cdeadb672..67b14c304 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -29,21 +29,21 @@ public class CitusOptions extends PostgresOptions { public enum CitusOracleFactory implements OracleFactory { NOREC { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; return new CitusNoRECOracle(citusGlobalState); } }, PQS { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { return new PostgresPivotedQuerySynthesisOracle(globalState); } }, HAVING { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; return new CitusTLPHavingOracle(citusGlobalState); } @@ -51,13 +51,13 @@ public TestOracle create(PostgresGlobalState globalState) throws SQLException { }, QUERY_PARTITIONING { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; - List oracles = new ArrayList<>(); + List> oracles = new ArrayList<>(); oracles.add(new CitusTLPWhereOracle(citusGlobalState)); oracles.add(new CitusTLPHavingOracle(citusGlobalState)); oracles.add(new CitusTLPAggregateOracle(citusGlobalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index aaebd6709..c897d503e 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -319,15 +319,16 @@ public void generateDatabase(PostgresGlobalState globalState) throws Exception { } @Override - protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { - List oracles = ((CitusOptions) globalState.getDbmsSpecificOptions()).citusOracle.stream().map(o -> { - try { - return o.create(globalState); - } catch (Exception e1) { - throw new AssertionError(e1); - } - }).collect(Collectors.toList()); - return new CompositeTestOracle(oracles, globalState); + protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { + List> oracles = ((CitusOptions) globalState + .getDbmsSpecificOptions()).citusOracle.stream().map(o -> { + try { + return o.create(globalState); + } catch (Exception e1) { + throw new AssertionError(e1); + } + }).collect(Collectors.toList()); + return new CompositeTestOracle(oracles, globalState); } private List readCitusWorkerNodes(PostgresGlobalState globalState, SQLConnection con) diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 8a2056c1b..0aafab84d 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -34,37 +34,37 @@ public class ClickHouseOptions implements DBMSSpecificOptions { TLPWhere { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPWhereOracle(globalState); } }, TLPDistinct { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPDistinctOracle(globalState); } }, TLPGroupBy { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPGroupByOracle(globalState); } }, TLPAggregate { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPAggregateOracle(globalState); } }, TLPHaving { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPHavingOracle(globalState); } }, NoREC { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseNoRECOracle(globalState); } }; diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java index c7d56ab99..cda529cbe 100644 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -28,7 +28,8 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -public class ClickHouseNoRECOracle extends NoRECBase implements TestOracle { +public class ClickHouseNoRECOracle extends NoRECBase + implements TestOracle { private final ClickHouseSchema s; diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index a9b09fd92..690994ec0 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -26,7 +26,7 @@ import sqlancer.common.oracle.TestOracle; public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { ClickHouseSchema schema; ClickHouseTables targetTables; diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index aa41259b0..38b845781 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -34,59 +34,59 @@ public class CockroachDBOptions implements DBMSSpecificOptions { NOREC { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBNoRECOracle(globalState); } }, AGGREGATE { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPAggregateOracle(globalState); } }, GROUP_BY { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPGroupByOracle(globalState); } }, HAVING { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPHavingOracle(globalState); } }, WHERE { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPWhereOracle(globalState); } }, DISTINCT { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPDistinctOracle(globalState); } }, EXTENDED_WHERE { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPExtendedWhereOracle(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new CockroachDBTLPAggregateOracle(globalState)); oracles.add(new CockroachDBTLPHavingOracle(globalState)); oracles.add(new CockroachDBTLPWhereOracle(globalState)); oracles.add(new CockroachDBTLPGroupByOracle(globalState)); oracles.add(new CockroachDBTLPExtendedWhereOracle(globalState)); oracles.add(new CockroachDBTLPDistinctOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 3020f3238..77ecbcbed 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -29,7 +29,8 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -public class CockroachDBNoRECOracle extends NoRECBase implements TestOracle { +public class CockroachDBNoRECOracle extends NoRECBase + implements TestOracle { private CockroachDBExpressionGenerator gen; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index bc2601ef5..0368f5222 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -35,7 +35,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -public class CockroachDBTLPAggregateOracle implements TestOracle { +public class CockroachDBTLPAggregateOracle implements TestOracle { private final CockroachDBGlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index 7e7f03e26..21914eae2 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -22,8 +22,9 @@ import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; -public class CockroachDBTLPBase extends - TernaryLogicPartitioningOracleBase implements TestOracle { +public class CockroachDBTLPBase + extends TernaryLogicPartitioningOracleBase + implements TestOracle { CockroachDBSchema s; CockroachDBTables targetTables; diff --git a/src/sqlancer/common/oracle/CompositeTestOracle.java b/src/sqlancer/common/oracle/CompositeTestOracle.java index fbb66960a..0fdc13140 100644 --- a/src/sqlancer/common/oracle/CompositeTestOracle.java +++ b/src/sqlancer/common/oracle/CompositeTestOracle.java @@ -4,27 +4,27 @@ import sqlancer.GlobalState; -public class CompositeTestOracle implements TestOracle { +public class CompositeTestOracle> implements TestOracle { - private final TestOracle[] oracles; - private final GlobalState globalState; + private final List> oracles; + private final G globalState; private int i; - public CompositeTestOracle(List oracles, GlobalState globalState) { + public CompositeTestOracle(List> oracles, G globalState) { this.globalState = globalState; - this.oracles = oracles.toArray(new TestOracle[oracles.size()]); + this.oracles = oracles; } @Override public void check() throws Exception { try { - oracles[i].check(); - boolean lastOracleIndex = i == oracles.length - 1; + oracles.get(i).check(); + boolean lastOracleIndex = i == oracles.size() - 1; if (!lastOracleIndex) { globalState.getManager().incrementSelectQueryCount(); } } finally { - i = (i + 1) % oracles.length; + i = (i + 1) % oracles.size(); } } } diff --git a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java b/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java index a48a6f17e..b6c0ee509 100644 --- a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java +++ b/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java @@ -3,7 +3,7 @@ import sqlancer.GlobalState; import sqlancer.common.gen.ExpressionGenerator; -public abstract class DocumentRemovalOracleBase> implements TestOracle { +public abstract class DocumentRemovalOracleBase> implements TestOracle { protected E predicate; diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java index 734e04a86..c099b2592 100644 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ b/src/sqlancer/common/oracle/NoRECBase.java @@ -6,7 +6,7 @@ import sqlancer.SQLGlobalState; import sqlancer.common.query.ExpectedErrors; -public abstract class NoRECBase> implements TestOracle { +public abstract class NoRECBase> implements TestOracle { protected final S state; protected final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index cd2580b76..6c551d18e 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -12,7 +12,7 @@ import sqlancer.common.schema.AbstractRowValue; public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E, C extends SQLancerDBConnection> - implements TestOracle { + implements TestOracle { protected final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java index 3b5d87814..b4d6add4b 100644 --- a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java @@ -14,7 +14,7 @@ * @param * the global state type */ -public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { +public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { protected E predicate; protected E negatedPredicate; diff --git a/src/sqlancer/common/oracle/TestOracle.java b/src/sqlancer/common/oracle/TestOracle.java index 737adb692..2dfd854a0 100644 --- a/src/sqlancer/common/oracle/TestOracle.java +++ b/src/sqlancer/common/oracle/TestOracle.java @@ -1,7 +1,13 @@ package sqlancer.common.oracle; -public interface TestOracle { +import sqlancer.GlobalState; +import sqlancer.Reproducer; + +public interface TestOracle> { void check() throws Exception; + default Reproducer getLastReproducer() { + return null; + } } diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index ebacd21cf..efcdec4eb 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -99,53 +99,53 @@ public enum DatabendOracleFactory implements OracleFactory NOREC { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendNoRECOracle(globalState); } }, HAVING { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningHavingTester(globalState); } }, WHERE { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningWhereTester(globalState); } }, GROUP_BY { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningGroupByTester(globalState); } }, AGGREGATE { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningAggregateTester(globalState); } }, DISTINCT { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningDistinctTester(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new DatabendQueryPartitioningWhereTester(globalState)); oracles.add(new DatabendQueryPartitioningHavingTester(globalState)); oracles.add(new DatabendQueryPartitioningAggregateTester(globalState)); oracles.add(new DatabendQueryPartitioningDistinctTester(globalState)); oracles.add(new DatabendQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index cb27fa78b..79c369f12 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -33,7 +33,7 @@ import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendNoRECOracle extends NoRECBase implements TestOracle { +public class DatabendNoRECOracle extends NoRECBase implements TestOracle { private final DatabendSchema s; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index 59d6695ca..b6e423980 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -30,7 +30,8 @@ import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; import sqlancer.databend.gen.DatabendNewExpressionGenerator.DatabendAggregateFunction; -public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { +public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase + implements TestOracle { private String firstResult; private String secondResult; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java index 6ef132572..2242bf99f 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -24,8 +24,9 @@ import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendQueryPartitioningBase extends - TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { +public class DatabendQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, DatabendGlobalState> + implements TestOracle { DatabendSchema s; DatabendTables targetTables; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java index 8ca1489f2..a64be36b0 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -16,7 +16,8 @@ import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; -public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase implements TestOracle { +public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase + implements TestOracle { public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { super(state); diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index 4e1c49cd1..f14379163 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -97,53 +97,53 @@ public enum DuckDBOracleFactory implements OracleFactory { NOREC { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBNoRECOracle(globalState); } }, HAVING { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningHavingTester(globalState); } }, WHERE { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningWhereTester(globalState); } }, GROUP_BY { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningGroupByTester(globalState); } }, AGGREGATE { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningAggregateTester(globalState); } }, DISTINCT { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningDistinctTester(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new DuckDBQueryPartitioningWhereTester(globalState)); oracles.add(new DuckDBQueryPartitioningHavingTester(globalState)); oracles.add(new DuckDBQueryPartitioningAggregateTester(globalState)); oracles.add(new DuckDBQueryPartitioningDistinctTester(globalState)); oracles.add(new DuckDBQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 550043615..de89bfa7d 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -33,7 +33,7 @@ import sqlancer.duckdb.gen.DuckDBExpressionGenerator; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBCastOperation; -public class DuckDBNoRECOracle extends NoRECBase implements TestOracle { +public class DuckDBNoRECOracle extends NoRECBase implements TestOracle { private final DuckDBSchema s; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index a6b8d8f27..8d7835d29 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -30,7 +30,8 @@ import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPostfixOperator; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPrefixOperator; -public class DuckDBQueryPartitioningAggregateTester extends DuckDBQueryPartitioningBase implements TestOracle { +public class DuckDBQueryPartitioningAggregateTester extends DuckDBQueryPartitioningBase + implements TestOracle { private String firstResult; private String secondResult; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 68456ef26..837247f62 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -24,7 +24,8 @@ import sqlancer.duckdb.gen.DuckDBExpressionGenerator; public class DuckDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, DuckDBGlobalState> implements TestOracle { + extends TernaryLogicPartitioningOracleBase, DuckDBGlobalState> + implements TestOracle { DuckDBSchema s; DuckDBTables targetTables; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index fc8843729..aef003fa5 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -14,7 +14,8 @@ import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBQueryPartitioningHavingTester extends DuckDBQueryPartitioningBase implements TestOracle { +public class DuckDBQueryPartitioningHavingTester extends DuckDBQueryPartitioningBase + implements TestOracle { public DuckDBQueryPartitioningHavingTester(DuckDBGlobalState state) { super(state); diff --git a/src/sqlancer/h2/H2Options.java b/src/sqlancer/h2/H2Options.java index 45b47f203..e7141fcf2 100644 --- a/src/sqlancer/h2/H2Options.java +++ b/src/sqlancer/h2/H2Options.java @@ -20,7 +20,7 @@ public enum H2OracleFactory implements OracleFactory { TLP_WHERE { @Override - public TestOracle create(H2GlobalState globalState) throws SQLException { + public TestOracle create(H2GlobalState globalState) throws SQLException { return new H2QueryPartitioningWhereTester(globalState); } diff --git a/src/sqlancer/h2/H2QueryPartitioningBase.java b/src/sqlancer/h2/H2QueryPartitioningBase.java index b15f1c312..ec5af4e2e 100644 --- a/src/sqlancer/h2/H2QueryPartitioningBase.java +++ b/src/sqlancer/h2/H2QueryPartitioningBase.java @@ -17,7 +17,7 @@ import sqlancer.h2.H2Schema.H2Tables; public class H2QueryPartitioningBase extends TernaryLogicPartitioningOracleBase, H2GlobalState> - implements TestOracle { + implements TestOracle { H2Schema s; H2Tables targetTables; diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index 9cf6dc228..3e3dcc4dc 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -22,13 +22,15 @@ public class HSQLDBOptions implements DBMSSpecificOptions { WHERE { @Override - public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) throws SQLException { + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws SQLException { return new HSQLDBQueryPartitioningWhereTester(globalState); } }, NOREC { @Override - public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) throws Exception { + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws Exception { return new HSQLDBNoRECOracle(globalState); } } diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java index 7de1918b4..ca376e348 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -31,7 +31,7 @@ import sqlancer.hsqldb.ast.HSQLDBSelect; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { +public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { private final HSQLDBSchema s; diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java index ec501ab5e..5e62a39ac 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -22,7 +22,7 @@ public class HSQLDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> - implements TestOracle { + implements TestOracle { HSQLDBSelect select; HSQLDBExpressionGenerator expressionGenerator; diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 02e6eab64..70e0aedbd 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -28,7 +28,7 @@ public enum MariaDBOracleFactory implements OracleFactory { NOREC { @Override - public TestOracle create(MariaDBGlobalState globalState) throws SQLException { + public TestOracle create(MariaDBGlobalState globalState) throws SQLException { return new MariaDBNoRECOracle(globalState); } diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index 882b49c08..bba0627e4 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -27,7 +27,7 @@ import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; -public class MariaDBNoRECOracle extends NoRECBase implements TestOracle { +public class MariaDBNoRECOracle extends NoRECBase implements TestOracle { private final MariaDBSchema s; private static final int NOT_FOUND = -1; diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 9e02de67b..78bd73ffc 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -56,18 +56,18 @@ public List getTestOracleFactory() { public enum MongoDBOracleFactory implements OracleFactory { QUERY_PARTITIONING { @Override - public TestOracle create(MongoDBGlobalState globalState) throws Exception { - List oracles = new ArrayList<>(); + public TestOracle create(MongoDBGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); oracles.add(new MongoDBQueryPartitioningWhereTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }, DOCUMENT_REMOVAL { @Override - public TestOracle create(MongoDBGlobalState globalState) throws Exception { - List oracles = new ArrayList<>(); + public TestOracle create(MongoDBGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); oracles.add(new MongoDBDocumentRemovalTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } } } diff --git a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java index 6faae42bc..bf39643c1 100644 --- a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java +++ b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java @@ -15,8 +15,9 @@ import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator; import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; -public class MongoDBDocumentRemovalBase extends - DocumentRemovalOracleBase, MongoDBProvider.MongoDBGlobalState> implements TestOracle { +public class MongoDBDocumentRemovalBase + extends DocumentRemovalOracleBase, MongoDBProvider.MongoDBGlobalState> + implements TestOracle { protected MongoDBSchema schema; protected MongoDBSchema.MongoDBTables targetTables; diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java index 30ebe56bd..ea6d39bbf 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -19,7 +19,8 @@ import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; public class MongoDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, MongoDBGlobalState> implements TestOracle { + extends TernaryLogicPartitioningOracleBase, MongoDBGlobalState> + implements TestOracle { protected MongoDBSchema schema; protected MongoDBTables targetTables; diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 2a3f16968..ad8060090 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -28,7 +28,7 @@ public enum MySQLOracleFactory implements OracleFactory { TLP_WHERE { @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { + public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLTLPWhereOracle(globalState); } @@ -36,7 +36,7 @@ public TestOracle create(MySQLGlobalState globalState) throws SQLException { PQS { @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { + public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLPivotedQuerySynthesisOracle(globalState); } diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index 8946136fe..7b7ba8b34 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -19,8 +19,8 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.gen.MySQLExpressionGenerator; -public abstract class MySQLQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase implements TestOracle { +public abstract class MySQLQueryPartitioningBase extends + TernaryLogicPartitioningOracleBase implements TestOracle { MySQLSchema s; MySQLTables targetTables; diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index b19de3983..86507f769 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -28,20 +28,20 @@ public enum OceanBaseOracleFactory implements OracleFactory create(OceanBaseGlobalState globalState) throws SQLException { return new OceanBaseTLPWhereOracle(globalState); } }, NoREC { @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { return new OceanBaseNoRECOracle(globalState); } }, PQS { @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { return new OceanBasePivotedQuerySynthesisOracle(globalState); } diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java index 900ac8caa..cc98dcf0a 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java @@ -28,7 +28,7 @@ import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { +public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { // SELECT COUNT(*) FROM t0 WHERE ; // SELECT SUM(count) FROM (SELECT IS TRUE as count FROM t0); diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java index 51f06a3fc..73ff19bfb 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java @@ -21,7 +21,8 @@ import sqlancer.oceanbase.gen.OceanBaseHintGenerator; public abstract class OceanBaseTLPBase - extends TernaryLogicPartitioningOracleBase implements TestOracle { + extends TernaryLogicPartitioningOracleBase + implements TestOracle { OceanBaseSchema s; OceanBaseTables targetTables; diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index b6c79d752..948a539ce 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -44,13 +44,13 @@ public class PostgresOptions implements DBMSSpecificOptions { NOREC { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { return new PostgresNoRECOracle(globalState); } }, PQS { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { return new PostgresPivotedQuerySynthesisOracle(globalState); } @@ -62,19 +62,19 @@ public boolean requiresAllTablesToContainRows() { HAVING { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { return new PostgresTLPHavingOracle(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new PostgresTLPWhereOracle(globalState)); oracles.add(new PostgresTLPHavingOracle(globalState)); oracles.add(new PostgresTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 995eb207f..f4bc10e74 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -36,7 +36,7 @@ import sqlancer.postgres.gen.PostgresExpressionGenerator; import sqlancer.postgres.oracle.tlp.PostgresTLPBase; -public class PostgresNoRECOracle extends NoRECBase implements TestOracle { +public class PostgresNoRECOracle extends NoRECBase implements TestOracle { private final PostgresSchema s; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index b9ea695e2..a1eaa43c7 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -29,7 +29,7 @@ import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.gen.PostgresCommon; -public class PostgresTLPAggregateOracle extends PostgresTLPBase implements TestOracle { +public class PostgresTLPAggregateOracle extends PostgresTLPBase implements TestOracle { private String firstResult; private String secondResult; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 34f5e602a..5e349b5dc 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -29,7 +29,7 @@ import sqlancer.postgres.oracle.PostgresNoRECOracle; public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { protected PostgresSchema s; protected PostgresTables targetTables; diff --git a/src/sqlancer/questdb/QuestDBOptions.java b/src/sqlancer/questdb/QuestDBOptions.java index 527e80cc8..f84ca3d47 100644 --- a/src/sqlancer/questdb/QuestDBOptions.java +++ b/src/sqlancer/questdb/QuestDBOptions.java @@ -27,7 +27,7 @@ public enum QuestDBOracleFactory implements OracleFactory { // TODO (anxing): implement test oracles WHERE { @Override - public TestOracle create(QuestDBGlobalState globalState) throws SQLException { + public TestOracle create(QuestDBGlobalState globalState) throws SQLException { return new QuestDBQueryPartitioningWhereTester(globalState); } } diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index fb6bd3c3f..45e3db9c4 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -24,7 +24,8 @@ import sqlancer.questdb.gen.QuestDBExpressionGenerator; public class QuestDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, QuestDBGlobalState> implements TestOracle { + extends TernaryLogicPartitioningOracleBase, QuestDBGlobalState> + implements TestOracle { QuestDBSchema s; QuestDBTables targetTables; diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index dabc3ce13..6df9c378c 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -93,7 +93,7 @@ public class SQLite3Options implements DBMSSpecificOptions public enum SQLite3OracleFactory implements OracleFactory { PQS { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3PivotedQuerySynthesisOracle(globalState); } @@ -105,14 +105,14 @@ public boolean requiresAllTablesToContainRows() { }, NoREC { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3NoRECOracle(globalState); } }, AGGREGATE { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPAggregateOracle(globalState); } @@ -120,45 +120,45 @@ public TestOracle create(SQLite3GlobalState globalState) throws SQLException { WHERE { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPWhereOracle(globalState); } }, DISTINCT { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPDistinctOracle(globalState); } }, GROUP_BY { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPGroupByOracle(globalState); } }, HAVING { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPHavingOracle(globalState); } }, FUZZER { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3Fuzzer(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new SQLite3TLPWhereOracle(globalState)); oracles.add(new SQLite3TLPDistinctOracle(globalState)); oracles.add(new SQLite3TLPGroupByOracle(globalState)); oracles.add(new SQLite3TLPHavingOracle(globalState)); oracles.add(new SQLite3TLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index 8e868dfd0..bf13ffc80 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -7,7 +7,7 @@ import sqlancer.sqlite3.SQLite3Visitor; // tries to trigger a crash -public class SQLite3Fuzzer implements TestOracle { +public class SQLite3Fuzzer implements TestOracle { private final SQLite3GlobalState globalState; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index db9f55531..826a2727b 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -4,9 +4,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Function; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -29,11 +31,28 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3NoRECOracle extends NoRECBase implements TestOracle { +public class SQLite3NoRECOracle extends NoRECBase implements TestOracle { private static final int NO_VALID_RESULT = -1; private final SQLite3Schema s; private SQLite3ExpressionGenerator gen; + private Reproducer reproducer; + + private class SQLite3NoRECReproducer implements Reproducer { + private final Function optimizedQuery; + private final Function unoptimizedQuery; + + SQLite3NoRECReproducer(Function optimizedQuery, + Function unoptimizedQuery) { + this.optimizedQuery = optimizedQuery; + this.unoptimizedQuery = unoptimizedQuery; + } + + @Override + public boolean bugStillTriggers(SQLite3GlobalState globalState) { + return optimizedQuery.apply(globalState) != unoptimizedQuery.apply(globalState); + } + } public SQLite3NoRECOracle(SQLite3GlobalState globalState) { super(globalState); @@ -51,6 +70,7 @@ public SQLite3NoRECOracle(SQLite3GlobalState globalState) { @Override public void check() throws SQLException { + reproducer = null; SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); gen = new SQLite3ExpressionGenerator(state).setColumns(columns); @@ -62,19 +82,28 @@ public void check() throws SQLException { select.setFromTables(tableRefs); select.setJoinClauses(joinStatements); - int optimizedCount = getOptimizedQuery(select, randomWhereCondition); - int unoptimizedCount = getUnoptimizedQuery(select, randomWhereCondition); + Function optimizedQuery = getOptimizedQuery(select, randomWhereCondition); + Function unoptimizedQuery = getUnoptimizedQuery(select, randomWhereCondition); + int optimizedCount = optimizedQuery.apply(state); + int unoptimizedCount = unoptimizedQuery.apply(state); if (optimizedCount == NO_VALID_RESULT || unoptimizedCount == NO_VALID_RESULT) { throw new IgnoreMeException(); } if (optimizedCount != unoptimizedCount) { + reproducer = new SQLite3NoRECReproducer(optimizedQuery, unoptimizedQuery); state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); throw new AssertionError(optimizedCount + " " + unoptimizedCount); } } - private int getUnoptimizedQuery(SQLite3Select select, SQLite3Expression randomWhereCondition) throws SQLException { + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + + private Function getUnoptimizedQuery(SQLite3Select select, + SQLite3Expression randomWhereCondition) throws SQLException { SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, randomWhereCondition); SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); @@ -85,10 +114,17 @@ private int getUnoptimizedQuery(SQLite3Select select, SQLite3Expression randomWh logger.writeCurrent(unoptimizedQueryString); } SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - return extractCounts(q); + return new Function() { + + @Override + public Integer apply(SQLite3GlobalState state) { + return extractCounts(q, state); + } + }; } - private int getOptimizedQuery(SQLite3Select select, SQLite3Expression randomWhereCondition) throws SQLException { + private Function getOptimizedQuery(SQLite3Select select, + SQLite3Expression randomWhereCondition) throws SQLException { boolean useAggregate = Randomly.getBoolean(); if (Randomly.getBoolean()) { select.setOrderByExpressions(gen.generateOrderBys()); @@ -106,12 +142,19 @@ private int getOptimizedQuery(SQLite3Select select, SQLite3Expression randomWher logger.writeCurrent(optimizedQueryString); } SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - return useAggregate ? extractCounts(q) : countRows(q); + return new Function() { + + @Override + public Integer apply(SQLite3GlobalState state) { + return useAggregate ? extractCounts(q, state) : countRows(q, state); + } + + }; } - private int countRows(SQLQueryAdapter q) { + private int countRows(SQLQueryAdapter q, SQLite3GlobalState globalState) { int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(state)) { + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs == null) { return NO_VALID_RESULT; } else { @@ -132,9 +175,9 @@ private int countRows(SQLQueryAdapter q) { return count; } - private int extractCounts(SQLQueryAdapter q) { + private int extractCounts(SQLQueryAdapter q, SQLite3GlobalState globalState) { int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(state)) { + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs == null) { return NO_VALID_RESULT; } else { diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index c0243425f..715770257 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -28,7 +28,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3TLPAggregateOracle implements TestOracle { +public class SQLite3TLPAggregateOracle implements TestOracle { private final SQLite3GlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index de1f01c92..650df8537 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -23,7 +23,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; public class SQLite3TLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { SQLite3Schema s; SQLite3Tables targetTables; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index 2524d979b..6a960021d 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -29,7 +29,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3TLPHavingOracle implements TestOracle { +public class SQLite3TLPHavingOracle implements TestOracle { private final SQLite3GlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index f9137f6f7..51afa2a9a 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -29,23 +29,23 @@ public class TiDBOptions implements DBMSSpecificOptions { public enum TiDBOracleFactory implements OracleFactory { HAVING { @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { + public TestOracle create(TiDBGlobalState globalState) throws SQLException { return new TiDBTLPHavingOracle(globalState); } }, WHERE { @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { + public TestOracle create(TiDBGlobalState globalState) throws SQLException { return new TiDBTLPWhereOracle(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(TiDBGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new TiDBTLPWhereOracle(globalState)); oracles.add(new TiDBTLPHavingOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index ba2ec9df0..9e86ffd12 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -23,7 +23,7 @@ import sqlancer.tidb.gen.TiDBHintGenerator; public abstract class TiDBTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { TiDBSchema s; TiDBTables targetTables; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java index a9f64671b..68b4a56c1 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java @@ -12,7 +12,7 @@ import sqlancer.tidb.ast.TiDBExpression; import sqlancer.tidb.visitor.TiDBVisitor; -public class TiDBTLPHavingOracle extends TiDBTLPBase implements TestOracle { +public class TiDBTLPHavingOracle extends TiDBTLPBase implements TestOracle { public TiDBTLPHavingOracle(TiDBGlobalState state) { super(state); diff --git a/src/sqlancer/yugabyte/ycql/YCQLOptions.java b/src/sqlancer/yugabyte/ycql/YCQLOptions.java index 7e35e2cea..df4854757 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLOptions.java +++ b/src/sqlancer/yugabyte/ycql/YCQLOptions.java @@ -37,7 +37,7 @@ public class YCQLOptions implements DBMSSpecificOptions { public enum YCQLOracleFactory implements OracleFactory { FUZZER { @Override - public TestOracle create(YCQLGlobalState globalState) throws SQLException { + public TestOracle create(YCQLGlobalState globalState) throws SQLException { return new YCQLFuzzer(globalState); } diff --git a/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java index 8dee32c3e..7c21b7db8 100644 --- a/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java +++ b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java @@ -11,7 +11,7 @@ import sqlancer.yugabyte.ycql.YCQLToStringVisitor; import sqlancer.yugabyte.ycql.gen.YCQLRandomQuerySynthesizer; -public class YCQLFuzzer implements TestOracle { +public class YCQLFuzzer implements TestOracle { private final YCQLProvider.YCQLGlobalState globalState; private final List testQueries; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/yugabyte/ysql/YSQLOptions.java b/src/sqlancer/yugabyte/ysql/YSQLOptions.java index 4cd299660..6b46b09bf 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLOptions.java +++ b/src/sqlancer/yugabyte/ysql/YSQLOptions.java @@ -48,25 +48,25 @@ public List getTestOracleFactory() { public enum YSQLOracleFactory implements OracleFactory { FUZZER { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLFuzzer(globalState); } }, CATALOG { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLCatalog(globalState); } }, NOREC { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLNoRECOracle(globalState); } }, PQS { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLPivotedQuerySynthesisOracle(globalState); } @@ -77,19 +77,19 @@ public boolean requiresAllTablesToContainRows() { }, HAVING { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLTLPHavingOracle(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new YSQLTLPWhereOracle(globalState)); oracles.add(new YSQLTLPHavingOracle(globalState)); oracles.add(new YSQLTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } } diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java index d6763e799..e5ae3fe1b 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java @@ -18,7 +18,7 @@ import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; -public class YSQLCatalog implements TestOracle { +public class YSQLCatalog implements TestOracle { protected final YSQLGlobalState state; protected final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java index 0aa16993e..8d0cea997 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java @@ -13,7 +13,7 @@ import sqlancer.yugabyte.ysql.YSQLVisitor; import sqlancer.yugabyte.ysql.gen.YSQLRandomQueryGenerator; -public class YSQLFuzzer implements TestOracle { +public class YSQLFuzzer implements TestOracle { private final YSQLGlobalState globalState; private final List testQueries; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java index 6b2a57560..3ae27fd73 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -33,7 +33,7 @@ import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPBase; -public class YSQLNoRECOracle extends NoRECBase implements TestOracle { +public class YSQLNoRECOracle extends NoRECBase implements TestOracle { private final YSQLSchema s; diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java index 26e4d9179..537a074ae 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -29,7 +29,7 @@ import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation.PrefixOperator; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { +public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { private String firstResult; private String secondResult; diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java index e574391de..863e85968 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -26,7 +26,7 @@ import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; public class YSQLTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { protected YSQLSchema s; protected YSQLTables targetTables; From f060d92551755f6c24dde4a84f31cdd560b222e4 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 5 Dec 2022 10:52:35 +0800 Subject: [PATCH 0679/1284] tidb: skip invalid regexp pattern (#628) Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 0586846fc..0022908a9 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -32,6 +32,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("error parsing regexp"); errors.add("from regexp"); errors.add("Empty pattern is invalid"); + errors.add("Invalid regexp pattern"); // To avoid bugs errors.add("Unknown column"); // https://github.com/pingcap/tidb/issues/35522 From f5228187b7a2d25738a846816d9c3333c1ff4764 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 11 Dec 2022 18:29:31 +0800 Subject: [PATCH 0680/1284] fix ci when to install citus Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6ae3bd74e..5a15ac75b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,9 +43,9 @@ jobs: run: | echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list curl https://install.citusdata.com/community/deb.sh | sudo bash - sudo apt-get -y install postgresql-13-citus-10.1 + sudo apt-get -y install postgresql-15-citus-11.1 sudo chown -R $USER:$USER /var/run/postgresql - export PATH=/usr/lib/postgresql/13/bin:$PATH + export PATH=/usr/lib/postgresql/15/bin:$PATH cd ~ mkdir -p citus/coordinator citus/worker1 citus/worker2 initdb -D citus/coordinator From ea05d14bf7cc59cdd6badb472208a3756b13170c Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 11 Dec 2022 22:55:12 +0800 Subject: [PATCH 0681/1284] feature(databend): enhance norec, tlp, and implement pqs Signed-off-by: Yisong Han --- src/sqlancer/ProviderAdapter.java | 8 +- src/sqlancer/databend/DatabendErrors.java | 4 + .../DatabendExpectedValueVisitor.java | 155 ++++++++ src/sqlancer/databend/DatabendExprToNode.java | 25 ++ src/sqlancer/databend/DatabendOptions.java | 19 +- src/sqlancer/databend/DatabendProvider.java | 6 +- src/sqlancer/databend/DatabendSchema.java | 70 +++- .../ast/DatabendAggregateOperation.java | 49 +++ .../ast/DatabendBetweenOperation.java | 40 +++ .../DatabendBinaryArithmeticOperation.java | 82 ++++- .../DatabendBinaryComparisonOperation.java | 122 ++++++- .../ast/DatabendBinaryLogicalOperation.java | 88 ++++- .../databend/ast/DatabendCastOperation.java | 28 +- .../databend/ast/DatabendColumnValue.java | 29 ++ .../databend/ast/DatabendConstant.java | 340 ++++++++++++++++-- .../databend/ast/DatabendExpression.java | 9 + .../ast/DatabendFunctionOperation.java | 4 + .../databend/ast/DatabendInOperation.java | 53 +++ src/sqlancer/databend/ast/DatabendJoin.java | 7 +- .../databend/ast/DatabendLikeOperation.java | 43 ++- .../databend/ast/DatabendOrderByTerm.java | 10 + .../ast/DatabendUnaryPostfixOperation.java | 63 ++-- .../ast/DatabendUnaryPrefixOperation.java | 88 +++-- .../databend/gen/DatabendDeleteGenerator.java | 13 +- .../databend/gen/DatabendInsertGenerator.java | 6 +- .../gen/DatabendNewExpressionGenerator.java | 179 +++++---- .../gen/DatabendRandomQuerySynthesizer.java | 51 ++- .../databend/gen/DatabendTableGenerator.java | 7 +- .../databend/test/DatabendNoRECOracle.java | 35 +- .../DatabendPivotedQuerySynthesisOracle.java | 154 ++++++++ ...abendQueryPartitioningAggregateTester.java | 20 +- .../DatabendQueryPartitioningBase.java | 27 +- ...tabendQueryPartitioningDistinctTester.java | 12 +- ...atabendQueryPartitioningGroupByTester.java | 11 +- ...DatabendQueryPartitioningHavingTester.java | 20 +- .../DatabendQueryPartitioningWhereTester.java | 9 +- test/sqlancer/dbms/TestDatabend.java | 32 +- 37 files changed, 1626 insertions(+), 292 deletions(-) create mode 100644 src/sqlancer/databend/DatabendExpectedValueVisitor.java create mode 100644 src/sqlancer/databend/DatabendExprToNode.java create mode 100644 src/sqlancer/databend/ast/DatabendAggregateOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendBetweenOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendColumnValue.java create mode 100644 src/sqlancer/databend/ast/DatabendFunctionOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendInOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendOrderByTerm.java create mode 100644 src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningAggregateTester.java (91%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningBase.java (73%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningDistinctTester.java (78%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningGroupByTester.java (85%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningHavingTester.java (80%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningWhereTester.java (85%) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 46475f260..a16f8388c 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -45,8 +45,12 @@ public void generateAndTestDatabase(G globalState) throws Exception { for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { assert localState != null; - oracle.check(); - globalState.getManager().incrementSelectQueryCount(); + try { + oracle.check(); + globalState.getManager().incrementSelectQueryCount(); + } catch (IgnoreMeException e) { + + } assert localState != null; localState.executedWithoutError(); } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index ab9724a5f..0a12f6b84 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -12,6 +12,10 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("/ by zero"); errors.add("ORDER BY position"); errors.add("GROUP BY position"); + errors.add("downcast column error"); // bug + errors.add("index out of bounds"); // bug + errors.add("validity's length must be equal"); // bug + errors.add("validity must be equal to the array's length"); // bug /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE * databend2; CREATE TABLE t0(c0VARCHAR VARCHAR NULL, c1VARCHAR VARCHAR NULL, c2FLOAT FLOAT NOT NULL diff --git a/src/sqlancer/databend/DatabendExpectedValueVisitor.java b/src/sqlancer/databend/DatabendExpectedValueVisitor.java new file mode 100644 index 000000000..f6ff93819 --- /dev/null +++ b/src/sqlancer/databend/DatabendExpectedValueVisitor.java @@ -0,0 +1,155 @@ +package sqlancer.databend; + +import java.util.List; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; + +public class DatabendExpectedValueVisitor { + + protected final StringBuilder sb = new StringBuilder(); + + private void print(Node expr) { + sb.append(DatabendToStringVisitor.asString(expr)); + sb.append(" -- "); + sb.append(((DatabendExpression) expr).getExpectedValue()); + sb.append("\n"); + } + + @SuppressWarnings("unchecked") + public void visit(Node expr) { + assert expr != null; + if (expr instanceof ColumnReferenceNode) { + visit((ColumnReferenceNode) expr); + } else if (expr instanceof NewUnaryPostfixOperatorNode) { + visit((NewUnaryPostfixOperatorNode) expr); + } else if (expr instanceof NewUnaryPrefixOperatorNode) { + visit((NewUnaryPrefixOperatorNode) expr); + } else if (expr instanceof NewBinaryOperatorNode) { + visit((NewBinaryOperatorNode) expr); + } else if (expr instanceof TableReferenceNode) { + visit((TableReferenceNode) expr); + } else if (expr instanceof NewFunctionNode) { + visit((NewFunctionNode) expr); + } else if (expr instanceof NewBetweenOperatorNode) { + visit((NewBetweenOperatorNode) expr); + } else if (expr instanceof NewInOperatorNode) { + visit((NewInOperatorNode) expr); + } else if (expr instanceof NewOrderingTerm) { + visit((NewOrderingTerm) expr); + } else if (expr instanceof NewAliasNode) { + visit((NewAliasNode) expr); + } else if (expr instanceof NewPostfixTextNode) { + visit((NewPostfixTextNode) expr); + } else if (expr instanceof DatabendConstant) { + visit((DatabendConstant) expr); + } else if (expr instanceof DatabendSelect) { + visit((DatabendSelect) expr); + } else if (expr instanceof DatabendJoin) { + visit((DatabendJoin) expr); + } else { + throw new AssertionError(expr); + } + } + + public void visit(ColumnReferenceNode c) { + print(c); + } + + public void visit(NewUnaryPostfixOperatorNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewUnaryPrefixOperatorNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewBinaryOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(TableReferenceNode t) { + print(t); + } + + public void visit(NewFunctionNode fun) { + print(fun); + visit(fun.getArgs()); + } + + public void visit(List> expressions) { + for (Node expression : expressions) { + visit(expression); + } + } + + public void visit(NewBetweenOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getMiddle()); + visit(op.getRight()); + } + + public void visit(NewInOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(NewOrderingTerm op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewAliasNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewPostfixTextNode postFixText) { + print(postFixText); + visit(postFixText.getExpr()); + } + + public void visit(DatabendConstant constant) { + print(constant); + } + + public void visit(DatabendSelect select) { + print(select.getWhereClause()); + } + + public void visit(DatabendJoin join) { + print(join.getOnCondition()); + } + + public String get() { + return sb.toString(); + } + + public static String asExpectedValues(Node expr) { + DatabendExpectedValueVisitor v = new DatabendExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + +} diff --git a/src/sqlancer/databend/DatabendExprToNode.java b/src/sqlancer/databend/DatabendExprToNode.java new file mode 100644 index 000000000..e81269893 --- /dev/null +++ b/src/sqlancer/databend/DatabendExprToNode.java @@ -0,0 +1,25 @@ +package sqlancer.databend; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.ast.DatabendExpression; + +public class DatabendExprToNode { + + private DatabendExprToNode() { + + } + + @SuppressWarnings("unchecked") + public static Node cast(DatabendExpression expression) { + return (Node) expression; + } + + @SuppressWarnings("unchecked") + public static List> casts(List expressions) { + return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index ebacd21cf..f88449475 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -15,11 +15,12 @@ import sqlancer.databend.DatabendOptions.DatabendOracleFactory; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.test.DatabendNoRECOracle; -import sqlancer.databend.test.DatabendQueryPartitioningAggregateTester; -import sqlancer.databend.test.DatabendQueryPartitioningDistinctTester; -import sqlancer.databend.test.DatabendQueryPartitioningGroupByTester; -import sqlancer.databend.test.DatabendQueryPartitioningHavingTester; -import sqlancer.databend.test.DatabendQueryPartitioningWhereTester; +import sqlancer.databend.test.DatabendPivotedQuerySynthesisOracle; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningAggregateTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningDistinctTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningGroupByTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningHavingTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningWhereTester; @Parameters(commandDescription = "Databend") public class DatabendOptions implements DBMSSpecificOptions { @@ -147,7 +148,13 @@ public TestOracle create(DatabendGlobalState globalState) throws SQLException { oracles.add(new DatabendQueryPartitioningGroupByTester(globalState)); return new CompositeTestOracle(oracles, globalState); } - }; + }, + PQS { + @Override + public TestOracle create(DatabendGlobalState globalState) throws Exception { + return new DatabendPivotedQuerySynthesisOracle(globalState); + } + } } diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index f22a7cec9..0ab0d4c9b 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -35,12 +35,10 @@ public DatabendProvider() { public enum Action implements AbstractAction { - INSERT(DatabendInsertGenerator::getQuery), - DELETE(DatabendDeleteGenerator::generate), + INSERT(DatabendInsertGenerator::getQuery), DELETE(DatabendDeleteGenerator::generate), // TODO 等待databend实现update // UPDATE(DatabendUpdateGenerator::getQuery), // - CREATE_VIEW(DatabendViewGenerator::generate), - EXPLAIN((g) -> { + CREATE_VIEW(DatabendViewGenerator::generate), EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); DatabendErrors.addGroupByErrors(errors); diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 0e12689ee..c82ad82ca 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -1,22 +1,29 @@ package sqlancer.databend; +import static sqlancer.databend.DatabendSchema.DatabendDataType.INT; + import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.ast.DatabendConstant; public class DatabendSchema extends AbstractSchema { @@ -158,6 +165,55 @@ public DatabendTables(List tables) { super(tables); } + public DatabendRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String rowValueQuery = String.format("SELECT %s FROM %s ORDER BY 1 LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet rs = s.executeQuery(rowValueQuery); + if (!rs.next()) { + throw new AssertionError("could not find random row " + rowValueQuery + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + DatabendColumn column = getColumns().get(i); + int columnIndex = rs.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + DatabendConstant constant; + if (rs.getString(columnIndex) == null) { + constant = DatabendConstant.createNullConstant(); + } else { + switch (column.getType().getPrimitiveDataType()) { + case INT: + constant = DatabendConstant.createIntConstant(rs.getLong(columnIndex)); + break; + case BOOLEAN: + constant = DatabendConstant.createBooleanConstant(rs.getBoolean(columnIndex)); + break; + case VARCHAR: + constant = DatabendConstant.createStringConstant(rs.getString(columnIndex)); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !rs.next(); + return new DatabendRowValue(this, values); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } + + } + + public static class DatabendRowValue extends AbstractRowValue { + + DatabendRowValue(DatabendTables tables, Map values) { + super(tables, values); + } + } public DatabendSchema(List databaseTables) { @@ -168,6 +224,12 @@ public DatabendTables getRandomTableNonEmptyTables() { return new DatabendTables(Randomly.nonEmptySubset(getDatabaseTables())); } + public DatabendTables getRandomTableNonEmptyAndViewTables() { + List tables = getDatabaseTables().stream().filter(t -> !t.isView()).collect(Collectors.toList()); + tables = Randomly.nonEmptySubset(tables); + return new DatabendTables(tables); + } + private static DatabendCompositeDataType getColumnType(String typeString) { DatabendDataType primitiveType; int size = -1; @@ -176,19 +238,19 @@ private static DatabendCompositeDataType getColumnType(String typeString) { } switch (typeString) { case "INT": - primitiveType = DatabendDataType.INT; + primitiveType = INT; size = 4; break; case "SMALLINT": - primitiveType = DatabendDataType.INT; + primitiveType = INT; size = 2; break; case "BIGINT": - primitiveType = DatabendDataType.INT; + primitiveType = INT; size = 8; break; case "TINYINT": - primitiveType = DatabendDataType.INT; + primitiveType = INT; size = 1; break; case "VARCHAR": diff --git a/src/sqlancer/databend/ast/DatabendAggregateOperation.java b/src/sqlancer/databend/ast/DatabendAggregateOperation.java new file mode 100644 index 000000000..df5e96199 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendAggregateOperation.java @@ -0,0 +1,49 @@ +package sqlancer.databend.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendAggregateOperation + extends NewFunctionNode + implements DatabendExpression { + public DatabendAggregateOperation(List args, DatabendAggregateFunction func) { + super(DatabendExprToNode.casts(args), func); + } + + public enum DatabendAggregateFunction { + MAX(1), MIN(1), AVG(1, DatabendSchema.DatabendDataType.INT, DatabendSchema.DatabendDataType.FLOAT), COUNT(1), + SUM(1, DatabendSchema.DatabendDataType.INT, DatabendSchema.DatabendDataType.FLOAT), STDDEV_POP(1), COVAR_POP(1), + COVAR_SAMP(2); + // , *_IF, *_DISTINCT + + private int nrArgs; + private DatabendSchema.DatabendDataType[] dataTypes; + + DatabendAggregateFunction(int nrArgs, DatabendSchema.DatabendDataType... dataTypes) { + this.nrArgs = nrArgs; + this.dataTypes = dataTypes.clone(); + } + + public static DatabendAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public DatabendSchema.DatabendDataType getRandomType() { + if (dataTypes.length == 0) { + return Randomly.fromOptions(DatabendSchema.DatabendDataType.values()); + } else { + return Randomly.fromOptions(dataTypes); + } + } + + public int getNrArgs() { + return nrArgs; + } + + } + +} diff --git a/src/sqlancer/databend/ast/DatabendBetweenOperation.java b/src/sqlancer/databend/ast/DatabendBetweenOperation.java new file mode 100644 index 000000000..27f5532c4 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBetweenOperation.java @@ -0,0 +1,40 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendBetweenOperation extends NewBetweenOperatorNode implements DatabendExpression { + public DatabendBetweenOperation(DatabendExpression left, DatabendExpression middle, DatabendExpression right, + boolean isTrue) { + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(middle), DatabendExprToNode.cast(right), isTrue); + } + + public DatabendExpression getLeftExpr() { + return (DatabendExpression) left; + } + + public DatabendExpression getMiddleExpr() { + return (DatabendExpression) middle; + } + + public DatabendExpression getRightExpr() { + return (DatabendExpression) right; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendBinaryComparisonOperation leftComparison = new DatabendBinaryComparisonOperation(getMiddleExpr(), + getLeftExpr(), DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator.LESS_EQUALS); + DatabendBinaryComparisonOperation rightComparison = new DatabendBinaryComparisonOperation(getLeftExpr(), + getRightExpr(), DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator.LESS_EQUALS); + return new DatabendBinaryLogicalOperation(leftComparison, rightComparison, + DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator.AND).getExpectedValue(); + } + + @Override + public DatabendSchema.DatabendDataType getExpectedType() { + return DatabendSchema.DatabendDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java index dfc44ee22..f052b3ccb 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -1,18 +1,51 @@ package sqlancer.databend.ast; +import java.util.function.BinaryOperator; + import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode { +public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode + implements DatabendExpression { - public DatabendBinaryArithmeticOperation(Node left, Node right, + public DatabendBinaryArithmeticOperation(DatabendExpression left, DatabendExpression right, BinaryOperatorNode.Operator op) { - super(left, right, op); + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); } public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Operator { - ADDITION("+"), SUBTRACTION("-"), MULTIPLICATION("*"), DIVISION("/"), MODULO("%"); + ADDITION("+") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> l + r); + } + }, + SUBTRACTION("-") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + } + }, + MODULO("%") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + } + }; private final String textRepresentation; @@ -20,10 +53,49 @@ public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Opera textRepresentation = text; } + public abstract DatabendConstant apply(DatabendConstant left, DatabendConstant right); + + public DatabendConstant applyOperation(DatabendConstant left, DatabendConstant right, BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + long leftVal = left.cast(DatabendDataType.INT).asInt(); + long rightVal = right.cast(DatabendDataType.INT).asInt(); + return DatabendConstant.createIntConstant(op.apply(leftVal, rightVal)); + } + } + @Override public String getTextRepresentation() { return textRepresentation; } } + public DatabendExpression getLeftExpr() { + return (DatabendExpression) super.getLeft(); + } + + public DatabendExpression getRightExpr() { + return (DatabendExpression) super.getRight(); + } + + public DatabendBinaryArithmeticOperator getOp() { + return (DatabendBinaryArithmeticOperator) op; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftValue = getLeftExpr().getExpectedValue(); + DatabendConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.INT; + } + } diff --git a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java index cf63ad17e..4f72dbc84 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java @@ -2,18 +2,126 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode { +public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode + implements DatabendExpression { - public DatabendBinaryComparisonOperation(Node left, Node right, + public DatabendBinaryComparisonOperation(DatabendExpression left, DatabendExpression right, DatabendBinaryComparisonOperator op) { - super(left, right, op); + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); + } + + public DatabendExpression getLeftExpression() { + return (DatabendExpression) super.getLeft(); + } + + public DatabendExpression getRightExpression() { + return (DatabendExpression) super.getRight(); + } + + public DatabendBinaryComparisonOperator getOp() { + return (DatabendBinaryComparisonOperator) op; + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftExpectedValue = getLeftExpression().getExpectedValue(); + DatabendConstant rightExpectedValue = getRightExpression().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); } public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Operator { - EQUALS("="), IS_DISTINCT("IS DISTINCT FROM"), IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), NOT_EQUALS("!="), - LESS("<"), LESS_EQUALS("<="), GREATER(">"), GREATER_EQUALS(">="); + EQUALS("=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return left.isEquals(right); + } + }, + NOT_EQUALS("!=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isEquals = left.isEquals(right); + if (isEquals.isBoolean()) { + return DatabendConstant.createBooleanConstant(!isEquals.asBoolean()); + } + return isEquals; + } + }, + IS_DISTINCT("IS DISTINCT FROM") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return DatabendConstant.createBooleanConstant(!IS_NOT_DISTINCT.apply(left, right).asBoolean()); + } + }, + IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + if (left.isNull()) { + return DatabendConstant.createBooleanConstant(right.isNull()); + } else if (right.isNull()) { + return DatabendConstant.createBooleanConstant(false); + } else { + return left.isEquals(right); + } + } + }, + LESS("<") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return left.isLessThan(right); + } + }, + LESS_EQUALS("<=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isLessThan = left.isLessThan(right); + if (isLessThan.isBoolean() && !isLessThan.asBoolean()) { + return left.isEquals(right); + } else { + return isLessThan; + } + } + }, + GREATER(">") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isEquals = left.isEquals(right); + if (isEquals.isBoolean() && isEquals.asBoolean()) { + return DatabendConstant.createBooleanConstant(false); + } else { + DatabendConstant less = left.isLessThan(right); + if (less.isNull()) { + return DatabendConstant.createNullConstant(); + } + return DatabendConstant.createBooleanConstant(!less.asBoolean()); + } + } + }, + GREATER_EQUALS(">=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isEquals = left.isEquals(right); + if (isEquals.isBoolean() && isEquals.asBoolean()) { + return DatabendConstant.createBooleanConstant(true); + } else { + DatabendConstant less = left.isLessThan(right); + if (less.isNull()) { + return DatabendConstant.createNullConstant(); + } + return DatabendConstant.createBooleanConstant(!less.asBoolean()); + } + } + }; private final String textRepresentation; @@ -21,6 +129,8 @@ public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Opera textRepresentation = text; } + public abstract DatabendConstant apply(DatabendConstant left, DatabendConstant right); + @Override public String getTextRepresentation() { return textRepresentation; diff --git a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java index 355ec88a5..4ee7aa705 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java @@ -3,17 +3,92 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode { +public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode + implements DatabendExpression { - public DatabendBinaryLogicalOperation(Node left, Node right, + public DatabendBinaryLogicalOperation(DatabendExpression left, DatabendExpression right, DatabendBinaryLogicalOperator op) { - super(left, right, op); + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); + } + + public DatabendExpression getLeftExpr() { + return (DatabendExpression) super.getLeft(); + } + + public DatabendExpression getRightExpr() { + return (DatabendExpression) super.getRight(); + } + + public DatabendBinaryLogicalOperator getOp() { + return (DatabendBinaryLogicalOperator) op; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftValue = getLeftExpr().getExpectedValue(); + DatabendConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; } public enum DatabendBinaryLogicalOperator implements BinaryOperatorNode.Operator { - AND("AND", "and"), OR("OR", "or"); + AND("AND", "and") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant leftVal = left.cast(DatabendDataType.BOOLEAN); + DatabendConstant rightVal = right.cast(DatabendDataType.BOOLEAN); + assert (leftVal.isNull() || leftVal.isBoolean()) : leftVal + "不是NULL也不是Boolean类型"; + assert (rightVal.isNull() || rightVal.isBoolean()) : rightVal + "不是NULL也不是Boolean类型"; + if (leftVal.isNull()) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + if (rightVal.asBoolean()) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(false); + } + } + } else if (!leftVal.asBoolean()) { + return DatabendConstant.createBooleanConstant(false); + } + assert leftVal.asBoolean(); + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(rightVal.asBoolean()); + } + } + }, + OR("OR", "or") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant leftVal = left.cast(DatabendDataType.BOOLEAN); + DatabendConstant rightVal = right.cast(DatabendDataType.BOOLEAN); + assert (leftVal.isNull() || leftVal.isBoolean()) : leftVal + "不是NULL也不是Boolean类型"; + assert (rightVal.isNull() || rightVal.isBoolean()) : rightVal + "不是NULL也不是Boolean类型"; + if (leftVal.isBoolean() && leftVal.asBoolean()) { + return DatabendConstant.createBooleanConstant(true); + } + if (rightVal.isBoolean() && rightVal.asBoolean()) { + return DatabendConstant.createBooleanConstant(true); + } + if (leftVal.isNull() || rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } + return DatabendConstant.createBooleanConstant(false); + } + }; private final String[] textRepresentations; @@ -33,6 +108,9 @@ public DatabendBinaryLogicalOperator getRandomOp() { public static DatabendBinaryLogicalOperator getRandom() { return Randomly.fromOptions(values()); } + + public abstract DatabendConstant apply(DatabendConstant left, DatabendConstant right); + } } diff --git a/src/sqlancer/databend/ast/DatabendCastOperation.java b/src/sqlancer/databend/ast/DatabendCastOperation.java index c1b3ffe66..534836d59 100644 --- a/src/sqlancer/databend/ast/DatabendCastOperation.java +++ b/src/sqlancer/databend/ast/DatabendCastOperation.java @@ -3,17 +3,39 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.ast.newast.Node; -import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendCastOperation extends NewUnaryPostfixOperatorNode { +public class DatabendCastOperation extends NewUnaryPostfixOperatorNode + implements DatabendExpression { - public DatabendCastOperation(Node expr, DatabendSchema.DatabendCompositeDataType type) { + DatabendDataType type; + + public DatabendCastOperation(Node expr, DatabendCompositeDataType type) { super(expr, new BinaryOperatorNode.Operator() { @Override public String getTextRepresentation() { return "::" + type.toString(); } }); + this.type = type.getPrimitiveDataType(); + } + + DatabendExpression getExpression() { + return (DatabendExpression) getExpr(); } + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type); + } + + @Override + public DatabendDataType getExpectedType() { + return type; + } } diff --git a/src/sqlancer/databend/ast/DatabendColumnValue.java b/src/sqlancer/databend/ast/DatabendColumnValue.java new file mode 100644 index 000000000..b6067a050 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendColumnValue.java @@ -0,0 +1,29 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendColumnValue extends ColumnReferenceNode + implements DatabendExpression { + + private final DatabendConstant expectedValue; + + public DatabendColumnValue(DatabendColumn column, DatabendConstant value) { + super(column); + this.expectedValue = value; + } + + public DatabendConstant getExpectedValue() { + return expectedValue; + } + + public DatabendDataType getExpectedType() { + return getColumn().getType().getPrimitiveDataType(); + } + + public static DatabendColumnValue create(DatabendColumn column, DatabendConstant value) { + return new DatabendColumnValue(column, value); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 39056aff9..0cc3d0bd2 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -4,12 +4,57 @@ import java.text.SimpleDateFormat; import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendConstant implements Node { +public abstract class DatabendConstant implements Node, DatabendExpression { private DatabendConstant() { } + public boolean isNull() { + return false; + } + + public boolean isInt() { + return false; + } + + public boolean isBoolean() { + return false; + } + + public boolean isString() { + return false; + } + + public boolean isFloat() { + return false; + } + + public abstract DatabendConstant cast(DatabendDataType dataType); + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asFloat() { + throw new UnsupportedOperationException(this.toString()); + } + + public abstract DatabendConstant isEquals(DatabendConstant rightVal); + + public abstract DatabendConstant isLessThan(DatabendConstant rightVal); + + // public abstract String getTextRepresentation(); + public static class DatabendNullConstant extends DatabendConstant { @Override @@ -17,6 +62,35 @@ public String toString() { return "NULL"; } + @Override + public boolean isNull() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + return DatabendConstant.createNullConstant(); + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return DatabendConstant.createNullConstant(); + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + return DatabendConstant.createNullConstant(); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.NULL; + } + + // @Override + // public DatabendConstant getExpectedValue() { + // return super.getExpectedValue(); + // } } public static class DatabendIntConstant extends DatabendConstant { @@ -36,13 +110,66 @@ public long getValue() { return value; } + @Override + public boolean isInt() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case BOOLEAN: + return new DatabendBooleanConstant(value != 0); + case INT: + return this; + case VARCHAR: + return new DatabendStringConstant(String.valueOf(value)); + default: + return null; + } + } + + @Override + public long asInt() { + return value; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DatabendConstant.createBooleanConstant(value == rightVal.asInt()); + } else { + throw new AssertionError(rightVal); + } + + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.INT; + } } - public static class DatabendDoubleConstant extends DatabendConstant { + public static class DatabendFloatConstant extends DatabendConstant { private final double value; - public DatabendDoubleConstant(double value) { + public DatabendFloatConstant(double value) { this.value = value; } @@ -50,6 +177,11 @@ public double getValue() { return value; } + @Override + public boolean isFloat() { + return true; + } + @Override public String toString() { if (value == Double.POSITIVE_INFINITY) { @@ -61,13 +193,51 @@ public String toString() { return String.valueOf(value); } + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case FLOAT: + return this; + case INT: + return DatabendConstant.createIntConstant((long) value); + case BOOLEAN: + return DatabendConstant.createBooleanConstant(value != 0); + case VARCHAR: + return DatabendConstant.createStringConstant(String.valueOf(value)); + default: + return null; + } + } + + @Override + public double asFloat() { + return value; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return null; + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } } - public static class DatabendTextConstant extends DatabendConstant { + public static class DatabendStringConstant extends DatabendConstant { private final String value; - public DatabendTextConstant(String value) { + public DatabendStringConstant(String value) { this.value = value; } @@ -80,25 +250,68 @@ public String toString() { return "'" + value.replace("'", "''") + "'"; } - } - - public static class DatabendBitConstant extends DatabendConstant { - - private final String value; + @Override + public String asString() { + return value; + } - public DatabendBitConstant(long value) { - this.value = Long.toBinaryString(value); + @Override + public boolean isString() { + return true; } - public String getValue() { - return value; + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case VARCHAR: + return this; + case INT: + try { + return new DatabendIntConstant(Long.parseLong(value)); + } catch (NumberFormatException e) { + return new DatabendIntConstant(-1); + } + case BOOLEAN: + if ("false".contentEquals(value.toLowerCase())) { + return new DatabendBooleanConstant(false); + } else if ("true".contentEquals(value.toLowerCase())) { + return new DatabendBooleanConstant(true); + } else { + throw new AssertionError(String.format("string: %s, cannot be forced to boolean", value)); + } + case FLOAT: + try { + return new DatabendFloatConstant(Double.parseDouble(value)); + } catch (NumberFormatException e) { + return new DatabendFloatConstant(-1); + } + default: + return null; + } } @Override - public String toString() { - return "B'" + value + "'"; + public DatabendConstant isEquals(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isString()) { + return DatabendConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + // TODO 可以比较 date和timestamp类型，待添加 + throw new AssertionError(rightVal); + } } + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isString()) { + return DatabendConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } } public static class DatabendDateConstant extends DatabendConstant { @@ -120,6 +333,20 @@ public String toString() { return String.format("DATE '%s'", textRepr); } + @Override + public DatabendConstant cast(DatabendDataType dataType) { + return null; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return null; + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + return null; + } } public static class DatabendTimestampConstant extends DatabendConstant { @@ -141,6 +368,20 @@ public String toString() { return String.format("TIMESTAMP '%s'", textRepr); } + @Override + public DatabendConstant cast(DatabendDataType dataType) { + return null; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return null; + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + return null; + } } public static class DatabendBooleanConstant extends DatabendConstant { @@ -160,33 +401,82 @@ public String toString() { return String.valueOf(value); } + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case BOOLEAN: + return this; + case INT: + return new DatabendIntConstant(value ? 1 : 0); + case FLOAT: + return new DatabendFloatConstant(value ? 1 : 0); + case VARCHAR: + return new DatabendStringConstant(value ? "1" : "0"); + default: + return null; + } + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return DatabendConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else { + throw new AssertionError(rightVal); + } + + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return DatabendConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } else { + throw new AssertionError(rightVal); + } + } + } - public static Node createStringConstant(String text) { - return new DatabendTextConstant(text); + public static DatabendConstant createStringConstant(String text) { + return new DatabendStringConstant(text); } - public static Node createFloatConstant(double val) { - return new DatabendDoubleConstant(val); + public static DatabendConstant createFloatConstant(double val) { + return new DatabendFloatConstant(val); } - public static Node createIntConstant(long val) { + public static DatabendConstant createIntConstant(long val) { return new DatabendIntConstant(val); } - public static Node createNullConstant() { + public static DatabendConstant createNullConstant() { return new DatabendNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static DatabendConstant createBooleanConstant(boolean val) { return new DatabendBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static DatabendConstant createDateConstant(long integer) { return new DatabendDateConstant(integer); } - public static Node createTimestampConstant(long integer) { + public static DatabendConstant createTimestampConstant(long integer) { return new DatabendTimestampConstant(integer); } diff --git a/src/sqlancer/databend/ast/DatabendExpression.java b/src/sqlancer/databend/ast/DatabendExpression.java index 31b9bf859..d53952140 100644 --- a/src/sqlancer/databend/ast/DatabendExpression.java +++ b/src/sqlancer/databend/ast/DatabendExpression.java @@ -1,5 +1,14 @@ package sqlancer.databend.ast; +import sqlancer.databend.DatabendSchema.DatabendDataType; + public interface DatabendExpression { + default DatabendDataType getExpectedType() { + return null; + } + + default DatabendConstant getExpectedValue() { + return null; + } } diff --git a/src/sqlancer/databend/ast/DatabendFunctionOperation.java b/src/sqlancer/databend/ast/DatabendFunctionOperation.java new file mode 100644 index 000000000..855e1c8f1 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendFunctionOperation.java @@ -0,0 +1,4 @@ +package sqlancer.databend.ast; + +public class DatabendFunctionOperation { // TODO add databend function +} diff --git a/src/sqlancer/databend/ast/DatabendInOperation.java b/src/sqlancer/databend/ast/DatabendInOperation.java new file mode 100644 index 000000000..304c3e3f9 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendInOperation.java @@ -0,0 +1,53 @@ +package sqlancer.databend.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendInOperation extends NewInOperatorNode implements DatabendExpression { + + private final DatabendExpression leftExpr; + private final List rightExpr; + + public DatabendInOperation(DatabendExpression left, List right, boolean isNegated) { + super(DatabendExprToNode.cast(left), DatabendExprToNode.casts(right), isNegated); + this.leftExpr = left; + this.rightExpr = right; + } + + @Override + public DatabendSchema.DatabendDataType getExpectedType() { + return DatabendSchema.DatabendDataType.BOOLEAN; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftValue = leftExpr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return DatabendConstant.createNullConstant(); + } + boolean isNull = false; + for (DatabendExpression expr : rightExpr) { + DatabendConstant rightValue = expr.getExpectedValue(); + if (rightValue == null) { + return null; + } + if (rightValue.isNull()) { + isNull = true; + } else if (rightValue.isEquals(leftValue).isBoolean() && rightValue.isEquals(leftValue).asBoolean()) { + return DatabendConstant.createBooleanConstant(!isNegated()); + } + } + + if (isNull) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(isNegated()); + } + } +} diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index 899929873..123916542 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -6,6 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -83,18 +84,18 @@ public static List> getJoins( switch (JoinType.getRandom()) { case INNER: joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); break; case NATURAL: joinExpressions.add(DatabendJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); break; case LEFT: joinExpressions.add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); break; case RIGHT: joinExpressions.add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); break; default: throw new AssertionError(); diff --git a/src/sqlancer/databend/ast/DatabendLikeOperation.java b/src/sqlancer/databend/ast/DatabendLikeOperation.java index 2755970e7..36ac73603 100644 --- a/src/sqlancer/databend/ast/DatabendLikeOperation.java +++ b/src/sqlancer/databend/ast/DatabendLikeOperation.java @@ -1,15 +1,48 @@ package sqlancer.databend.ast; +import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendLikeOperation extends NewBinaryOperatorNode { +public class DatabendLikeOperation extends NewBinaryOperatorNode implements DatabendExpression { - public DatabendLikeOperation(Node left, Node right, - DatabendLikeOperator op) { - super(left, right, op); + public DatabendLikeOperation(DatabendExpression left, DatabendExpression right, DatabendLikeOperator op) { + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; + } + + public DatabendExpression getLeftExpr() { + return (DatabendExpression) super.getLeft(); + } + + public DatabendExpression getRightExpr() { + return (DatabendExpression) super.getRight(); + } + + public DatabendLikeOperator getOp() { + return (DatabendLikeOperator) op; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftVal = getLeftExpr().getExpectedValue(); + DatabendConstant rightVal = getRightExpr().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } + if (leftVal.isNull() || rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + boolean result = LikeImplementationHelper.match(leftVal.asString(), rightVal.asString(), 0, 0, true); + return DatabendConstant.createBooleanConstant(result); + } } public enum DatabendLikeOperator implements BinaryOperatorNode.Operator { diff --git a/src/sqlancer/databend/ast/DatabendOrderByTerm.java b/src/sqlancer/databend/ast/DatabendOrderByTerm.java new file mode 100644 index 000000000..5c78bae3f --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendOrderByTerm.java @@ -0,0 +1,10 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.Node; + +public class DatabendOrderByTerm extends NewOrderingTerm implements DatabendExpression { + public DatabendOrderByTerm(Node expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java index e01c35282..a19512edf 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -3,23 +3,36 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { +public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements DatabendExpression { - // private final Node expr; - // private final DatabendUnaryPostfixOperator op; - private boolean negate; + public DatabendUnaryPostfixOperation(DatabendExpression expr, DatabendUnaryPostfixOperator op) { + super(DatabendExprToNode.cast(expr), op); + } + + public DatabendExpression getExpression() { + return (DatabendExpression) getExpr(); + } + + public DatabendUnaryPostfixOperator getOp() { + return (DatabendUnaryPostfixOperator) op; + } - public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op, - boolean negate) { - super(expr, op); - setNegate(negate); + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; } - public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op) { - super(expr, op); + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); } public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator { @@ -28,12 +41,22 @@ public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator public DatabendDataType[] getInputDataTypes() { return DatabendDataType.values(); } + + @Override + public DatabendConstant apply(DatabendConstant value) { + return DatabendConstant.createBooleanConstant(value.isNull()); + } }, IS_NOT_NULL("IS NOT NULL") { @Override public DatabendDataType[] getInputDataTypes() { return DatabendDataType.values(); } + + @Override + public DatabendConstant apply(DatabendConstant value) { + return DatabendConstant.createBooleanConstant(!value.isNull()); + } }; // IS @@ -54,19 +77,7 @@ public String getTextRepresentation() { public abstract DatabendDataType[] getInputDataTypes(); - } - - public boolean isNegated() { - return negate; - } - - public void setNegate(boolean negate) { - this.negate = negate; - } - - // @Override - public Node getExpression() { - return getExpr(); + public abstract DatabendConstant apply(DatabendConstant value); } @Override @@ -74,8 +85,4 @@ public String getOperatorRepresentation() { return this.op.getTextRepresentation(); } - // @Override - // public OperatorKind getOperatorKind() { - // return OperatorKind.POSTFIX; - // } } diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index 69c95a745..f115684e0 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -3,72 +3,88 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { +public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements DatabendExpression { - // private final Node expr; - // private final DatabendUnaryPrefixOperator op; - // private boolean negate; - - public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op) { - super(expr, op); + public DatabendUnaryPrefixOperation(DatabendExpression expr, DatabendUnaryPrefixOperator op) { + super(DatabendExprToNode.cast(expr), op); } - // public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op, boolean - // negate) { - // super(expr,op); - // setNegate(negate); - // } + public DatabendExpression getExpression() { + return (DatabendExpression) getExpr(); + } - // void setNegate(boolean negate){ - // this.negate = negate; - // } + public DatabendUnaryPrefixOperator getOp() { + return (DatabendUnaryPrefixOperator) op; + } - // @Override - public Node getExpression() { - return getExpr(); + @Override + public DatabendDataType getExpectedType() { + return getOp().getExpressionType(getExpression()); } - // @Override - // public OperatorKind getOperatorKind() { - // return OperatorKind.PREFIX; - // } + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } public enum DatabendUnaryPrefixOperator implements BinaryOperatorNode.Operator { NOT("NOT", DatabendDataType.BOOLEAN, DatabendDataType.INT) { @Override - public DatabendDataType getExpressionType() { + public DatabendDataType getExpressionType(DatabendExpression expr) { return DatabendDataType.BOOLEAN; } @Override - protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { - return null; // TODO + protected DatabendConstant apply(DatabendConstant value) { + if (value.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(!value.cast(DatabendDataType.BOOLEAN).asBoolean()); + } } }, UNARY_PLUS("+", DatabendDataType.INT) { @Override - public DatabendDataType getExpressionType() { - return DatabendDataType.INT; + public DatabendDataType getExpressionType(DatabendExpression expr) { + return expr.getExpectedType(); } @Override - protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { - return expectedValue; + protected DatabendConstant apply(DatabendConstant value) { + return value; } }, UNARY_MINUS("-", DatabendDataType.INT) { @Override - public DatabendDataType getExpressionType() { - return DatabendDataType.INT; + public DatabendDataType getExpressionType(DatabendExpression expr) { + return expr.getExpectedType(); } @Override - protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { - return null; + protected DatabendConstant apply(DatabendConstant value) { + if (value.isNull()) { + return DatabendConstant.createNullConstant(); + } + try { + if (value.isInt()) { + return DatabendConstant.createIntConstant(-value.asInt()); + } else if (value.isFloat()) { + return DatabendConstant.createFloatConstant(-value.asFloat()); + } else { + return null; + } + } catch (UnsupportedOperationException e) { + return null; + } } }; @@ -80,13 +96,13 @@ protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { this.dataTypes = dataTypes.clone(); } - public abstract DatabendDataType getExpressionType(); + public abstract DatabendDataType getExpressionType(DatabendExpression expr); public DatabendDataType getRandomInputDataTypes() { return Randomly.fromOptions(dataTypes); } - protected abstract DatabendConstant getExpectedValue(DatabendConstant expectedValue); + protected abstract DatabendConstant apply(DatabendConstant value); @Override public String getTextRepresentation() { diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index dcef8d7a8..e13ccdc7c 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -4,26 +4,27 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; public class DatabendDeleteGenerator { - public DatabendDeleteGenerator() { + private DatabendDeleteGenerator() { } - public static SQLQueryAdapter generate(DatabendGlobalState globalState){ + public static SQLQueryAdapter generate(DatabendGlobalState globalState) { StringBuilder sb = new StringBuilder("DELETE FROM "); ExpectedErrors errors = new ExpectedErrors(); sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); - if(Randomly.getBoolean()) { + if (Randomly.getBoolean()) { sb.append(" WHERE "); - sb.append(DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) - .generateExpression(DatabendDataType.BOOLEAN))); + sb.append(DatabendToStringVisitor.asString(DatabendExprToNode.cast( + new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN)))); DatabendErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(),errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 340e494ef..a86f0f3a9 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -8,6 +8,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; @@ -48,8 +49,9 @@ protected void insertValue(DatabendColumn column) { // } else { // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); // } - String value = DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) - .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 + String value = DatabendToStringVisitor + .asString(DatabendExprToNode.cast(new DatabendNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); // 生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index a67a5d4ff..7e2c48e00 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -2,50 +2,106 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendRowValue; +import sqlancer.databend.ast.DatabendAggregateOperation; +import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; +import sqlancer.databend.ast.DatabendBetweenOperation; import sqlancer.databend.ast.DatabendBinaryArithmeticOperation; import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; import sqlancer.databend.ast.DatabendBinaryComparisonOperation; import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; import sqlancer.databend.ast.DatabendBinaryLogicalOperation; import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; +import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendInOperation; import sqlancer.databend.ast.DatabendLikeOperation; +import sqlancer.databend.ast.DatabendOrderByTerm; import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; public class DatabendNewExpressionGenerator - extends TypedExpressionGenerator, DatabendColumn, DatabendDataType> { + extends TypedExpressionGenerator { private final DatabendGlobalState globalState; + + private final int maxDepth; private boolean allowAggregateFunctions; + private DatabendRowValue rowValue; + + private Set columnOfLeafNode; + + public DatabendNewExpressionGenerator setRowValue(DatabendRowValue rowValue) { + this.rowValue = rowValue; + return this; + } + + public void setColumnOfLeafNode(Set columnOfLeafNode) { + this.columnOfLeafNode = columnOfLeafNode; + } public DatabendNewExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); } @Override - public Node generateLeafNode(DatabendDataType dataType) { - return generateConstant(dataType); + public DatabendExpression generateLeafNode(DatabendDataType dataType) { + if (Randomly.getBoolean()) { + return generateConstant(dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(dataType); + } else { + return createColumnOfType(dataType); + } + } + } + + final List filterColumns(DatabendDataType dataType) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType().getPrimitiveDataType() == dataType) + .collect(Collectors.toList()); + } + } + + private DatabendExpression createColumnOfType(DatabendDataType type) { + List columns = filterColumns(type); + DatabendColumn column = Randomly.fromList(columns); + DatabendConstant value = rowValue == null ? null : rowValue.getValues().get(column); + if (columnOfLeafNode != null) { + columnOfLeafNode.add(DatabendColumnValue.create(column, value)); + } + return DatabendColumnValue.create(column, value); + } + + public List> generateOrderBy() { + List randomColumns = Randomly.subset(columns); + return randomColumns.stream() + .map(c -> new DatabendOrderByTerm(new DatabendColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) + .collect(Collectors.toList()); } @Override - protected Node generateExpression(DatabendDataType type, int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + protected DatabendExpression generateExpression(DatabendDataType type, int depth) { + if (Randomly.getBooleanWithRatherLowProbability() || depth >= maxDepth) { return generateLeafNode(type); } @@ -63,8 +119,8 @@ protected Node generateExpression(DatabendDataType type, int } } - public List> generateExpressions(int nr, DatabendDataType type) { - List> expressions = new ArrayList<>(); + public List generateExpressions(int nr, DatabendDataType type) { + List expressions = new ArrayList<>(); for (int i = 0; i < nr; i++) { expressions.add(generateExpression(type)); } @@ -75,7 +131,7 @@ private enum IntExpression { UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION } - private Node generateIntExpression(int depth) { + private DatabendExpression generateIntExpression(int depth) { if (allowAggregateFunctions) { allowAggregateFunctions = false; } @@ -99,7 +155,7 @@ private enum BooleanExpression { // SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; } - Node generateBooleanExpression(int depth) { + DatabendExpression generateBooleanExpression(int depth) { if (allowAggregateFunctions) { allowAggregateFunctions = false; } @@ -126,40 +182,39 @@ Node generateBooleanExpression(int depth) { } - Node getPostfix(int depth) { + DatabendExpression getPostfix(int depth) { DatabendUnaryPostfixOperator randomOp = DatabendUnaryPostfixOperator.getRandom(); return new DatabendUnaryPostfixOperation( - generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp, - Randomly.getBoolean()); + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp); } - Node getNOT(int depth) { + DatabendExpression getNOT(int depth) { DatabendUnaryPrefixOperator op = DatabendUnaryPrefixOperator.NOT; return new DatabendUnaryPrefixOperation(generateExpression(op.getRandomInputDataTypes(), depth), op); } - Node getBetween(int depth) { + DatabendExpression getBetween(int depth) { // 跳过boolean DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); - return new NewBetweenOperatorNode(generateExpression(dataType, depth), - generateExpression(dataType, depth), generateExpression(dataType, depth), Randomly.getBoolean()); + return new DatabendBetweenOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + generateExpression(dataType, depth), Randomly.getBoolean()); } - Node getIn(int depth) { + DatabendExpression getIn(int depth) { DatabendDataType dataType = Randomly.fromOptions(DatabendDataType.values()); - Node leftExpr = generateExpression(dataType, depth); - List> rightExprs = new ArrayList<>(); + DatabendExpression leftExpr = generateExpression(dataType, depth); + List rightExprs = new ArrayList<>(); int nr = Randomly.smallNumber() + 1; for (int i = 0; i < nr; i++) { rightExprs.add(generateExpression(dataType, depth)); } - return new NewInOperatorNode(leftExpr, rightExprs, Randomly.getBoolean()); + return new DatabendInOperation(leftExpr, rightExprs, Randomly.getBoolean()); } - Node getBinaryLogical(int depth, DatabendDataType dataType) { - Node expr = generateExpression(dataType, depth); + DatabendExpression getBinaryLogical(int depth, DatabendDataType dataType) { + DatabendExpression expr = generateExpression(dataType, depth); int nr = Randomly.smallNumber() + 1; for (int i = 0; i < nr; i++) { expr = new DatabendBinaryLogicalOperation(expr, generateExpression(DatabendDataType.BOOLEAN, depth), @@ -168,37 +223,47 @@ Node getBinaryLogical(int depth, DatabendDataType dataType) return expr; } - Node getComparison(int depth) { + DatabendExpression getComparison(int depth) { // 跳过boolean DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); - Node leftExpr = generateExpression(dataType, depth); - Node rightExpr = generateExpression(dataType, depth); + DatabendExpression leftExpr = generateExpression(dataType, depth); + DatabendExpression rightExpr = generateExpression(dataType, depth); return new DatabendBinaryComparisonOperation(leftExpr, rightExpr, Randomly.fromOptions(DatabendBinaryComparisonOperator.values())); } - Node getLike(int depth, DatabendDataType dataType) { + DatabendExpression getLike(int depth, DatabendDataType dataType) { return new DatabendLikeOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), DatabendLikeOperation.DatabendLikeOperator.LIKE_OPERATOR); } + public DatabendExpression generateExpressionWithExpectedResult(DatabendDataType type) { + // DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + // gen.setRowValue(rowValue); + DatabendExpression expr; + do { + expr = this.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + @Override - public Node generatePredicate() { + public DatabendExpression generatePredicate() { return generateExpression(DatabendDataType.BOOLEAN); } @Override - public Node negatePredicate(Node predicate) { + public DatabendExpression negatePredicate(DatabendExpression predicate) { return new DatabendUnaryPrefixOperation(predicate, DatabendUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node predicate) { + public DatabendExpression isNull(DatabendExpression predicate) { return new DatabendUnaryPostfixOperation(predicate, DatabendUnaryPostfixOperator.IS_NULL); } - public Node generateConstant(DatabendDataType type, boolean isNullable) { + public DatabendExpression generateConstant(DatabendDataType type, boolean isNullable) { if (isNullable && Randomly.getBooleanWithSmallProbability()) { createConstant(DatabendDataType.NULL); } @@ -206,14 +271,14 @@ public Node generateConstant(DatabendDataType type, boolean } @Override - public Node generateConstant(DatabendDataType type) { + public DatabendExpression generateConstant(DatabendDataType type) { if (Randomly.getBooleanWithSmallProbability()) { return DatabendConstant.createNullConstant(); } return createConstant(type); } - public Node createConstant(DatabendDataType type) { + public DatabendExpression createConstant(DatabendDataType type) { Randomly r = globalState.getRandomly(); switch (type) { case INT: @@ -234,7 +299,7 @@ public Node createConstant(DatabendDataType type) { } @Override - protected Node generateColumn(DatabendDataType type) { + protected DatabendExpression generateColumn(DatabendDataType type) { return null; } @@ -248,52 +313,20 @@ protected boolean canGenerateColumnOfType(DatabendDataType type) { return false; } - public enum DatabendAggregateFunction { - MAX(1), MIN(1), AVG(1, DatabendDataType.INT, DatabendDataType.FLOAT), COUNT(1), - SUM(1, DatabendDataType.INT, DatabendDataType.FLOAT), STDDEV_POP(1), COVAR_POP(1), COVAR_SAMP(2); - //, *_IF, *_DISTINCT - - private int nrArgs; - private DatabendDataType[] dataTypes; - - DatabendAggregateFunction(int nrArgs, DatabendDataType... dataTypes) { - this.nrArgs = nrArgs; - this.dataTypes = dataTypes.clone(); - } - - public static DatabendAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public DatabendDataType getRandomType() { - if (dataTypes.length == 0) { - return Randomly.fromOptions(DatabendDataType.values()); - } else { - return Randomly.fromOptions(dataTypes); - } - } - - public int getNrArgs() { - return nrArgs; - } - - } - - public NewFunctionNode generateArgsForAggregate( - DatabendAggregateFunction aggregateFunction) { - return new NewFunctionNode( + public DatabendExpression generateArgsForAggregate(DatabendAggregateFunction aggregateFunction) { + return new DatabendAggregateOperation( generateExpressions(aggregateFunction.getNrArgs(), aggregateFunction.getRandomType()), aggregateFunction); } - public Node generateAggregate() { + public DatabendExpression generateAggregate() { DatabendAggregateFunction aggrFunc = DatabendAggregateFunction.getRandom(); return generateArgsForAggregate(aggrFunc); } - public Node generateHavingClause() { + public DatabendExpression generateHavingClause() { allowAggregateFunctions = true; - Node expression = generateExpression(DatabendDataType.BOOLEAN); + DatabendExpression expression = generateExpression(DatabendDataType.BOOLEAN); allowAggregateFunctions = false; return expression; } diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 70422b324..93f7b5262 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -1,18 +1,20 @@ package sqlancer.databend.gen; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -24,26 +26,29 @@ private DatabendRandomQuerySynthesizer() { } public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { - DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyAndViewTables(); List targetColumns = targetTables.getColumns(); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(targetColumns); - DatabendSelect select = new DatabendSelect(); - // TODO distinct - select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); List> columns = new ArrayList<>(); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); int freeColumns = targetColumns.size(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = null; + Node column = null; if (freeColumns > 0 && Randomly.getBoolean()) { - expression = new ColumnReferenceNode<>(targetColumns.get(freeColumns - 1)); + column = new DatabendColumnValue(targetColumns.get(freeColumns - 1), null); freeColumns -= 1; + columnOfLeafNode.add((DatabendColumnValue) column); } else { - expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); + column = DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - columns.add(expression); + columns.add(column); } + DatabendSelect select = new DatabendSelect(); + boolean isDistinct = Randomly.getBoolean(); + select.setDistinct(isDistinct); select.setFetchColumns(columns); List tables = targetTables.getTables(); List> tableList = tables.stream() @@ -52,14 +57,28 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setJoinList(joins); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); + select.setWhereClause( + DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); } - if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + + List> noExprColumns = new ArrayList<>(columnOfLeafNode); + + if (Randomly.getBoolean() && noExprColumns.size() > 0) { + if (!isDistinct) { + select.setOrderByExpressions(Randomly.nonEmptySubset(noExprColumns)); + } + // TODO (for SELECT DISTINCT, ORDER BY expressions must appear in select list) isDistinct + // 需要orderby输入每个select list，可以用数字代替比如：1,2,3... } - if (Randomly.getBoolean()) { // TODO 该表达式生成的类型较多，需要对其限制 - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { // 可能产生新的column叶子结点 + select.setHavingClause(DatabendExprToNode.cast(gen.generateHavingClause())); + } + + noExprColumns = new ArrayList<>(columnOfLeafNode); + + if (Randomly.getBoolean() && noExprColumns.size() > 0) { + select.setGroupByExpressions(noExprColumns); } if (Randomly.getBoolean()) { @@ -71,10 +90,6 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } - if (Randomly.getBoolean()) { - select.setHavingClause(gen.generateHavingClause()); - } - return select; } diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 32505f98e..a34279fd2 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -4,10 +4,10 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -25,7 +25,7 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(tableName); sb.append("("); List columns = getNewColumns(); - TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = new DatabendNewExpressionGenerator( + TypedExpressionGenerator gen = new DatabendNewExpressionGenerator( globalState).setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -45,7 +45,8 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 - gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); + DatabendExprToNode + .cast(gen.generateConstant(columns.get(i).getType().getPrimitiveDataType())))); sb.append(")"); } } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index c6d9c9229..e0c014f2b 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -18,6 +18,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -44,10 +45,14 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { @Override public void check() throws SQLException { - DatabendTables randomTables = s.getRandomTableNonEmptyTables(); // 随机获得nr张表 + DatabendTables randomTables = s.getRandomTableNonEmptyAndViewTables(); // 随机获得nr张表 List columns = randomTables.getColumns(); + if (columns.isEmpty()) { + debugColumns(columns, randomTables); // 调试代码，可忽略 + } DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件 + Node randomWhereCondition = DatabendExprToNode + .cast(gen.generateExpression(DatabendDataType.BOOLEAN)); // 生成随机where条件 List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -108,8 +113,8 @@ private int getOptimizedQueryCount(SQLConnection con, List columns, DatabendTables randomTables) { + DatabendTables test = new DatabendTables(s.getDatabaseTables()); + System.out.println(String.format("tables size: %d", test.getTables().size())); + for (DatabendTable table : test.getTables()) { + System.out.println(String.format("%s", table.getName())); + for (DatabendColumn column : table.getColumns()) { + System.out.println(String.format("%s %s", column.getName(), column.getType())); + } + System.out.println("------------------------"); + } + System.out.println("+++++++++++++++++++++++++++++"); + for (DatabendTable table : randomTables.getTables()) { + System.out.println(String.format("%s", table.getName())); + for (DatabendColumn column : table.getColumns()) { + System.out.println(String.format("%s %s", column.getName(), column.getType())); + } + System.out.println("------------------------"); + } + throw new AssertionError( + String.format("randomTables size: %d,column is empty", randomTables.getTables().size())); + } + } diff --git a/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..bf0bf6bea --- /dev/null +++ b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java @@ -0,0 +1,154 @@ +package sqlancer.databend.test; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExpectedValueVisitor; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendRowValue; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendColumnValue; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; + +public class DatabendPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + + public DatabendPivotedQuerySynthesisOracle(DatabendGlobalState globalState) { + super(globalState); + DatabendErrors.addExpressionErrors(errors); + DatabendErrors.addInsertErrors(errors); + } + + @Override + protected Query getRectifiedQuery() throws Exception { + DatabendTables randomTables = globalState.getSchema().getRandomTableNonEmptyAndViewTables(); + List columns = randomTables.getColumns(); + DatabendSelect selectStatement = new DatabendSelect(); + boolean isDistinct = Randomly.getBoolean(); + selectStatement.setDistinct(isDistinct); + pivotRow = randomTables.getRandomRowValue(globalState.getConnection()); + fetchColumns = columns; + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new DatabendColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + selectStatement.setFromList(randomTables.getTables().stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList())); + DatabendExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(DatabendExprToNode.cast(whereClause)); + List> groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + Node limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + Node offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + if (!isDistinct) { + List> orderBys = gen.generateOrderBy(); + selectStatement.setOrderByExpressions(orderBys); + } + return new SQLQueryAdapter(DatabendToStringVisitor.asString(selectStatement), errors); + } + + private DatabendExpression generateRectifiedExpression(List columns, DatabendRowValue pivotRow) { + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + gen.setRowValue(pivotRow); + DatabendExpression expr = gen.generateExpressionWithExpectedResult(DatabendDataType.BOOLEAN); + DatabendExpression result = null; + if (expr.getExpectedValue().isNull()) { + result = new DatabendUnaryPostfixOperation(expr, + DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator.IS_NULL); + } else if (!expr.getExpectedValue().cast(DatabendDataType.BOOLEAN).asBoolean()) { + result = new DatabendUnaryPrefixOperation(expr, + DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator.NOT); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); + sb.append(pivotRowQuery.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (DatabendColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL "); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).toString()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + private DatabendColumn getFetchValueAliasedColumn(DatabendColumn c) { + DatabendColumn aliasedColumn = new DatabendColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType(), false, false); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + @Override + protected String getExpectedValues(DatabendExpression expr) { + return DatabendExpectedValueVisitor.asExpectedValues(DatabendExprToNode.cast(expr)); + } + + private List> generateGroupByClause(List columns, + DatabendRowValue rowValue) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> new DatabendColumnValue(c, rowValue.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private Node generateLimit() { + if (Randomly.getBoolean()) { + return DatabendConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private Node generateOffset() { + if (Randomly.getBoolean()) { + return DatabendConstant.createIntConstant(0); + } else { + return null; + } + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java similarity index 91% rename from src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 56f311f2b..ca591971f 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -18,17 +18,19 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendAggregateOperation; +import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; import sqlancer.databend.ast.DatabendCastOperation; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; -import sqlancer.databend.gen.DatabendNewExpressionGenerator.DatabendAggregateFunction; public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { @@ -48,12 +50,12 @@ public void check() throws SQLException { DatabendAggregateFunction aggregateFunction = Randomly.fromOptions(DatabendAggregateFunction.MAX, DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT, DatabendAggregateFunction.AVG/* , DatabendAggregateFunction.STDDEV_POP */); - NewFunctionNode aggregate = gen + NewFunctionNode aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); List> fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { - fetchColumns.add(gen.generateAggregate()); // TODO 更换成非聚合函数 + fetchColumns.add((DatabendAggregateOperation) gen.generateAggregate()); // TODO 更换成非聚合函数 } select.setFetchColumns(Arrays.asList(aggregate)); // if (Randomly.getBooleanWithRatherLowProbability()) { @@ -81,7 +83,8 @@ private String createMetamorphicUnionQuery(DatabendSelect select, NewFunctionNode aggregate, List> from) { String metamorphicQuery; - Node whereClause = gen.generateExpression(DatabendDataType.BOOLEAN); + Node whereClause = DatabendExprToNode + .cast(gen.generateExpression(DatabendDataType.BOOLEAN)); Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, DatabendUnaryPrefixOperator.NOT); Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, @@ -186,9 +189,10 @@ private DatabendSelect getSelect(List> aggregates, List select.setFromList(from); select.setWhereClause(whereClause); select.setJoinList(joinList); - if (Randomly.getBooleanWithSmallProbability()) { - select.setGroupByExpressions(List.of(gen.generateConstant(DatabendDataType.INT))); // TODO 仍可加强 - } + if (Randomly.getBooleanWithSmallProbability()) { + select.setGroupByExpressions(List.of(DatabendExprToNode.cast(gen.generateConstant(DatabendDataType.INT)))); // TODO + // 仍可加强 + } return select; } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java similarity index 73% rename from src/sqlancer/databend/test/DatabendQueryPartitioningBase.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 6ef132572..c836fad02 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -1,7 +1,8 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -19,19 +20,22 @@ import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendQueryPartitioningBase extends - TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { +public class DatabendQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase implements TestOracle { DatabendSchema s; DatabendTables targetTables; DatabendNewExpressionGenerator gen; DatabendSelect select; + List> groupByExpression; + public DatabendQueryPartitioningBase(DatabendGlobalState state) { super(state); DatabendErrors.addExpressionErrors(errors); @@ -49,11 +53,20 @@ public static String canonicalizeResultValue(String value) { @Override public void check() throws SQLException { s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); + targetTables = s.getRandomTableNonEmptyAndViewTables(); + List randomColumn = targetTables.getColumns(); + gen = new DatabendNewExpressionGenerator(state).setColumns(targetTables.getColumns()); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); initializeTernaryPredicateVariants(); select = new DatabendSelect(); - select.setFetchColumns(generateRandomColumns()); + columnOfLeafNode + .addAll(randomColumn.stream().map(c -> new DatabendColumnValue(c, null)).collect(Collectors.toList())); + groupByExpression = new ArrayList<>(columnOfLeafNode); + + select.setFetchColumns(randomColumn.stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList())); List tables = targetTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -65,7 +78,7 @@ public void check() throws SQLException { List> generateFetchColumns() { List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { // TODO 为什么会返回 false 或 true 字段 + if (Randomly.getBoolean()) { columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); } else { columns = generateRandomColumns(); @@ -81,7 +94,7 @@ List> generateRandomColumns() { } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java similarity index 78% rename from src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java index 8eae1e502..cb46e9196 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -7,7 +7,9 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; public class DatabendQueryPartitioningDistinctTester extends DatabendQueryPartitioningBase { @@ -21,18 +23,18 @@ public DatabendQueryPartitioningDistinctTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); select.setDistinct(true); - select.setWhereClause(null); + select.setWhereClause(DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); String originalQueryString = DatabendToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); if (Randomly.getBoolean()) { select.setDistinct(false); } - select.setWhereClause(predicate); + select.setWhereClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); + select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); + select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java similarity index 85% rename from src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java index dd25b6d7f..cbac6820b 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -10,6 +10,7 @@ import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendToStringVisitor; @@ -25,17 +26,17 @@ public DatabendQueryPartitioningGroupByTester(DatabendGlobalState state) { @Override public void check() throws SQLException { super.check(); - select.setGroupByExpressions(select.getFetchColumns()); + select.setGroupByExpressions(groupByExpression); select.setWhereClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setWhereClause(predicate); + select.setWhereClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); + select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); + select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java similarity index 80% rename from src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java index 8ca1489f2..419c2d558 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -10,6 +10,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; @@ -27,7 +28,8 @@ public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); + select.setWhereClause( + DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); } // boolean orderBy = Randomly.getBoolean(); boolean orderBy = false; // 关闭order by @@ -38,18 +40,16 @@ public void check() throws SQLException { Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByExpressions(constants); } - // select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - select.setGroupByExpressions(select.getFetchColumns()); + select.setGroupByExpressions(groupByExpression); select.setHavingClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); - // System.out.println(originalQueryString); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setHavingClause(predicate); + select.setHavingClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setHavingClause(negatedPredicate); + select.setHavingClause(DatabendExprToNode.cast(negatedPredicate)); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setHavingClause(isNullPredicate); + select.setHavingClause(DatabendExprToNode.cast(isNullPredicate)); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, @@ -59,13 +59,13 @@ public void check() throws SQLException { } @Override - protected Node generatePredicate() { + protected DatabendExpression generatePredicate() { return gen.generateHavingClause(); } @Override List> generateFetchColumns() { - return Collections.singletonList(gen.generateHavingClause()); + return Collections.singletonList(DatabendExprToNode.cast(gen.generateHavingClause())); } } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java similarity index 85% rename from src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index 2b43d0ba0..cff5ff1dd 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -6,6 +6,7 @@ import sqlancer.ComparatorHelper; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendToStringVisitor; @@ -29,11 +30,11 @@ public void check() throws SQLException { // if (orderBy) { //TODO 待开启 // select.setOrderByExpressions(gen.generateOrderBys()); // } - select.setWhereClause(predicate); + select.setWhereClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); + select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); + select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index 0c1755a54..ae1ff1d88 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -33,6 +33,18 @@ public void testDatabendTLPQueryPartitioning() { "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); } + @Test + public void testDatabendPQS() { + String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + assumeTrue(databendIsAvailable); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); + } + // @Test // public void testDatabendTLPWhere() { // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); @@ -41,7 +53,7 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "WHERE")); // } // @@ -53,7 +65,7 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "GROUP_BY")); // } // @@ -65,7 +77,7 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "HAVING")); // } // @@ -77,7 +89,7 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "DISTINCT")); // } // @@ -89,18 +101,8 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "AGGREGATE")); // } - // @Test - // void testConnection() { - // assertEquals(0, - // Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - // "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "--username", "user1", "--password", "1234", - // "databend", "--oracle", "HAVING" })); - // } - } From 01f4c67c9f2f001594b520997fab2fec51e9131f Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 12 Dec 2022 08:55:47 +0800 Subject: [PATCH 0682/1284] tidb: add BIT_AND and BIT_OR (#634) * tidb: add BIT_AND and BIT_OR Signed-off-by: Weizhen Wang * fmt * fmt * fmt Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/ast/TiDBAggregate.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sqlancer/tidb/ast/TiDBAggregate.java b/src/sqlancer/tidb/ast/TiDBAggregate.java index 21bd8eb0c..6650dc395 100644 --- a/src/sqlancer/tidb/ast/TiDBAggregate.java +++ b/src/sqlancer/tidb/ast/TiDBAggregate.java @@ -9,11 +9,7 @@ public class TiDBAggregate extends FunctionNode implements TiDBExpression { public enum TiDBAggregateFunction { - COUNT(1), // - SUM(1), // - AVG(1), // - MIN(1), // - MAX(1); + AVG(1), BIT_AND(1), BIT_OR(1), COUNT(1), SUM(1), MIN(1), MAX(1); private int nrArgs; From e2e037177ad50e50187adea2e29982428477021d Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Tue, 13 Dec 2022 15:11:16 +0800 Subject: [PATCH 0683/1284] fix(databend): distinct tester Signed-off-by: Yisong Han --- .../test/tlp/DatabendQueryPartitioningDistinctTester.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java index cb46e9196..5756a00be 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java @@ -9,7 +9,6 @@ import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; public class DatabendQueryPartitioningDistinctTester extends DatabendQueryPartitioningBase { @@ -23,7 +22,9 @@ public DatabendQueryPartitioningDistinctTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); select.setDistinct(true); - select.setWhereClause(DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + // TODO 后期可以使用and来进行扩展 + // select.setWhereClause(DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + select.setWhereClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); From a4dc79714435b8b9e226b0b299669d9a5874d6d0 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 15 Dec 2022 22:42:29 +0800 Subject: [PATCH 0684/1284] tidb: support HASH_JOIN_BUILD and HASH_JOIN_PROBE (#639) * tidb: support HASH_JOIN_BUILD and HASH_JOIN_PROBE Signed-off-by: Weizhen Wang * tidb: support HASH_JOIN_BUILD and HASH_JOIN_PROBE Signed-off-by: Weizhen Wang * tidb: support HASH_JOIN_BUILD and HASH_JOIN_PROBE Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBHintGenerator.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index f72eafe16..64f8540bb 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -31,7 +31,9 @@ enum IndexHint { // READ_FROM_STORAGE USE_INDEX_MERGE, // NO_INDEX_MERGE, // - USE_TOJA; + USE_TOJA, // + HASH_JOIN_BUILD, // + HASH_JOIN_PROBE; } public TiDBHintGenerator(TiDBSelect select, List tables) { @@ -100,6 +102,16 @@ private void generate() { sb.append(Randomly.getBoolean()); sb.append(")"); break; + case HASH_JOIN_BUILD: + sb.append("HASH_JOIN_BUILD("); + sb.append(table.getName()); + sb.append(")"); + break; + case HASH_JOIN_PROBE: + sb.append("HASH_JOIN_PROBE("); + sb.append(table.getName()); + sb.append(")"); + break; default: throw new AssertionError(); } From dfcb0feeafbee11c3e133d8b4cf61a4d955bf389 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 16 Dec 2022 11:14:43 +0800 Subject: [PATCH 0685/1284] feat(databend):add DatabendBugs Signed-off-by: Yisong Han --- src/sqlancer/databend/DatabendBugs.java | 15 +++++++++++++ src/sqlancer/databend/DatabendErrors.java | 21 ++++++++++++++++--- .../tlp/DatabendQueryPartitioningBase.java | 7 +++++-- 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 src/sqlancer/databend/DatabendBugs.java diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java new file mode 100644 index 000000000..9a8f788c7 --- /dev/null +++ b/src/sqlancer/databend/DatabendBugs.java @@ -0,0 +1,15 @@ +package sqlancer.databend; + +public final class DatabendBugs { + + public static boolean bug9162; // https://github.com/datafuselabs/databend/issues/9162 + public static boolean bug9163; // https://github.com/datafuselabs/databend/issues/9163 + public static boolean bug9018; // https://github.com/datafuselabs/databend/issues/9018 + public static boolean bug9224 = true; // https://github.com/datafuselabs/databend/issues/9224 + public static boolean bug9234 = true; // https://github.com/datafuselabs/databend/issues/9234 + public static boolean bug9236 = true; // https://github.com/datafuselabs/databend/issues/9236 + public static boolean bug9264 = true; // https://github.com/datafuselabs/databend/issues/9264 + + private DatabendBugs() { + } +} diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 46202830b..6f061d901 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -12,10 +12,25 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("/ by zero"); errors.add("ORDER BY position"); errors.add("GROUP BY position"); - errors.add("downcast column error"); // bug - errors.add("index out of bounds"); // bug + if (DatabendBugs.bug9162) { + errors.add("downcast column error"); + } + if (DatabendBugs.bug9018) { + errors.add("index out of bounds"); + } + if (DatabendBugs.bug9163) { + errors.add("validity must be equal to the array's length"); + } + if (DatabendBugs.bug9224) { + errors.add("Can't cast column from nullable data into non-nullable type"); + } + if (DatabendBugs.bug9234) { + errors.add("called `Option::unwrap()` on a `None` value"); + } + if (DatabendBugs.bug9264) { + errors.add("assertion failed: offset + length <= self.length"); + } errors.add("validity's length must be equal"); // bug - errors.add("validity must be equal to the array's length"); // bug /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 74257310e..28d516f65 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -14,6 +14,7 @@ import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; @@ -71,8 +72,10 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DatabendJoin.getJoins(tableList, state); - select.setJoinList(joins.stream().collect(Collectors.toList())); + if (!DatabendBugs.bug9236) { + List> joins = DatabendJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + } select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } From cec99caa91dbcb5dcb0176fbabbd26d0e5e8a8ac Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Fri, 16 Dec 2022 16:16:31 +0800 Subject: [PATCH 0686/1284] Escaping newlines --- src/sqlancer/common/log/SQLLoggableFactory.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java index 24a28a603..790d10709 100644 --- a/src/sqlancer/common/log/SQLLoggableFactory.java +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -14,6 +14,8 @@ protected Loggable createLoggable(String input, String suffix) { if (!input.endsWith(";")) { completeString += ";"; } + completeString = completeString.replace("\n", "\\n"); + completeString = completeString.replace("\r", "\\r"); if (suffix != null && suffix.length() != 0) { completeString += suffix; } From f3f44c313fb7eb3936edc612f5241c806a03658b Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Fri, 16 Dec 2022 16:50:30 +0800 Subject: [PATCH 0687/1284] fix bug in CREATE TABLE --- src/sqlancer/common/query/SQLQueryAdapter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 01fcc8539..ab9124230 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -46,7 +46,7 @@ private String canonicalizeString(String s) { } private void checkQueryString() { - if (query.contains("CREATE TABLE") && !couldAffectSchema) { + if (query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN") && !couldAffectSchema) { throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); } } From f65cadd2ee3d9c61cd4ed8fb1a32fcb5287cfb91 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Fri, 16 Dec 2022 21:47:03 +0800 Subject: [PATCH 0688/1284] random virtual/rtree table names generation (#647) --- .../sqlite3/schema/SQLite3Schema.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index b4bf11002..586d7e776 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -462,4 +462,32 @@ public List getDatabaseTablesWithoutViewsWithoutVirtualTables() { return getDatabaseTables().stream().filter(t -> !t.isView() && !t.isVirtual).collect(Collectors.toList()); } + public String getFreeVirtualTableName() { + int i = 0; + if (Randomly.getBooleanWithRatherLowProbability()) { + i = (int) Randomly.getNotCachedInteger(0, 100); + } + do { + String tableName = String.format("vt%d", i++); + if (getDatabaseTables().stream().noneMatch(t -> t.getName().equalsIgnoreCase(tableName))) { + return tableName; + } + } while (true); + + } + + public String getFreeRtreeTableName() { + int i = 0; + if (Randomly.getBooleanWithRatherLowProbability()) { + i = (int) Randomly.getNotCachedInteger(0, 100); + } + do { + String tableName = String.format("rt%d", i++); + if (getDatabaseTables().stream().noneMatch(t -> t.getName().equalsIgnoreCase(tableName))) { + return tableName; + } + } while (true); + + } + } From a3357c13efba0724435724dec0147ec71c27c5e7 Mon Sep 17 00:00:00 2001 From: Yisong Han <1149869798@qq.com> Date: Fri, 16 Dec 2022 22:00:34 +0800 Subject: [PATCH 0689/1284] add(databend): databend bugs (#646) Signed-off-by: Yisong Han Signed-off-by: Yisong Han --- src/sqlancer/databend/DatabendBugs.java | 7 ++++++- src/sqlancer/databend/DatabendErrors.java | 2 +- src/sqlancer/databend/DatabendSchema.java | 3 ++- test/sqlancer/dbms/TestDatabend.java | 8 ++++---- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index 9a8f788c7..336443511 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -2,11 +2,16 @@ public final class DatabendBugs { + public static boolean bug9018; // https://github.com/datafuselabs/databend/issues/9018 public static boolean bug9162; // https://github.com/datafuselabs/databend/issues/9162 public static boolean bug9163; // https://github.com/datafuselabs/databend/issues/9163 - public static boolean bug9018; // https://github.com/datafuselabs/databend/issues/9018 + public static boolean bug9164 = true; // https://github.com/datafuselabs/databend/issues/9164 + public static boolean bug9196 = true; // https://github.com/datafuselabs/databend/issues/9196 + public static boolean bug9232 = true; // https://github.com/datafuselabs/databend/issues/9232 public static boolean bug9224 = true; // https://github.com/datafuselabs/databend/issues/9224 + public static boolean bug9226 = true; // https://github.com/datafuselabs/databend/issues/9226 public static boolean bug9234 = true; // https://github.com/datafuselabs/databend/issues/9234 + public static boolean bug9235 = true; // https://github.com/datafuselabs/databend/issues/9235 public static boolean bug9236 = true; // https://github.com/datafuselabs/databend/issues/9236 public static boolean bug9264 = true; // https://github.com/datafuselabs/databend/issues/9264 diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 6f061d901..8f9d2b942 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -30,7 +30,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (DatabendBugs.bug9264) { errors.add("assertion failed: offset + length <= self.length"); } - errors.add("validity's length must be equal"); // bug + // errors.add("validity's length must be equal"); // bug /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index c82ad82ca..24d47ec46 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -173,7 +173,8 @@ public DatabendRowValue getRandomRowValue(SQLConnection con) throws SQLException try (Statement s = con.createStatement()) { ResultSet rs = s.executeQuery(rowValueQuery); if (!rs.next()) { - throw new AssertionError("could not find random row " + rowValueQuery + "\n"); + throw new IgnoreMeException(); + // throw new AssertionError("could not find random row " + rowValueQuery + "\n"); } for (int i = 0; i < getColumns().size(); i++) { DatabendColumn column = getColumns().get(i); diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index ae1ff1d88..707d7b8a9 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -22,7 +22,7 @@ public void testDatabendNoREC() { } @Test - public void testDatabendTLPQueryPartitioning() { + public void testDatabendPQS() { String databendAvailable = System.getenv("DATABEND_AVAILABLE"); boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); assumeTrue(databendIsAvailable); @@ -30,11 +30,11 @@ public void testDatabendTLPQueryPartitioning() { Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); } @Test - public void testDatabendPQS() { + public void testDatabendTLPQueryPartitioning() { String databendAvailable = System.getenv("DATABEND_AVAILABLE"); boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); assumeTrue(databendIsAvailable); @@ -42,7 +42,7 @@ public void testDatabendPQS() { Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); } // @Test From 706cff79edab7df2282b975caff7495ef1f6f2f3 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 17 Dec 2022 04:41:46 +0800 Subject: [PATCH 0690/1284] tidb: add more hint (#649) * tidb: add SHUFFLE_JOIN, BROADCAST_JOIN, LIMIT_TO_COP, MPP_1PHASE_AGG, MPP_2PHASE_AGG Signed-off-by: Weizhen Wang * tidb: add SHUFFLE_JOIN, BROADCAST_JOIN, LIMIT_TO_COP, MPP_1PHASE_AGG, MPP_2PHASE_AGG Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBHintGenerator.java | 55 ++++++++++++++------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index 64f8540bb..355cb2934 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -33,7 +33,12 @@ enum IndexHint { NO_INDEX_MERGE, // USE_TOJA, // HASH_JOIN_BUILD, // - HASH_JOIN_PROBE; + HASH_JOIN_PROBE, // + MPP_1PHASE_AGG, // + MPP_2PHASE_AGG, // + LIMIT_TO_COP, // + SHUFFLE_JOIN, // + BROADCAST_JOIN } public TiDBHintGenerator(TiDBSelect select, List tables) { @@ -73,6 +78,15 @@ private void generate() { case STREAM_AGG: sb.append("STREAM_AGG()"); break; + case MPP_1PHASE_AGG: + sb.append("MPP_1PHASE_AGG()"); + break; + case MPP_2PHASE_AGG: + sb.append("MPP_2PHASE_AGG()"); + break; + case LIMIT_TO_COP: + sb.append("LIMIT_TO_COP()"); + break; case USE_INDEX: indexesHint("USE_INDEX"); break; @@ -82,17 +96,11 @@ private void generate() { case AGG_TO_COP: sb.append("AGG_TO_COP()"); break; + case SHUFFLE_JOIN: + twoTablesHint("SHUFFLE_JOIN", table); + break; case USE_INDEX_MERGE: - if (table.hasIndexes()) { - sb.append("USE_INDEX_MERGE("); - sb.append(table.getName()); - sb.append(", "); - List indexes = Randomly.nonEmptySubset(table.getIndexes()); - sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining(", "))); - sb.append(")"); - } else { - throw new IgnoreMeException(); - } + twoTablesHint("USE_INDEX_MERGE", table); break; case NO_INDEX_MERGE: sb.append("NO_INDEX_MERGE()"); @@ -103,14 +111,13 @@ private void generate() { sb.append(")"); break; case HASH_JOIN_BUILD: - sb.append("HASH_JOIN_BUILD("); - sb.append(table.getName()); - sb.append(")"); + tablesHint("HASH_JOIN_BUILD"); break; case HASH_JOIN_PROBE: - sb.append("HASH_JOIN_PROBE("); - sb.append(table.getName()); - sb.append(")"); + tablesHint("HASH_JOIN_PROBE"); + break; + case BROADCAST_JOIN: + twoTablesHint("BROADCAST_JOIN", table); break; default: throw new AssertionError(); @@ -141,6 +148,20 @@ private void tablesHint(String string) { sb.append(")"); } + private void twoTablesHint(String string, TiDBTable table) { + if (table.hasIndexes()) { + sb.append(string); + sb.append("("); + sb.append(table.getName()); + sb.append(", "); + List indexes = Randomly.nonEmptySubset(table.getIndexes()); + sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining(", "))); + sb.append(")"); + } else { + throw new IgnoreMeException(); + } + } + private void appendTables() { List tableSubset = Randomly.nonEmptySubset(tables); sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); From 45cd972c90e0b4dafe5c126a795907fbcccaa265 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Dec 2022 21:00:47 +0000 Subject: [PATCH 0691/1284] build(deps): bump spotbugs-maven-plugin from 4.7.2.1 to 4.7.3.0 (#608) Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.2.1 to 4.7.3.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.2.1...spotbugs-maven-plugin-4.7.3.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 950ea4243..f0afd61c0 100644 --- a/pom.xml +++ b/pom.xml @@ -252,7 +252,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.2.1 + 4.7.3.0 spotbugs From 4c88763d64a98af4fc7bdf1d9b23afcb672b9166 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Dec 2022 21:26:30 +0000 Subject: [PATCH 0692/1284] build(deps): bump mariadb-java-client from 3.0.8 to 3.1.0 (#614) Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.8 to 3.1.0. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/commits) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f0afd61c0..573e17c1e 100644 --- a/pom.xml +++ b/pom.xml @@ -309,7 +309,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.8 + 3.1.0 org.duckdb From fcb2b13b398ff62c5ef6b4e8d37ed9be51cf0fa2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Dec 2022 22:15:05 +0000 Subject: [PATCH 0693/1284] build(deps): bump postgresql from 42.5.0 to 42.5.1 (#620) Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.5.0 to 42.5.1. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.5.0...REL42.5.1) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 573e17c1e..1ddb71c02 100644 --- a/pom.xml +++ b/pom.xml @@ -284,7 +284,7 @@ org.postgresql postgresql - 42.5.0 + 42.5.1 com.ing.data From b5eb2153dfd06bf50aca0f98eb32383b389a933f Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 16:59:34 +0800 Subject: [PATCH 0694/1284] Add interface to return queries by oracles (#648) Co-authored-by: Manuel Rigger --- .../cockroachdb/oracle/CockroachDBNoRECOracle.java | 5 +++++ .../oracle/tlp/CockroachDBTLPAggregateOracle.java | 5 +++++ .../oracle/tlp/CockroachDBTLPDistinctOracle.java | 9 ++++++++- .../oracle/tlp/CockroachDBTLPExtendedWhereOracle.java | 7 +++++++ .../oracle/tlp/CockroachDBTLPGroupByOracle.java | 9 ++++++++- .../oracle/tlp/CockroachDBTLPHavingOracle.java | 8 ++++++++ .../oracle/tlp/CockroachDBTLPWhereOracle.java | 9 ++++++++- src/sqlancer/common/oracle/CompositeTestOracle.java | 7 +++++++ src/sqlancer/common/oracle/TestOracle.java | 4 ++++ src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java | 5 +++++ .../sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java | 8 +++++++- .../sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java | 9 ++++++++- .../sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java | 9 ++++++++- .../sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java | 8 +++++++- .../sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java | 9 ++++++++- src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java | 8 ++++++++ src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java | 9 ++++++++- 17 files changed, 119 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 77ecbcbed..6ab133b1c 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -150,4 +150,9 @@ private int getCount(SQLGlobalState globalState, SQLQueryAdapter q) throws return count; } + @Override + public String getLastQueryString() { + return optimizedQueryString; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index 0368f5222..3cf614cea 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -195,4 +195,9 @@ private CockroachDBSelect getSelect(List aggregates, List return leftSelect; } + @Override + public String getLastQueryString() { + return originalQuery; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java index bdf8f59c3..dd174e805 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java @@ -15,6 +15,8 @@ public class CockroachDBTLPDistinctOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPDistinctOracle(CockroachDBGlobalState state) { super(state); errors.add("GROUP BY term out of range"); @@ -25,7 +27,7 @@ public void check() throws SQLException { super.check(); select.setDistinct(true); String originalQueryString = CockroachDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setDistinct(false); CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); @@ -41,4 +43,9 @@ public void check() throws SQLException { ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java index 36ac46733..7463e767f 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java @@ -19,6 +19,7 @@ public class CockroachDBTLPExtendedWhereOracle extends CockroachDBTLPBase { private CockroachDBExpression originalPredicate; + private String generatedQueryString; public CockroachDBTLPExtendedWhereOracle(CockroachDBGlobalState state) { super(state); @@ -32,6 +33,7 @@ public void check() throws SQLException { originalPredicate = generatePredicate(); select.setWhereClause(originalPredicate); String originalQueryString = CockroachDBVisitor.asString(select); + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean allowOrderBy = Randomly.getBoolean(); @@ -56,4 +58,9 @@ public CockroachDBExpression combinePredicate(CockroachDBExpression expr) { return new CockroachDBBinaryLogicalOperation(originalPredicate, expr, CockroachDBBinaryLogicalOperator.AND); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java index 535fea3e1..466252e56 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java @@ -14,6 +14,8 @@ public class CockroachDBTLPGroupByOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPGroupByOracle(CockroachDBGlobalState state) { super(state); } @@ -24,7 +26,7 @@ public void check() throws SQLException { select.setGroupByExpressions(select.getFetchColumns()); select.setWhereClause(null); String originalQueryString = CockroachDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setWhereClause(predicate); @@ -46,4 +48,9 @@ List generateFetchColumns() { .collect(Collectors.toList())); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java index 42abad00b..d5a96d80b 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java @@ -13,6 +13,8 @@ public class CockroachDBTLPHavingOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPHavingOracle(CockroachDBGlobalState state) { super(state); errors.add("GROUP BY term out of range"); @@ -31,6 +33,7 @@ public void check() throws SQLException { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); String originalQueryString = CockroachDBVisitor.asString(select); + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); @@ -52,4 +55,9 @@ protected CockroachDBExpression generatePredicate() { return gen.generateHavingClause(); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java index 8fe0899d0..db0ea6162 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java @@ -16,6 +16,8 @@ public class CockroachDBTLPWhereOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPWhereOracle(CockroachDBGlobalState state) { super(state); errors.add("GROUP BY term out of range"); @@ -25,7 +27,7 @@ public CockroachDBTLPWhereOracle(CockroachDBGlobalState state) { public void check() throws SQLException { super.check(); String originalQueryString = CockroachDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean allowOrderBy = Randomly.getBoolean(); @@ -45,4 +47,9 @@ public void check() throws SQLException { ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/common/oracle/CompositeTestOracle.java b/src/sqlancer/common/oracle/CompositeTestOracle.java index 0fdc13140..e2f96f785 100644 --- a/src/sqlancer/common/oracle/CompositeTestOracle.java +++ b/src/sqlancer/common/oracle/CompositeTestOracle.java @@ -9,6 +9,7 @@ public class CompositeTestOracle> implements Test private final List> oracles; private final G globalState; private int i; + private int iLast; public CompositeTestOracle(List> oracles, G globalState) { this.globalState = globalState; @@ -19,6 +20,7 @@ public CompositeTestOracle(List> oracles, G globalState) { public void check() throws Exception { try { oracles.get(i).check(); + iLast = i; boolean lastOracleIndex = i == oracles.size() - 1; if (!lastOracleIndex) { globalState.getManager().incrementSelectQueryCount(); @@ -27,4 +29,9 @@ public void check() throws Exception { i = (i + 1) % oracles.size(); } } + + @Override + public String getLastQueryString() { + return oracles.get(iLast).getLastQueryString(); + } } diff --git a/src/sqlancer/common/oracle/TestOracle.java b/src/sqlancer/common/oracle/TestOracle.java index 2dfd854a0..0ef993b89 100644 --- a/src/sqlancer/common/oracle/TestOracle.java +++ b/src/sqlancer/common/oracle/TestOracle.java @@ -10,4 +10,8 @@ public interface TestOracle> { default Reproducer getLastReproducer() { return null; } + + default String getLastQueryString() { + throw new AssertionError("Not supported!"); + } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 826a2727b..3a5a5c9d0 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -102,6 +102,11 @@ public Reproducer getLastReproducer() { return reproducer; } + @Override + public String getLastQueryString() { + return optimizedQueryString; + } + private Function getUnoptimizedQuery(SQLite3Select select, SQLite3Expression randomWhereCondition) throws SQLException { SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 715770257..90ca0d96b 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -33,6 +33,7 @@ public class SQLite3TLPAggregateOracle implements TestOracle private final SQLite3GlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); private SQLite3ExpressionGenerator gen; + private String generatedQueryString; public SQLite3TLPAggregateOracle(SQLite3GlobalState state) { this.state = state; @@ -56,7 +57,7 @@ public void check() throws SQLException { select.setOrderByExpressions(gen.generateOrderBys()); } String originalQuery = SQLite3Visitor.asString(select); - + generatedQueryString = originalQuery; SQLite3Expression whereClause = gen.generateExpression(); SQLite3UnaryOperation negatedClause = new SQLite3UnaryOperation(UnaryOperator.NOT, whereClause); SQLite3PostfixUnaryOperation notNullClause = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, @@ -122,4 +123,9 @@ private SQLite3Select getSelect(SQLite3Aggregate aggregate, List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setWhereClause(predicate); @@ -37,4 +39,9 @@ public void check() throws SQLException { state); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java index 91977d973..3b39ef4c8 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java @@ -14,6 +14,8 @@ public class SQLite3TLPGroupByOracle extends SQLite3TLPBase { + private String generatedQueryString; + public SQLite3TLPGroupByOracle(SQLite3GlobalState state) { super(state); } @@ -24,7 +26,7 @@ public void check() throws SQLException { select.setGroupByClause(select.getFetchColumns()); select.setWhereClause(null); String originalQueryString = SQLite3Visitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setWhereClause(predicate); @@ -46,4 +48,9 @@ List generateFetchColumns() { .collect(Collectors.toList()); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index 6a960021d..718c0d4ce 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -33,6 +33,7 @@ public class SQLite3TLPHavingOracle implements TestOracle { private final SQLite3GlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); + private String generatedQueryString; public SQLite3TLPHavingOracle(SQLite3GlobalState state) { this.state = state; @@ -61,7 +62,7 @@ public void check() throws SQLException { select.setGroupByClause(groupByColumns); select.setHavingClause(null); String originalQueryString = SQLite3Visitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); SQLite3Expression predicate = gen.getHavingClause(); @@ -84,4 +85,9 @@ public void check() throws SQLException { throw new AssertionError(originalQueryString + ";\n" + combinedString + ";"); } } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java index 95026de86..2f1540f2b 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java @@ -11,6 +11,8 @@ public class SQLite3TLPWhereOracle extends SQLite3TLPBase { + private String generatedQueryString; + public SQLite3TLPWhereOracle(SQLite3GlobalState state) { super(state); } @@ -20,7 +22,7 @@ public void check() throws SQLException { super.check(); select.setWhereClause(null); String originalQueryString = SQLite3Visitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean orderBy = Randomly.getBooleanWithSmallProbability(); @@ -40,4 +42,9 @@ public void check() throws SQLException { state); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java index 68b4a56c1..23b3cb035 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java @@ -14,6 +14,8 @@ public class TiDBTLPHavingOracle extends TiDBTLPBase implements TestOracle { + private String generatedQueryString; + public TiDBTLPHavingOracle(TiDBGlobalState state) { super(state); TiDBErrors.addExpressionHavingErrors(errors); @@ -32,6 +34,7 @@ public void check() throws SQLException { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); String originalQueryString = TiDBVisitor.asString(select); + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setHavingClause(predicate); @@ -51,4 +54,9 @@ public void check() throws SQLException { protected TiDBExpression generatePredicate() { return gen.generateHavingClause(); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java index 4c8496e94..e03b347f3 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java @@ -12,6 +12,8 @@ public class TiDBTLPWhereOracle extends TiDBTLPBase { + private String generatedQueryString; + public TiDBTLPWhereOracle(TiDBGlobalState state) { super(state); TiDBErrors.addExpressionErrors(errors); @@ -22,7 +24,7 @@ public void check() throws SQLException { super.check(); select.setWhereClause(null); String originalQueryString = TiDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); @@ -42,4 +44,9 @@ public void check() throws SQLException { state); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } From ca70c6a9f9346b7eab8ecc44573e890068afd47b Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 18:00:14 +0800 Subject: [PATCH 0695/1284] fix: broken grammar of LIKELIHOOD (#650) --- src/sqlancer/sqlite3/SQLite3Errors.java | 1 - .../sqlite3/gen/SQLite3ExpressionGenerator.java | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index b872fcd1d..6cd4c40c9 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -21,7 +21,6 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("FTS expression tree is too large"); errors.add("String or BLOB exceeds size limit"); errors.add("[SQLITE_ERROR] SQL error or missing database (integer overflow)"); - errors.add("second argument to likelihood() must be a constant between 0.0 and 1.0"); errors.add("ORDER BY term out of range"); errors.add("GROUP BY term out of range"); errors.add("not authorized"); // load_extension diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index d21f00af3..5e7e32cb6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -547,6 +547,12 @@ private SQLite3Expression getFunction(SQLite3GlobalState globalState, int depth) nrArgs += Randomly.smallNumber(); } List expressions = randomFunction.generateArguments(nrArgs, depth + 1, this); + // The second argument of LIKELIHOOD must be a float number within 0.0 -1.0 + if (randomFunction == AnyFunction.LIKELIHOOD) { + SQLite3Expression lastArg = SQLite3Constant.createRealConstant(Randomly.getPercentage()); + expressions.remove(expressions.size() - 1); + expressions.add(lastArg); + } return new SQLite3Expression.Function(randomFunction.toString(), expressions.toArray(new SQLite3Expression[0])); } @@ -604,6 +610,11 @@ private SQLite3Expression getComputableFunction(int depth) { args[i] = new SQLite3Distinct(args[i]); } } + // The second argument of LIKELIHOOD must be a float number within 0.0 -1.0 + if (func == ComputableFunction.LIKELIHOOD) { + SQLite3Expression lastArg = SQLite3Constant.createRealConstant(Randomly.getPercentage()); + args[args.length - 1] = lastArg; + } return new SQLite3Function(func, args); } From 977edb45d11410286753dd3102fff34263239203 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 17 Dec 2022 19:09:47 +0800 Subject: [PATCH 0696/1284] tidb: turn off bug switch (#638) * tidb: turn off bug switch Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBBugs.java | 43 ------------------- .../tidb/TiDBExpressionGenerator.java | 9 ---- .../tidb/gen/TiDBAlterTableGenerator.java | 7 --- src/sqlancer/tidb/gen/TiDBHintGenerator.java | 4 -- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 12 ++---- .../tidb/visitor/TiDBToStringVisitor.java | 5 --- 6 files changed, 3 insertions(+), 77 deletions(-) diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index a4f5cc2dd..d56629e44 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -2,49 +2,6 @@ // do not make the fields final to avoid warnings public final class TiDBBugs { - - // https://github.com/pingcap/tidb/issues/15987 - public static boolean bug15987; - - // // https://github.com/pingcap/tidb/issues/15988 - public static boolean bug15988; - - // https://github.com/pingcap/tidb/issues/16028 - public static boolean bug16028; - - // https://github.com/pingcap/tidb/issues/16020 - public static boolean bug16020; - - // https://github.com/pingcap/tidb/issues/15990 - public static boolean bug15990; - - // https://github.com/pingcap/tidb/issues/15844 - public static boolean bug15844; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/10 - public static boolean bug10; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/14 - public static boolean bug14; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/15 - public static boolean bug15; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/16 - public static boolean bug16; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/19 - public static boolean bug19; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/48 - public static boolean bug48; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/50 - public static boolean bug50; - - // https://github.com/citusdata/citus/issues/4079 - public static boolean bug4079; - // https://github.com/pingcap/tidb/issues/35677 public static boolean bug35677 = true; diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index 535647c53..a8fb75da0 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -63,9 +63,6 @@ protected TiDBExpression generateExpression(int depth) { } switch (Randomly.fromOptions(Gen.values())) { case DEFAULT: - if (TiDBBugs.bug15) { - throw new IgnoreMeException(); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } @@ -92,9 +89,6 @@ protected TiDBExpression generateExpression(int depth) { return new TiDBBinaryBitOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryBitOperator.getRandom()); case BINARY_LOGICAL: - if (TiDBBugs.bug48) { - throw new IgnoreMeException(); - } return new TiDBBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryLogicalOperator.getRandom()); case CAST: @@ -102,9 +96,6 @@ protected TiDBExpression generateExpression(int depth) { "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 "DECIMAL", "SIGNED", "UNSIGNED" /* https://github.com/pingcap/tidb/issues/16028 */)); case CASE: - if (TiDBBugs.bug19) { - throw new IgnoreMeException(); - } int nr = Randomly.fromOptions(1, 2); return new TiDBCase(generateExpression(depth + 1), generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), generateExpression(depth + 1)); diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index a327dde39..b6f65a780 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; @@ -39,9 +38,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append(" "); switch (a) { case MODIFY_COLUMN: - if (TiDBBugs.bug10) { - throw new IgnoreMeException(); - } sb.append("MODIFY "); sb.append(column.getName()); sb.append(" "); @@ -92,9 +88,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("'Defining a virtual generated column as primary key' is not supported for generated columns"); break; case CHANGE: - if (TiDBBugs.bug10) { - throw new IgnoreMeException(); - } sb.append(" CHANGE "); sb.append(column.getName()); sb.append(" "); diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index 355cb2934..06105ed4c 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -6,7 +6,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.schema.TableIndex; -import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.ast.TiDBSelect; import sqlancer.tidb.ast.TiDBText; @@ -61,9 +60,6 @@ private void generate() { tablesHint("INL_JOIN"); break; case INL_HASH_JOIN: - if (TiDBBugs.bug50) { - throw new IgnoreMeException(); - } tablesHint("INL_HASH_JOIN"); break; case INL_MERGE_JOIN: diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 25d2cbb99..4ca210b90 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -8,7 +8,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; @@ -113,8 +112,7 @@ && canUseAsUnique(type) && !isGeneratedColumn) { errors.add(" used in key specification without a key length"); } sb.append(")"); - if (Randomly.getBooleanWithRatherLowProbability() - && !TiDBBugs.bug14 /* there are also a number of unresolved other partitioning bugs */) { + if (Randomly.getBooleanWithRatherLowProbability()) { sb.append("PARTITION BY HASH("); sb.append(TiDBVisitor.asString(gen.generateExpression())); sb.append(") "); @@ -127,9 +125,6 @@ && canUseAsUnique(type) && !isGeneratedColumn) { errors.add("A UNIQUE INDEX must include all columns in the table's partitioning function"); errors.add("is of a not allowed type for this type of partitioning"); errors.add("The PARTITION function returns the wrong type"); - if (TiDBBugs.bug16) { - errors.add("UnknownType: *ast.WhenClause"); - } } } @@ -144,11 +139,10 @@ private void appendType(StringBuilder sb, TiDBCompositeDataType type) { } private void appendSizeSpecifiers(StringBuilder sb, TiDBDataType type) { - if (type.isNumeric() && Randomly.getBoolean() && !TiDBBugs.bug16028) { + if (type.isNumeric() && Randomly.getBoolean()) { sb.append(" UNSIGNED"); } - if (type.isNumeric() && Randomly.getBoolean() - && !TiDBBugs.bug16028 /* seems to be the same bug as https://github.com/pingcap/tidb/issues/16028 */) { + if (type.isNumeric() && Randomly.getBoolean()) { sb.append(" ZEROFILL"); } } diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index 5e0c56841..8bd47ed70 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -1,9 +1,7 @@ package sqlancer.tidb.visitor; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; -import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.ast.TiDBAggregate; import sqlancer.tidb.ast.TiDBCase; import sqlancer.tidb.ast.TiDBCastOperation; @@ -130,9 +128,6 @@ public void visit(TiDBJoin join) { sb.append("LEFT "); break; case RIGHT: - if (TiDBBugs.bug15844) { - throw new IgnoreMeException(); - } sb.append("RIGHT "); break; default: From ec1cbe243eba8470706fe09b6c4d785c96be4f76 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 20:09:32 +0800 Subject: [PATCH 0697/1284] fix: broken grammar of aggreate in SQLite (#651) --- .../sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 90ca0d96b..849a8640f 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -66,7 +66,9 @@ public void check() throws SQLException { SQLite3Select leftSelect = getSelect(aggregate, from, whereClause); SQLite3Select middleSelect = getSelect(aggregate, from, negatedClause); SQLite3Select rightSelect = getSelect(aggregate, from, notNullClause); - String metamorphicText = "SELECT " + aggregate.getFunc().toString() + "(aggr) FROM ("; + String aggreateMethod = aggregate.getFunc() == SQLite3AggregateFunction.COUNT_ALL + ? SQLite3AggregateFunction.COUNT.toString() : aggregate.getFunc().toString(); + String metamorphicText = "SELECT " + aggreateMethod + "(aggr) FROM ("; metamorphicText += SQLite3Visitor.asString(leftSelect) + " UNION ALL " + SQLite3Visitor.asString(middleSelect) + " UNION ALL " + SQLite3Visitor.asString(rightSelect); metamorphicText += ")"; From 5533c4e3c32a7f474bf1e047bda5f529fb9ef7ba Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 20:38:37 +0800 Subject: [PATCH 0698/1284] Bump SQLite version to 3.40.0.0 (#652) * bump SQLite to 3.40.0.0 * Support RIGHT/FULL JOINs --- pom.xml | 2 +- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 6 ++++++ src/sqlancer/sqlite3/ast/SQLite3Expression.java | 2 +- .../sqlite3/gen/SQLite3ExpressionGenerator.java | 14 ++++++++++---- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 1ddb71c02..7b163a49f 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ org.xerial sqlite-jdbc - 3.36.0.3 + 3.40.0.0 mysql diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index f81259732..475c16335 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -237,6 +237,12 @@ public void visit(Join join) { case OUTER: sb.append("LEFT OUTER"); break; + case RIGHT: + sb.append("RIGHT OUTER"); + break; + case FULL: + sb.append("FULL OUTER"); + break; default: throw new AssertionError(join.getType()); } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 748c70375..8f99da8e5 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -131,7 +131,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { public static class Join extends SQLite3Expression { public enum JoinType { - INNER, CROSS, OUTER, NATURAL; + INNER, CROSS, OUTER, NATURAL, RIGHT, FULL; } private final SQLite3Table table; diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 5e7e32cb6..89e7e5860 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -139,19 +139,25 @@ public List getRandomJoinClauses(List tables) { if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } + List options = new ArrayList<>(Arrays.asList(JoinType.values())); if (Randomly.getBoolean() && tables.size() > 1) { int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1) { + options.remove(JoinType.NATURAL); + } for (int i = 0; i < nrJoinClauses; i++) { SQLite3Expression joinClause = generateExpression(); SQLite3Table table = Randomly.fromList(tables); tables.remove(table); - JoinType options; - options = Randomly.fromOptions(JoinType.INNER, JoinType.CROSS, JoinType.OUTER, JoinType.NATURAL); - if (options == JoinType.NATURAL) { + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { // NATURAL joins do not have an ON clause joinClause = null; } - Join j = new SQLite3Expression.Join(table, joinClause, options); + Join j = new SQLite3Expression.Join(table, joinClause, selectedOption); joinStatements.add(j); } From a535520766808fd5bb71db41a0351e9d1f0d5331 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 21:22:32 +0800 Subject: [PATCH 0699/1284] fix: temp table generation (#653) --- src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index ffa3af6cd..5d8a40547 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -31,7 +31,7 @@ public static SQLQueryAdapter dropView(SQLite3GlobalState globalState) { public static SQLQueryAdapter generate(SQLite3GlobalState globalState) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("CREATE"); - if (Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testTempTables && Randomly.getBoolean()) { sb.append(" "); sb.append(Randomly.fromOptions("TEMP", "TEMPORARY")); } From a195cdd0b4a97cf1113c4bded2e1a41281d4a26a Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 13:20:47 +0800 Subject: [PATCH 0700/1284] opt: support new features in SQLite (#654) * opt: support maximum number of table/index * opt: support explain interface --- src/sqlancer/sqlite3/SQLite3Options.java | 7 +++ src/sqlancer/sqlite3/SQLite3Provider.java | 60 +++++++++++-------- ...Lite3CreateVirtualRtreeTabelGenerator.java | 9 +++ .../sqlite3/gen/SQLite3ExplainGenerator.java | 7 +++ ...SQLite3CreateVirtualFTSTableGenerator.java | 10 ++++ .../gen/ddl/SQLite3IndexGenerator.java | 4 ++ .../gen/ddl/SQLite3TableGenerator.java | 9 +++ .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 5 ++ 8 files changed, 85 insertions(+), 26 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 6df9c378c..e740a5eea 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -90,6 +90,13 @@ public class SQLite3Options implements DBMSSpecificOptions "--execute-queries" }, description = "Specifies whether the query in the fuzzer should be executed", arity = 1) public boolean executeQuery = true; + @Parameter(names = { + "--max-num-tables" }, description = "The maximum number of tables/virtual tables/ rtree tables/ views that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + public enum SQLite3OracleFactory implements OracleFactory { PQS { @Override diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index eb88a0e0a..1cb5e0633 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -58,47 +58,50 @@ public SQLite3Provider() { } public enum Action implements AbstractAction { - PRAGMA(SQLite3PragmaGenerator::insertPragma), // - INDEX(SQLite3IndexGenerator::insertIndex), // - INSERT(SQLite3InsertGenerator::insertRow), // - VACUUM(SQLite3VacuumGenerator::executeVacuum), // - REINDEX(SQLite3ReindexGenerator::executeReindex), // - ANALYZE(SQLite3AnalyzeGenerator::generateAnalyze), // - DELETE(SQLite3DeleteGenerator::deleteContent), // + PRAGMA(SQLite3PragmaGenerator::insertPragma), // 0 + CREATE_INDEX(SQLite3IndexGenerator::insertIndex), // 1 + CREATE_VIEW(SQLite3ViewGenerator::generate), // 2 + CREATE_TRIGGER(SQLite3CreateTriggerGenerator::create), // 3 + CREATE_TABLE(SQLite3TableGenerator::createRandomTableStatement), // 4 + CREATE_VIRTUALTABLE(SQLite3CreateVirtualFTSTableGenerator::createRandomTableStatement), // 5 + CREATE_RTREETABLE(SQLite3CreateVirtualRtreeTabelGenerator::createRandomTableStatement), // 6 + INSERT(SQLite3InsertGenerator::insertRow), // 7 + DELETE(SQLite3DeleteGenerator::deleteContent), // 8 + ALTER(SQLite3AlterTable::alterTable), // 9 + UPDATE(SQLite3UpdateGenerator::updateRow), // 10 + DROP_INDEX(SQLite3DropIndexGenerator::dropIndex), // 11 + DROP_TABLE(SQLite3DropTableGenerator::dropTable), // 12 + DROP_VIEW(SQLite3ViewGenerator::dropView), // 13 + VACUUM(SQLite3VacuumGenerator::executeVacuum), // 14 + REINDEX(SQLite3ReindexGenerator::executeReindex), // 15 + ANALYZE(SQLite3AnalyzeGenerator::generateAnalyze), // 16 + EXPLAIN(SQLite3ExplainGenerator::explain), // 17 + CHECK_RTREE_TABLE((g) -> { + SQLite3Table table = g.getSchema().getRandomTableOrBailout(t -> t.getName().startsWith("r")); + String format = String.format("SELECT rtreecheck('%s');", table.getName()); + return new SQLQueryAdapter(format, ExpectedErrors.from("The database file is locked")); + }), // 18 + VIRTUAL_TABLE_ACTION(SQLite3VirtualFTSTableCommandGenerator::create), // 19 + MANIPULATE_STAT_TABLE(SQLite3StatTableGenerator::getQuery), // 20 TRANSACTION_START(SQLite3TransactionGenerator::generateBeginTransaction) { @Override public boolean canBeRetried() { return false; } - }, // - ALTER(SQLite3AlterTable::alterTable), // - DROP_INDEX(SQLite3DropIndexGenerator::dropIndex), // - UPDATE(SQLite3UpdateGenerator::updateRow), // + }, // 21 ROLLBACK_TRANSACTION(SQLite3TransactionGenerator::generateRollbackTransaction) { @Override public boolean canBeRetried() { return false; } - }, // + }, // 22 COMMIT(SQLite3TransactionGenerator::generateCommit) { @Override public boolean canBeRetried() { return false; } - }, // - DROP_TABLE(SQLite3DropTableGenerator::dropTable), // - DROP_VIEW(SQLite3ViewGenerator::dropView), // - EXPLAIN(SQLite3ExplainGenerator::explain), // - CHECK_RTREE_TABLE((g) -> { - SQLite3Table table = g.getSchema().getRandomTableOrBailout(t -> t.getName().startsWith("r")); - String format = String.format("SELECT rtreecheck('%s');", table.getName()); - return new SQLQueryAdapter(format, ExpectedErrors.from("The database file is locked")); - }), // - VIRTUAL_TABLE_ACTION(SQLite3VirtualFTSTableCommandGenerator::create), // - CREATE_VIEW(SQLite3ViewGenerator::generate), // - CREATE_TRIGGER(SQLite3CreateTriggerGenerator::create), // - MANIPULATE_STAT_TABLE(SQLite3StatTableGenerator::getQuery); + }; // 23 private final SQLQueryProvider sqlQueryProvider; @@ -146,7 +149,7 @@ private static int mapActions(SQLite3GlobalState globalState, Action a) { case MANIPULATE_STAT_TABLE: nrPerformed = r.getInteger(0, 5); break; - case INDEX: + case CREATE_INDEX: nrPerformed = r.getInteger(0, 5); break; case VIRTUAL_TABLE_ACTION: @@ -156,6 +159,11 @@ private static int mapActions(SQLite3GlobalState globalState, Action a) { case PRAGMA: nrPerformed = r.getInteger(0, 20); break; + case CREATE_TABLE: + case CREATE_VIRTUALTABLE: + case CREATE_RTREETABLE: + nrPerformed = 0; + break; case TRANSACTION_START: case REINDEX: case ANALYZE: diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index 08effcdc9..61f821382 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -15,6 +16,14 @@ public final class SQLite3CreateVirtualRtreeTabelGenerator { private SQLite3CreateVirtualRtreeTabelGenerator() { } + public static SQLQueryAdapter createRandomTableStatement(SQLite3GlobalState globalState) { + if (globalState.getSchema().getTables().getTables() + .size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return createTableStatement(globalState.getSchema().getFreeRtreeTableName(), globalState); + } + public static SQLQueryAdapter createTableStatement(String rTreeTableName, SQLite3GlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); List columns = new ArrayList<>(); diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java index e9ede64d9..8c00b3231 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java @@ -26,4 +26,11 @@ public static SQLQueryAdapter explain(SQLite3GlobalState globalState) throws Exc return new SQLQueryAdapter(sb.toString(), query.getExpectedErrors()); } + public static String explain(String selectStr) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN QUERY PLAN "); + sb.append(selectStr); + return sb.toString(); + } + } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java index a0f79fa7e..3ae9d16d5 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java @@ -4,10 +4,12 @@ import java.util.Arrays; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; public class SQLite3CreateVirtualFTSTableGenerator { @@ -20,6 +22,14 @@ public SQLite3CreateVirtualFTSTableGenerator(String tableName, Randomly r) { this.r = r; } + public static SQLQueryAdapter createRandomTableStatement(SQLite3GlobalState globalState) { + if (globalState.getSchema().getTables().getTables() + .size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return createTableStatement(globalState.getSchema().getFreeVirtualTableName(), globalState.getRandomly()); + } + public static SQLQueryAdapter createTableStatement(String tableName, Randomly r) { return new SQLite3CreateVirtualFTSTableGenerator(tableName, r).create(); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 0a33016fd..c81d895ac 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -23,6 +24,9 @@ public class SQLite3IndexGenerator { private final SQLite3GlobalState globalState; public static SQLQueryAdapter insertIndex(SQLite3GlobalState globalState) throws SQLException { + if (globalState.getSchema().getIndexNames().size() >= globalState.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } return new SQLite3IndexGenerator(globalState).create(); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index cc4731911..6c27a5d58 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -46,6 +47,14 @@ public SQLite3TableGenerator(String tableName, SQLite3GlobalState globalState) { this.existingSchema = globalState.getSchema(); } + public static SQLQueryAdapter createRandomTableStatement(SQLite3GlobalState globalState) { + if (globalState.getSchema().getTables().getTables() + .size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return createTableStatement(globalState.getSchema().getFreeTableName(), globalState); + } + public static SQLQueryAdapter createTableStatement(String tableName, SQLite3GlobalState globalState) { SQLite3TableGenerator sqLite3TableGenerator = new SQLite3TableGenerator(tableName, globalState); sqLite3TableGenerator.start(); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 5d8a40547..1c483e59d 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -2,6 +2,7 @@ import java.sql.SQLException; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -29,6 +30,10 @@ public static SQLQueryAdapter dropView(SQLite3GlobalState globalState) { } public static SQLQueryAdapter generate(SQLite3GlobalState globalState) throws SQLException { + if (globalState.getSchema().getTables().getTables() + .size() >= globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } StringBuilder sb = new StringBuilder(); sb.append("CREATE"); if (globalState.getDbmsSpecificOptions().testTempTables && Randomly.getBoolean()) { From 7a083b6a9d8774c75ffea8d8be31fb10e4b7d9eb Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 14:51:25 +0800 Subject: [PATCH 0701/1284] fix: null pointer reference (#656) --- .../cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index 3cf614cea..0dae9331d 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -85,8 +85,8 @@ public void check() throws SQLException { state.getState().getLocalState().log( "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); - if (firstResult == null && secondResult != null - || firstResult != null && (!firstResult.contentEquals(secondResult) + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && secondResult != null && (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { if (secondResult.contains("Inf")) { throw new IgnoreMeException(); // FIXME: average computation From eb2ec91e04be5af671486e25ea54abdce4f79ae2 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 15:49:57 +0800 Subject: [PATCH 0702/1284] opt: some new features in CockroachDB (#657) * opt: maximum number of tables/indexes in CockroachDB * opt: support drop table/view * opt: update Actions --- .../cockroachdb/CockroachDBOptions.java | 6 +++ .../cockroachdb/CockroachDBProvider.java | 21 ++++++---- .../gen/CockroachDBDropTableGenerator.java | 35 ++++++++++++++++ .../gen/CockroachDBDropViewGenerator.java | 41 +++++++++++++++++++ .../gen/CockroachDBIndexGenerator.java | 4 ++ .../gen/CockroachDBTableGenerator.java | 3 ++ 6 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java create mode 100644 src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 38b845781..4fdf52b0e 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -103,6 +103,12 @@ public TestOracle create(CockroachDBGlobalState globalSt "--increased-vectorization" }, description = "Generate VECTORIZE=on with a higher probability (which found a number of bugs in the past)") public boolean makeVectorizationMoreLikely = true; + @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + @Override public List getTestOracleFactory() { return Arrays.asList(oracle); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index abc7e6e20..64da45bb2 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -6,7 +6,6 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -23,6 +22,8 @@ import sqlancer.cockroachdb.gen.CockroachDBCommentOnGenerator; import sqlancer.cockroachdb.gen.CockroachDBCreateStatisticsGenerator; import sqlancer.cockroachdb.gen.CockroachDBDeleteGenerator; +import sqlancer.cockroachdb.gen.CockroachDBDropTableGenerator; +import sqlancer.cockroachdb.gen.CockroachDBDropViewGenerator; import sqlancer.cockroachdb.gen.CockroachDBIndexGenerator; import sqlancer.cockroachdb.gen.CockroachDBInsertGenerator; import sqlancer.cockroachdb.gen.CockroachDBRandomQuerySynthesizer; @@ -45,15 +46,17 @@ public CockroachDBProvider() { } public enum Action { - INSERT(CockroachDBInsertGenerator::insert), // - TRUNCATE(CockroachDBTruncateGenerator::truncate), // + CREATE_TABLE(CockroachDBTableGenerator::generate), CREATE_INDEX(CockroachDBIndexGenerator::create), // + CREATE_VIEW(CockroachDBViewGenerator::generate), // CREATE_STATISTICS(CockroachDBCreateStatisticsGenerator::create), // - SET_SESSION(CockroachDBSetSessionGenerator::create), // - CREATE_INDEX(CockroachDBIndexGenerator::create), // + INSERT(CockroachDBInsertGenerator::insert), // UPDATE(CockroachDBUpdateGenerator::gen), // - CREATE_VIEW(CockroachDBViewGenerator::generate), // + SET_SESSION(CockroachDBSetSessionGenerator::create), // SET_CLUSTER_SETTING(CockroachDBSetClusterSettingGenerator::create), // DELETE(CockroachDBDeleteGenerator::delete), // + TRUNCATE(CockroachDBTruncateGenerator::truncate), // + DROP_TABLE(CockroachDBDropTableGenerator::drop), // + DROP_VIEW(CockroachDBDropViewGenerator::drop), // COMMENT_ON(CockroachDBCommentOnGenerator::comment), // SHOW(CockroachDBShowGenerator::show), // TRANSACTION((g) -> { @@ -65,8 +68,7 @@ public enum Action { ExpectedErrors errors = new ExpectedErrors(); if (Randomly.getBoolean()) { sb.append("("); - sb.append(Randomly.nonEmptySubset("VERBOSE", "TYPES", "OPT", "DISTSQL", "VEC").stream() - .collect(Collectors.joining(", "))); + sb.append(Randomly.fromOptions("VERBOSE", "TYPES", "OPT", "DISTSQL", "VEC")); sb.append(") "); errors.add("cannot set EXPLAIN mode more than once"); errors.add("unable to vectorize execution plan"); @@ -199,6 +201,9 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio */ break; case TRANSACTION: + case CREATE_TABLE: + case DROP_TABLE: + case DROP_VIEW: nrPerformed = 0; // r.getInteger(0, 0); break; default: diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java new file mode 100644 index 000000000..3834c8757 --- /dev/null +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java @@ -0,0 +1,35 @@ +package sqlancer.cockroachdb.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public final class CockroachDBDropTableGenerator { + + private CockroachDBDropTableGenerator() { + } + + public static SQLQueryAdapter drop(CockroachDBGlobalState globalState) { + if (globalState.getSchema().getTables(t -> !t.isView()).size() <= 1) { + throw new IgnoreMeException(); + } + + ExpectedErrors errors = new ExpectedErrors(); + errors.add("is referenced by foreign key"); + + StringBuilder sb = new StringBuilder(); + sb.append("DROP"); + sb.append(" TABLE"); + sb.append(" "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), true); + } + +} diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java new file mode 100644 index 000000000..5d440b564 --- /dev/null +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java @@ -0,0 +1,41 @@ +package sqlancer.cockroachdb.gen; + +import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public final class CockroachDBDropViewGenerator { + + private CockroachDBDropViewGenerator() { + } + + public static SQLQueryAdapter drop(CockroachDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("is referenced by foreign key"); + + StringBuilder sb = new StringBuilder(); + sb.append("DROP"); + if (Randomly.getBoolean()) { + sb.append(" MATERIALIZED"); + } + sb.append(" VIEW"); + sb.append(" "); + if (Randomly.getBooleanWithRatherLowProbability()) { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(globalState.getSchema().getRandomTable(t -> t.isView()).getName()); + } + } else { + sb.append(globalState.getSchema().getRandomTable(t -> t.isView()).getName()); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), true); + } + +} diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index e5b708338..4db3588ac 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -2,6 +2,7 @@ import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -17,6 +18,9 @@ public CockroachDBIndexGenerator(CockroachDBGlobalState globalState) { } public static SQLQueryAdapter create(CockroachDBGlobalState s) { + if (s.getSchema().getIndexCount() >= s.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } return new CockroachDBIndexGenerator(s).getQuery(); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index 4e99ab31a..c32fa76eb 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -29,6 +29,9 @@ public CockroachDBTableGenerator(CockroachDBGlobalState globalState) { } public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } return new CockroachDBTableGenerator(globalState).getQuery(); } From 1ca72afc1bc76f0584c3db84a0b1ef7a55a41e7e Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 16:13:14 +0800 Subject: [PATCH 0703/1284] fix: connection string cockroachdb (#658) --- src/sqlancer/cockroachdb/CockroachDBProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 64da45bb2..580fa1ebf 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -278,7 +278,7 @@ public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws S s.execute(createDatabaseCommand); } con.close(); - con = DriverManager.getConnection("jdbc:postgresql://localhost:26257/" + databaseName, + con = DriverManager.getConnection(String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); return new SQLConnection(con); } From 00cda2ec1538f8b1b2af8bf3770d233331447a48 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 16:46:19 +0800 Subject: [PATCH 0704/1284] fix: switch off vectorization cockroachdb (#659) --- src/sqlancer/cockroachdb/CockroachDBOptions.java | 4 ---- src/sqlancer/cockroachdb/CockroachDBProvider.java | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 4fdf52b0e..c41b018ee 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -99,10 +99,6 @@ public TestOracle create(CockroachDBGlobalState globalSt @Parameter(names = { "--test-temp-tables" }, description = "Test TEMPORARY tables") public boolean testTempTables; // default: false https://github.com/cockroachdb/cockroach/issues/85388 - @Parameter(names = { - "--increased-vectorization" }, description = "Generate VECTORIZE=on with a higher probability (which found a number of bugs in the past)") - public boolean makeVectorizationMoreLikely = true; - @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables that can be created") public int maxNumTables = 10; diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 580fa1ebf..a79d578db 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -247,9 +247,6 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio } total--; } - if (globalState.getDbmsSpecificOptions().makeVectorizationMoreLikely && Randomly.getBoolean()) { - manager.execute(new SQLQueryAdapter("SET vectorize=on;")); - } } @Override From 0168bb1d8a7a5d1ab68f06248d549c09c59a8203 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Sun, 18 Dec 2022 17:07:27 +0800 Subject: [PATCH 0705/1284] opt: maximum number of tables/indexes tidb --- src/sqlancer/tidb/TiDBOptions.java | 7 ++++++- src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 4 ++++ src/sqlancer/tidb/gen/TiDBTableGenerator.java | 8 ++++++++ src/sqlancer/tidb/gen/TiDBViewGenerator.java | 3 +++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 51afa2a9a..764c4bfd9 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -23,6 +23,12 @@ public class TiDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 4000; + @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables/views that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + @Parameter(names = "--oracle") public List oracle = Arrays.asList(TiDBOracleFactory.QUERY_PARTITIONING); @@ -55,5 +61,4 @@ public TestOracle create(TiDBGlobalState globalState) throws SQ public List getTestOracleFactory() { return oracle; } - } diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index 3444d3281..cb870a2df 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -16,6 +17,9 @@ private TiDBIndexGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { + if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } ExpectedErrors errors = new ExpectedErrors(); TiDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 4ca210b90..0ce8190e7 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -23,6 +24,13 @@ public class TiDBTableGenerator { private boolean primaryKeyAsTableConstraints; private final ExpectedErrors errors = new ExpectedErrors(); + public static SQLQueryAdapter createRandomTableStatement(TiDBGlobalState globalState) throws SQLException { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return new TiDBTableGenerator().getQuery(globalState); + } + public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { errors.add("Information schema is changed during the execution of the statement"); String tableName = globalState.getSchema().getFreeTableName(); diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index 9da8e433e..fdbbc0839 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -13,6 +13,9 @@ private TiDBViewGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } int nrColumns = Randomly.smallNumber() + 1; StringBuilder sb = new StringBuilder("CREATE "); if (Randomly.getBoolean()) { From 60394c128f39d8eae8e0a6478d1f3a3d30963d98 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Sun, 18 Dec 2022 17:07:56 +0800 Subject: [PATCH 0706/1284] opt: support dropping tables/views tidb --- .../tidb/gen/TiDBDropTableGenerator.java | 25 +++++++++++++++++++ .../tidb/gen/TiDBDropViewGenerator.java | 25 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 src/sqlancer/tidb/gen/TiDBDropTableGenerator.java create mode 100644 src/sqlancer/tidb/gen/TiDBDropViewGenerator.java diff --git a/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java b/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java new file mode 100644 index 000000000..727fffffc --- /dev/null +++ b/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.tidb.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; + +public final class TiDBDropTableGenerator { + + private TiDBDropTableGenerator() { + } + + public static SQLQueryAdapter dropTable(TiDBGlobalState globalState) { + if (globalState.getSchema().getTables(t -> !t.isView()).size() <= 1) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> !t.isView()).getName()); + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java new file mode 100644 index 000000000..48bfc8a46 --- /dev/null +++ b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.tidb.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; + +public final class TiDBDropViewGenerator { + + private TiDBDropViewGenerator() { + } + + public static SQLQueryAdapter dropView(TiDBGlobalState globalState) { + if (globalState.getSchema().getTables(t -> t.isView()).size() == 0) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP VIEW "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> t.isView()).getName()); + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} From fb93140c112281b72cc7a3e2db9df4eeae740879 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Sun, 18 Dec 2022 17:08:25 +0800 Subject: [PATCH 0707/1284] opt: action update tidb --- src/sqlancer/tidb/TiDBProvider.java | 41 +++++++++++++---------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index af41fb143..dd02a474f 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -16,16 +16,16 @@ import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; import sqlancer.tidb.gen.TiDBDeleteGenerator; +import sqlancer.tidb.gen.TiDBDropTableGenerator; +import sqlancer.tidb.gen.TiDBDropViewGenerator; import sqlancer.tidb.gen.TiDBIndexGenerator; import sqlancer.tidb.gen.TiDBInsertGenerator; -import sqlancer.tidb.gen.TiDBRandomQuerySynthesizer; import sqlancer.tidb.gen.TiDBSetGenerator; import sqlancer.tidb.gen.TiDBTableGenerator; import sqlancer.tidb.gen.TiDBUpdateGenerator; @@ -39,25 +39,20 @@ public TiDBProvider() { } public enum Action implements AbstractAction { - INSERT(TiDBInsertGenerator::getQuery), // - ANALYZE_TABLE(TiDBAnalyzeTableGenerator::getQuery), // - TRUNCATE((g) -> new SQLQueryAdapter("TRUNCATE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), // - CREATE_INDEX(TiDBIndexGenerator::getQuery), // - DELETE(TiDBDeleteGenerator::getQuery), // - SET(TiDBSetGenerator::getQuery), // - UPDATE(TiDBUpdateGenerator::getQuery), // + CREATE_TABLE(TiDBTableGenerator::createRandomTableStatement), // 0 + CREATE_INDEX(TiDBIndexGenerator::getQuery), // 1 + VIEW_GENERATOR(TiDBViewGenerator::getQuery), // 2 + INSERT(TiDBInsertGenerator::getQuery), // 3 + ALTER_TABLE(TiDBAlterTableGenerator::getQuery), // 4 + TRUNCATE((g) -> new SQLQueryAdapter("TRUNCATE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), // 5 + UPDATE(TiDBUpdateGenerator::getQuery), // 6 + DELETE(TiDBDeleteGenerator::getQuery), // 7 + SET(TiDBSetGenerator::getQuery), // 8 ADMIN_CHECKSUM_TABLE( - (g) -> new SQLQueryAdapter("ADMIN CHECKSUM TABLE " + g.getSchema().getRandomTable().getName())), // - VIEW_GENERATOR(TiDBViewGenerator::getQuery), // - ALTER_TABLE(TiDBAlterTableGenerator::getQuery), // - EXPLAIN((g) -> { - ExpectedErrors errors = new ExpectedErrors(); - TiDBErrors.addExpressionErrors(errors); - TiDBErrors.addExpressionHavingErrors(errors); - return new SQLQueryAdapter( - "EXPLAIN " + TiDBRandomQuerySynthesizer.generate(g, Randomly.smallNumber() + 1).getQueryString(), - errors); - }); + (g) -> new SQLQueryAdapter("ADMIN CHECKSUM TABLE " + g.getSchema().getRandomTable().getName())), // 9 + ANALYZE_TABLE(TiDBAnalyzeTableGenerator::getQuery), // 10 + DROP_TABLE(TiDBDropTableGenerator::dropTable), // 11 + DROP_VIEW(TiDBDropViewGenerator::dropView); // 12 private final SQLQueryProvider sqlQueryProvider; @@ -87,8 +82,6 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { case CREATE_INDEX: return r.getInteger(0, 2); case INSERT: - case EXPLAIN: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case TRUNCATE: case DELETE: case ADMIN_CHECKSUM_TABLE: @@ -101,6 +94,10 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { return r.getInteger(0, 2); case ALTER_TABLE: return r.getInteger(0, 10); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/10 + case CREATE_TABLE: + case DROP_TABLE: + case DROP_VIEW: + return 0; default: throw new AssertionError(a); } From 38bf6407a273d7900d29bea0aca9a6404389bf58 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 19 Dec 2022 16:28:32 +0800 Subject: [PATCH 0708/1284] Query Plan Guidance Framework (#661) * QPG framework * QPG for SQLite * QPG for CockroachDB * QPG for TiDB --- src/sqlancer/DatabaseProvider.java | 12 ++ src/sqlancer/Main.java | 54 +++++- src/sqlancer/MainOptions.java | 35 ++++ src/sqlancer/ProviderAdapter.java | 168 +++++++++++++++++- .../cockroachdb/CockroachDBProvider.java | 55 ++++++ src/sqlancer/common/DBMSCommon.java | 12 ++ src/sqlancer/sqlite3/SQLite3Provider.java | 58 ++++++ src/sqlancer/tidb/TiDBProvider.java | 41 +++++ 8 files changed, 430 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 48afb00f5..d169324fa 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -32,6 +32,18 @@ public interface DatabaseProvider, O extends DBMS */ Reproducer generateAndTestDatabase(G globalState) throws Exception; + /** + * The experimental feature: Query Plan Guidance. + * + * @param globalState + * the state created and is valid for this method call. + * + * @throws Exception + * if testing fails. + * + */ + void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws Exception; + C createDatabase(G globalState) throws Exception; /** diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index dfb2bb89b..29610f266 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -51,10 +51,13 @@ public static final class StateLogger { private final File loggerFile; private File curFile; + private File queryPlanFile; private FileWriter logFileWriter; public FileWriter currentFileWriter; + private FileWriter queryPlanFileWriter; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; + private final boolean logQueryPlan; private final DatabaseProvider databaseProvider; private static final class AlsoWriteToConsoleFileWriter extends FileWriter { @@ -87,6 +90,10 @@ public StateLogger(String databaseName, DatabaseProvider provider, Main if (logEachSelect) { curFile = new File(dir, databaseName + "-cur.log"); } + logQueryPlan = options.logQueryPlan(); + if (logQueryPlan) { + queryPlanFile = new File(dir, databaseName + "-plan.log"); + } this.databaseProvider = provider; } @@ -138,6 +145,20 @@ public FileWriter getCurrentFileWriter() { return currentFileWriter; } + public FileWriter getQueryPlanFileWriter() { + if (!logQueryPlan) { + throw new UnsupportedOperationException(); + } + if (queryPlanFileWriter == null) { + try { + queryPlanFileWriter = new FileWriter(queryPlanFile, true); + } catch (IOException e) { + throw new AssertionError(e); + } + } + return queryPlanFileWriter; + } + public void writeCurrent(StateToReproduce state) { if (!logEachSelect) { throw new UnsupportedOperationException(); @@ -172,6 +193,18 @@ private void write(Loggable loggable) { } } + public void writeQueryPlan(String queryPlan) { + if (!logQueryPlan) { + throw new UnsupportedOperationException(); + } + try { + getQueryPlanFileWriter().append(removeNamesFromQueryPlans(queryPlan)); + queryPlanFileWriter.flush(); + } catch (IOException e) { + throw new AssertionError(); + } + } + public void logException(Throwable reduce, StateToReproduce state) { Loggable stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); @@ -201,8 +234,7 @@ private void printState(FileWriter writer, StateToReproduce state) { .getInfo(state.getDatabaseName(), state.getDatabaseVersion(), state.getSeedValue()).getLogString()); for (Query s : state.getStatements()) { - sb.append(s.getLogString()); - sb.append('\n'); + sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); } try { writer.write(sb.toString()); @@ -211,6 +243,13 @@ private void printState(FileWriter writer, StateToReproduce state) { } } + private String removeNamesFromQueryPlans(String queryPlan) { + String result = queryPlan; + result = result.replaceAll("t[0-9]+", "t0"); // Avoid duplicate tables + result = result.replaceAll("v[0-9]+", "v0"); // Avoid duplicate views + result = result.replaceAll("i[0-9]+", "i0"); // Avoid duplicate indexes + return result + "\n"; + } } public static class QueryManager { @@ -243,6 +282,10 @@ public void incrementSelectQueryCount() { Main.nrQueries.addAndGet(1); } + public Long getSelectQueryCount() { + return Main.nrQueries.get(); + } + public void incrementCreateDatabase() { Main.nrDatabases.addAndGet(1); } @@ -314,7 +357,12 @@ public void run() throws Exception { if (options.logEachSelect()) { logger.writeCurrent(state.getState()); } - Reproducer reproducer = provider.generateAndTestDatabase(state); + Reproducer reproducer = null; + if (options.enableQPG()) { + provider.generateAndTestDatabaseWithQueryPlanGuidance(state); + } else { + reproducer = provider.generateAndTestDatabase(state); + } try { logger.getCurrentFileWriter().close(); logger.currentFileWriter = null; diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 329010b39..5283ea2a6 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -50,6 +50,21 @@ public class MainOptions { @Parameter(names = "--print-failed", description = "Logs failed insert, create and other statements without results", arity = 1) private boolean loggerPrintFailed = true; // NOPMD + @Parameter(names = "--qpg-enable", description = "Enable the experimental feature Query Plan Guidance (QPG)", arity = 1) + private boolean enableQPG; + + @Parameter(names = "--qpg-log-query-plan", description = "Logs the query plans of each query (requires --qpg-enable)", arity = 1) + private boolean logQueryPlan; + + @Parameter(names = "--qpg-max-interval", description = "The maximum number of iterations to mutate tables if no new query plans (requires --qpg-enable)") + private static int qpgMaxInterval = 1000; + + @Parameter(names = "--qpg-reward-weight", description = "The weight (0-1) of last reward when updating weighted average reward. A higher value denotes average reward is more affected by the last reward (requires --qpg-enable)") + private static double qpgk = 0.25; + + @Parameter(names = "--qpg-selection-probability", description = "The probability (0-1) of the random selection of mutators. A higher value (>0.5) favors exploration over exploitation. (requires --qpg-enable)") + private static double qpgProbability = 0.7; + @Parameter(names = "--username", description = "The user name used to log into the DBMS") private String userName = "sqlancer"; // NOPMD @@ -151,6 +166,26 @@ public boolean loggerPrintFailed() { return loggerPrintFailed; } + public boolean logQueryPlan() { + return logQueryPlan; + } + + public boolean enableQPG() { + return enableQPG; + } + + public int getQPGMaxMutationInterval() { + return qpgMaxInterval; + } + + public double getQPGk() { + return qpgk; + } + + public double getQPGProbability() { + return qpgProbability; + } + public int getNrQueries() { return nrQueries; } diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 1a530e93e..47b15ee39 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -1,9 +1,14 @@ package sqlancer; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import sqlancer.StateToReproduce.OracleRunReproductionState; +import sqlancer.common.DBMSCommon; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.schema.AbstractSchema; @@ -14,6 +19,13 @@ public abstract class ProviderAdapter globalClass; private final Class optionClass; + // Variables for QPG + Map queryPlanPool = new HashMap<>(); + static double[] weightedAverageReward; // static variable for sharing across all threads + int currentSelectRewards; + int currentSelectCounts; + int currentMutationOperator = -1; + public ProviderAdapter(Class globalClass, Class optionClass) { this.globalClass = globalClass; this.optionClass = optionClass; @@ -67,7 +79,7 @@ public Reproducer generateAndTestDatabase(G globalState) throws Exception { return null; } - protected abstract void checkViewsAreValid(G globalState); + protected abstract void checkViewsAreValid(G globalState) throws SQLException; protected TestOracle getTestOracle(G globalState) throws Exception { List> testOracleFactory = globalState.getDbmsSpecificOptions() @@ -77,7 +89,11 @@ protected TestOracle getTestOracle(G globalState) throws Exception { boolean userRequiresMoreThanZeroRows = globalState.getOptions().testOnlyWithMoreThanZeroRows(); boolean checkZeroRows = testOracleRequiresMoreThanZeroRows || userRequiresMoreThanZeroRows; if (checkZeroRows && globalState.getSchema().containsTableWithZeroRows(globalState)) { - throw new IgnoreMeException(); + if (globalState.getOptions().enableQPG()) { + addRowsToAllTables(globalState); + } else { + throw new IgnoreMeException(); + } } if (testOracleFactory.size() == 1) { return testOracleFactory.get(0).create(globalState); @@ -94,4 +110,152 @@ protected TestOracle getTestOracle(G globalState) throws Exception { public abstract void generateDatabase(G globalState) throws Exception; + // QPG: entry function + @Override + public void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws Exception { + if (weightedAverageReward == null) { + weightedAverageReward = initializeWeightedAverageReward(); // Same length as the list of mutators + } + try { + generateDatabase(globalState); + checkViewsAreValid(globalState); + globalState.getManager().incrementCreateDatabase(); + + Long executedQueryCount = 0L; + while (executedQueryCount < globalState.getOptions().getNrQueries()) { + int numOfNoNewQueryPlans = 0; + TestOracle oracle = getTestOracle(globalState); + while (true) { + try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { + assert localState != null; + try { + oracle.check(); + String query = oracle.getLastQueryString(); + executedQueryCount += 1; + if (addQueryPlan(query, globalState)) { + numOfNoNewQueryPlans = 0; + } else { + numOfNoNewQueryPlans++; + } + globalState.getManager().incrementSelectQueryCount(); + } catch (IgnoreMeException e) { + + } + assert localState != null; + localState.executedWithoutError(); + } + // exit loop to mutate tables if no new query plans have been found after a while + if (numOfNoNewQueryPlans > globalState.getOptions().getQPGMaxMutationInterval()) { + mutateTables(globalState); + break; + } + } + } + } finally { + globalState.getConnection().close(); + } + } + + // QPG: mutate tables for a new database state + private synchronized boolean mutateTables(G globalState) throws Exception { + // Update rewards based on a set of newly generated queries in last iteration + if (currentMutationOperator != -1) { + weightedAverageReward[currentMutationOperator] += ((double) currentSelectRewards + / (double) currentSelectCounts) * globalState.getOptions().getQPGk(); + } + currentMutationOperator = -1; + + // Choose mutator based on the rewards + int selectedActionIndex = 0; + if (Randomly.getPercentage() < globalState.getOptions().getQPGProbability()) { + selectedActionIndex = globalState.getRandomly().getInteger(0, weightedAverageReward.length); + } else { + selectedActionIndex = DBMSCommon.getMaxIndexInDoubleArrary(weightedAverageReward); + } + int reward = 0; + + try { + executeMutator(selectedActionIndex, globalState); + checkViewsAreValid(globalState); // Remove the invalid views + reward = checkQueryPlan(globalState); + } catch (IgnoreMeException | AssertionError e) { + } finally { + // Update rewards based on existing queries associated with the query plan pool + updateReward(selectedActionIndex, (double) reward / (double) queryPlanPool.size(), globalState); + currentMutationOperator = selectedActionIndex; + } + + // Clear the variables for storing the rewards of the action on a set of newly generated queries + currentSelectRewards = 0; + currentSelectCounts = 0; + return true; + } + + // QPG: add a query plan to the query plan pool and return true if the query plan is new + private boolean addQueryPlan(String selectStr, G globalState) throws Exception { + String queryPlan = getQueryPlan(selectStr, globalState); + + if (globalState.getOptions().logQueryPlan()) { + globalState.getLogger().writeQueryPlan(queryPlan); + } + + currentSelectCounts += 1; + if (queryPlanPool.containsKey(queryPlan)) { + return false; + } else { + queryPlanPool.put(queryPlan, selectStr); + currentSelectRewards += 1; + return true; + } + } + + // Obtain the reward of the current action based on the queries associated with the query plan pool + private int checkQueryPlan(G globalState) throws Exception { + int newQueryPlanFound = 0; + HashMap modifiedQueryPlan = new HashMap<>(); + for (Iterator> it = queryPlanPool.entrySet().iterator(); it.hasNext();) { + Map.Entry item = it.next(); + String queryPlan = item.getKey(); + String selectStr = item.getValue(); + String newQueryPlan = getQueryPlan(selectStr, globalState); + if (newQueryPlan.isEmpty()) { // Invalid query + it.remove(); + } else if (!queryPlan.equals(newQueryPlan)) { // A query plan has been changed + it.remove(); + modifiedQueryPlan.put(newQueryPlan, selectStr); + if (!queryPlanPool.containsKey(newQueryPlan)) { // A new query plan is found + newQueryPlanFound++; + } + } + } + queryPlanPool.putAll(modifiedQueryPlan); + return newQueryPlanFound; + } + + // QPG: update the reward of current action + private void updateReward(int actionIndex, double reward, G globalState) { + weightedAverageReward[actionIndex] += (reward - weightedAverageReward[actionIndex]) + * globalState.getOptions().getQPGk(); + } + + // QPG: initialize the weighted average reward of all mutation operators (required implementation in specific DBMS) + protected double[] initializeWeightedAverageReward() { + throw new UnsupportedOperationException(); + } + + // QPG: obtain the query plan of a query (required implementation in specific DBMS) + protected String getQueryPlan(String selectStr, G globalState) throws Exception { + throw new UnsupportedOperationException(); + } + + // QPG: execute a mutation operator (required implementation in specific DBMS) + protected void executeMutator(int index, G globalState) throws Exception { + throw new UnsupportedOperationException(); + } + + // QPG: add rows to all tables (required implementation in specific DBMS when enabling PQS oracle for QPG) + protected boolean addRowsToAllTables(G globalState) throws Exception { + throw new UnsupportedOperationException(); + } + } diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index a79d578db..9bc72b614 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -1,5 +1,6 @@ package sqlancer.cockroachdb; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -37,6 +38,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; @AutoService(DatabaseProvider.class) public class CockroachDBProvider extends SQLProviderAdapter { @@ -285,4 +287,57 @@ public String getDBMSName() { return "cockroachdb"; } + @Override + public String getQueryPlan(String selectStr, CockroachDBGlobalState globalState) throws Exception { + String queryPlan = ""; + String explainQuery = "EXPLAIN (OPT) " + selectStr; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, null); + boolean afterProjection = false; // Remove the concrete expression after each Projection operator + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String targetQueryPlan = rs.getString(1).replace("└──", "").replace("├──", "").replace("│", "") + .trim() + ";"; // Unify format + if (afterProjection) { + afterProjection = false; + continue; + } + if (targetQueryPlan.startsWith("projections")) { + afterProjection = true; + } + // Remove all concrete expressions by keywords + if (targetQueryPlan.contains(">") || targetQueryPlan.contains("<") || targetQueryPlan.contains("=") + || targetQueryPlan.contains("*") || targetQueryPlan.contains("+") + || targetQueryPlan.contains("'")) { + continue; + } + queryPlan += targetQueryPlan; + } + } + } catch (AssertionError e) { + throw new AssertionError("Explain failed: " + explainQuery); + } + + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, CockroachDBGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + } diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java index 2c7531d10..122b34a83 100644 --- a/src/sqlancer/common/DBMSCommon.java +++ b/src/sqlancer/common/DBMSCommon.java @@ -27,4 +27,16 @@ public static boolean matchesIndexName(String indexName) { return matcher.matches(); } + public static int getMaxIndexInDoubleArrary(double... doubleArray) { + int maxIndex = 0; + double maxValue = 0.0; + for (int j = 0; j < doubleArray.length; j++) { + double curReward = doubleArray[j]; + if (curReward > maxValue) { + maxIndex = j; + maxValue = curReward; + } + } + return maxIndex; + } } diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 1cb5e0633..b20701c95 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -1,11 +1,13 @@ package sqlancer.sqlite3; import java.io.File; +import java.io.IOException; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -20,6 +22,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; import sqlancer.sqlite3.gen.SQLite3CreateVirtualRtreeTabelGenerator; @@ -299,4 +302,59 @@ public SQLConnection createDatabase(SQLite3GlobalState globalState) throws SQLEx public String getDBMSName() { return "sqlite3"; } + + @Override + public String getQueryPlan(String selectStr, SQLite3GlobalState globalState) throws Exception { + String queryPlan = ""; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(selectStr); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + // Set up the expected errors for NoREC oracle. + ExpectedErrors errors = new ExpectedErrors(); + SQLite3Errors.addExpectedExpressionErrors(errors); + SQLite3Errors.addMatchQueryErrors(errors); + SQLite3Errors.addQueryErrors(errors); + SQLite3Errors.addInsertUpdateErrors(errors); + + SQLQueryAdapter q = new SQLQueryAdapter(SQLite3ExplainGenerator.explain(selectStr), errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + queryPlan += rs.getString(4) + ";"; + } + } + } catch (SQLException | AssertionError e) { + queryPlan = ""; + } + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, SQLite3GlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + + } + + @Override + protected boolean addRowsToAllTables(SQLite3GlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (SQLite3Table table : tablesNoRow) { + SQLQueryAdapter queryAddRows = SQLite3InsertGenerator.insertRow(globalState, table); + globalState.executeStatement(queryAddRows); + } + + return true; + } } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index dd02a474f..fbb905eea 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -1,5 +1,6 @@ package sqlancer.tidb; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -18,6 +19,7 @@ import sqlancer.StatementExecutor; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; @@ -169,4 +171,43 @@ public String getDBMSName() { return "tidb"; } + @Override + public String getQueryPlan(String selectStr, TiDBGlobalState globalState) throws Exception { + String queryPlan = ""; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(selectStr); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN " + selectStr, null); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String targetQueryPlan = rs.getString(1).replace("├─", "").replace("└─", "").replace("│", "").trim() + + ";"; // Unify format + queryPlan += targetQueryPlan; + } + } + } catch (Throwable e) { + e.printStackTrace(); + } + + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, TiDBGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + } From 327ac2976608374b7a6e673ac057f8aad3d45dcb Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 19 Dec 2022 16:58:40 +0800 Subject: [PATCH 0709/1284] doc: add QPG (#663) --- README.md | 7 ++++++- docs/PAPERS.md | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index af793f42c..283efcd29 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Man SQLancer operates in the following two phases: -1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. +1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. **New: we support query plan guided generation now. See Generation Approaches below.** 2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. # Getting Started @@ -40,6 +40,11 @@ If you launch SQLancer without parameters, available options and commands are di | Pivoted Query Synthesis (PQS) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described [here](https://arxiv.org/abs/2001.04174). PQS is the most powerful technique, but also requires more implementation effort than the other two techniques. It is currently unmaintained. | | Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | | Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | +# Generation Approaches +| Approach | Description | +|----------|-------------| +| Random Generation | Random generation is the default test case generation approach in SQLancer. First, random tables are generated. Then queries are randomly generated based on the schemas of the tables. | +| Query Plan Guidance (QPG) | QPG is a test case generation method guided by query plan coverage. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. This approach is enabled by option `--qpg-enable` and now supports TLP and NoREC oracles for SQLite, CockroachDB, and TiDB. | Please find the `.bib` entries [here](docs/PAPERS.md). diff --git a/docs/PAPERS.md b/docs/PAPERS.md index ca9d40de7..45f57e887 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -1,6 +1,6 @@ # Papers -The testing approaches implemented in SQLancer are described in the three papers below. +The testing approaches implemented in SQLancer are described in the four papers below. ## Testing Database Engines via Pivoted Query Synthesis @@ -51,6 +51,20 @@ This paper describes TLP, a metamorphic testing approach that can detect various } ``` +## Testing Database Engines via Query Plan Guidance + +This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. + +``` +@inproceedings{Ba2023QPG, + author = {Ba, Jinsheng and Rigger, Manuel}, + title = {Testing Database Engines via Query Plan Guidance}, + booktitle = {The 45th International Conference on Software Engineering (ICSE'23)}, + year = {2023}, + month = may +} +``` + # Comparing SQLancer With Other Tools that Find Logic Bugs If you want to fairly compare other tools with SQLancer, we would be glad to provide feedback (e.g., feel free to send an email to manuel.rigger@inf.ethz.ch). We have the following general recommendations and comments: From 7be4b6972858d048a9b1289d998491dd92cc39aa Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 19 Dec 2022 17:49:44 +0800 Subject: [PATCH 0710/1284] bypass new errors in SQLite (#664) --- src/sqlancer/sqlite3/SQLite3Errors.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 6cd4c40c9..6827116af 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -47,6 +47,7 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("json_insert() needs an odd number of arguments"); errors.add("json_object() labels must be TEXT"); errors.add("json_object() requires an even number of arguments"); + errors.add("argument of ntile must be a positive integer"); // fts5 functions errors.add("unable to use function highlight in the requested context"); @@ -61,6 +62,11 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("ORDER BY clause should come after"); errors.add("LIMIT clause should come after"); + errors.add("unsafe use of load_extension"); + errors.add("table does not support scanning"); + errors.add("circularly defined"); + errors.add("[SQLITE_ERROR] SQL error or missing database"); // A possible delay in the execution of DROP TABLE + // statement. } public static void addMatchQueryErrors(ExpectedErrors errors) { From d970f525c5edf8d4ae2d05c7581b71a844a3cd03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 10:16:07 +0000 Subject: [PATCH 0711/1284] build(deps): bump slf4j-simple from 2.0.3 to 2.0.6 (#668) Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 2.0.3 to 2.0.6. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_2.0.3...v_2.0.6) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7b163a49f..1651566d3 100644 --- a/pom.xml +++ b/pom.xml @@ -325,7 +325,7 @@ org.slf4j slf4j-simple - 2.0.3 + 2.0.6 ru.yandex.clickhouse From 4d42dad02d0b6d43cac2a05e7e7741e6587bf626 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 19 Dec 2022 20:21:18 +0800 Subject: [PATCH 0712/1284] test cases for QPG (#670) * testcase for QPG in SQLite * workflows: QPG Tests (SQLite) --- .github/workflows/main.yml | 17 +++++++++++++++++ test/sqlancer/dbms/TestSQLiteQPG.java | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 test/sqlancer/dbms/TestSQLiteQPG.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a15ac75b..5b498033d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -272,6 +272,23 @@ jobs: mvn -Dtest=TestSQLitePQS test mvn -Dtest=TestSQLite3 test + sqlite-qpg: + name: QPG Tests (SQLite) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build + run: mvn -B package -DskipTests=true + - name: SQLite Tests for QPG + run: | + mvn -Dtest=TestSQLiteQPG test tidb: name: DBMS Tests (TiDB) diff --git a/test/sqlancer/dbms/TestSQLiteQPG.java b/test/sqlancer/dbms/TestSQLiteQPG.java new file mode 100644 index 000000000..37be6f78d --- /dev/null +++ b/test/sqlancer/dbms/TestSQLiteQPG.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLiteQPG { + + @Test + public void testSqliteQPG() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", "1000", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", + "--database-prefix", "pqsdb" /* Workaround for connections not being closed */, "--qpg-enable", "true", + "sqlite3", "--oracle", "NoREC", "--test-fts", "false", "--test-rtree", "false", + "--test-check-constraints", "false", "--test-in-operator", "false" })); + } + +} From 2fdc987f53f23812abe76e31b14ffc9c177b2155 Mon Sep 17 00:00:00 2001 From: Nathee Jaywaree <73115539+eltfshr@users.noreply.github.com> Date: Mon, 19 Dec 2022 21:12:18 +0700 Subject: [PATCH 0713/1284] Refactor UpdateGenerator classes using AbstractUpdateGenerator class (#662) * Add AbstractUpdateGenerator class * Refactor UpdateGenerator classes * Add ExpectedErrors to AbstractUpdateGenerator Co-authored-by: Manuel Rigger --- .../gen/CockroachDBUpdateGenerator.java | 34 +++++---- .../common/gen/AbstractUpdateGenerator.java | 26 +++++++ .../duckdb/gen/DuckDBUpdateGenerator.java | 49 ++++++------ src/sqlancer/h2/H2UpdateGenerator.java | 35 +++++---- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 53 +++++++------ .../gen/OceanBaseUpdateGenerator.java | 36 +++++---- .../postgres/gen/PostgresUpdateGenerator.java | 74 ++++++++++--------- .../gen/dml/SQLite3UpdateGenerator.java | 30 +++----- .../tidb/gen/TiDBUpdateGenerator.java | 45 ++++++----- .../ycql/gen/YCQLUpdateGenerator.java | 50 +++++++------ .../ysql/gen/YSQLUpdateGenerator.java | 73 +++++++++--------- 11 files changed, 276 insertions(+), 229 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractUpdateGenerator.java diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index b8a96e4c7..8dcd605d7 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -9,20 +9,27 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; -public final class CockroachDBUpdateGenerator { +public final class CockroachDBUpdateGenerator extends AbstractUpdateGenerator { - private CockroachDBUpdateGenerator() { + private final CockroachDBGlobalState globalState; + private CockroachDBExpressionGenerator gen; + + private CockroachDBUpdateGenerator(CockroachDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); + return new CockroachDBUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); - StringBuilder sb = new StringBuilder("UPDATE "); + gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); + sb.append("UPDATE "); sb.append(table.getName()); if (Randomly.getBoolean()) { sb.append("@{FORCE_INDEX="); @@ -30,15 +37,7 @@ public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { sb.append("}"); } sb.append(" SET "); - int i = 0; - for (CockroachDBColumn c : columns) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(c.getName()); - sb.append("="); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(c.getType()))); - } + updateColumns(columns); if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); @@ -55,4 +54,9 @@ public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(CockroachDBColumn column) { + sb.append(CockroachDBVisitor.asString(gen.generateExpression(column.getType()))); + } + } diff --git a/src/sqlancer/common/gen/AbstractUpdateGenerator.java b/src/sqlancer/common/gen/AbstractUpdateGenerator.java new file mode 100644 index 000000000..f130c15a5 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractUpdateGenerator.java @@ -0,0 +1,26 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractUpdateGenerator> { + + protected final ExpectedErrors errors = new ExpectedErrors(); + protected StringBuilder sb = new StringBuilder(); + + protected void updateColumns(List columns) { + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + sb.append(columns.get(nrColumn).getName()); + sb.append("="); + updateValue(columns.get(nrColumn)); + } + } + + protected abstract void updateValue(C column); + +} diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index ba8c867fd..13b7f9d74 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -4,7 +4,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -13,36 +13,41 @@ import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -public final class DuckDBUpdateGenerator { +public final class DuckDBUpdateGenerator extends AbstractUpdateGenerator { - private DuckDBUpdateGenerator() { + private final DuckDBGlobalState globalState; + private DuckDBExpressionGenerator gen; + + private DuckDBUpdateGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); + return new DuckDBUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - Node expr; - if (Randomly.getBooleanWithSmallProbability()) { - expr = gen.generateExpression(); - DuckDBErrors.addExpressionErrors(errors); - } else { - expr = gen.generateConstant(); - } - sb.append(DuckDBToStringVisitor.asString(expr)); - } + updateColumns(columns); DuckDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(DuckDBColumn column) { + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + DuckDBErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(DuckDBToStringVisitor.asString(expr)); + } + } diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index 0dfc15938..158621409 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -3,33 +3,33 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2Table; -public final class H2UpdateGenerator { +public final class H2UpdateGenerator extends AbstractUpdateGenerator { - private H2UpdateGenerator() { + private final H2GlobalState globalState; + private H2ExpressionGenerator gen; + + private H2UpdateGenerator(H2GlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); + return new H2UpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new H2ExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); - H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - sb.append(H2ToStringVisitor.asString(gen.generateConstant())); - } + updateColumns(columns); H2Errors.addInsertErrors(errors); H2Errors.addDeleteErrors(errors); if (Randomly.getBoolean()) { @@ -40,4 +40,9 @@ public static SQLQueryAdapter getQuery(H2GlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(H2Column column) { + sb.append(H2ToStringVisitor.asString(gen.generateConstant())); + } + } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index d7e2ed1a4..bc1a220b1 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -4,54 +4,53 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBErrors; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBCompositeDataType; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBDataType; import sqlancer.hsqldb.HSQLDBToStringVisitor; import sqlancer.hsqldb.ast.HSQLDBExpression; -public final class HSQLDBUpdateGenerator { +public final class HSQLDBUpdateGenerator extends AbstractUpdateGenerator { - private static final ExpectedErrors EXPECTED_ERRORS = new ExpectedErrors(); private final HSQLDBProvider.HSQLDBGlobalState globalState; + private HSQLDBExpressionGenerator gen; private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { this.globalState = globalState; } public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { - return new HSQLDBUpdateGenerator(globalState).getQuery(); + return new HSQLDBUpdateGenerator(globalState).generate(); } - private SQLQueryAdapter getQuery() { - StringBuilder sb = new StringBuilder("UPDATE "); + private SQLQueryAdapter generate() { HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); - HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); - ExpectedErrors errors = new ExpectedErrors(); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - Node expr; - expr = gen.generateConstant(columns.get(i).getType()); - sb.append(HSQLDBToStringVisitor.asString(expr)); - if (Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); - expr = gen.generateExpression(columns.get(i).getType()); - sb.append(HSQLDBToStringVisitor.asString(expr)); - errors.add("data type of expression is not boolean"); - HSQLDBErrors.addExpressionErrors(errors); - } - + updateColumns(columns); + if (Randomly.getBooleanWithSmallProbability()) { + sb.append(" WHERE "); + sb.append(HSQLDBToStringVisitor.asString( + gen.generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)))); + errors.add("data type of expression is not boolean"); + HSQLDBErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), EXPECTED_ERRORS); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void updateValue(HSQLDBColumn column) { + Node expr; + expr = gen.generateConstant(column.getType()); + sb.append(HSQLDBToStringVisitor.asString(expr)); } } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index f89434498..950317bd2 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -3,17 +3,18 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; import sqlancer.oceanbase.OceanBaseVisitor; -public class OceanBaseUpdateGenerator { +public class OceanBaseUpdateGenerator extends AbstractUpdateGenerator { - private final StringBuilder sb = new StringBuilder(); private final OceanBaseGlobalState globalState; + private OceanBaseExpressionGenerator gen; private final Randomly r; public OceanBaseUpdateGenerator(OceanBaseGlobalState globalState) { @@ -26,29 +27,16 @@ public static SQLQueryAdapter update(OceanBaseGlobalState globalState) { } private SQLQueryAdapter generate() { - ExpectedErrors errors = new ExpectedErrors(); OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append("UPDATE "); if (Randomly.getBoolean()) { sb.append(" /*+parallel(" + r.getInteger(0, 10) + ") enable_parallel_dml*/ "); } sb.append(table.getName()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - if (Randomly.getBoolean()) { - sb.append(gen.generateConstant(columns.get(i))); - } else { - sb.append(OceanBaseVisitor.asString(gen.generateExpression())); - OceanBaseErrors.addExpressionErrors(errors); - } - } + updateColumns(columns); if (Randomly.getBoolean()) { sb.append(" WHERE "); OceanBaseErrors.addExpressionErrors(errors); @@ -60,4 +48,14 @@ private SQLQueryAdapter generate() { return new SQLQueryAdapter(sb.toString(), errors); } + + @Override + protected void updateValue(OceanBaseColumn column) { + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant(column)); + } else { + sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + OceanBaseErrors.addExpressionErrors(errors); + } + } } diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 1122b2b5a..7ce7fe882 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -1,9 +1,10 @@ package sqlancer.postgres.gen; +import java.util.Arrays; import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -12,51 +13,36 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public final class PostgresUpdateGenerator { +public final class PostgresUpdateGenerator extends AbstractUpdateGenerator { - private PostgresUpdateGenerator() { + private final PostgresGlobalState globalState; + private PostgresTable randomTable; + + private PostgresUpdateGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; + errors.addAll(Arrays.asList("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", + " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable")); } public static SQLQueryAdapter create(PostgresGlobalState globalState) { - PostgresTable randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); - StringBuilder sb = new StringBuilder(); + return new PostgresUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); sb.append(randomTable.getName()); sb.append(" SET "); - ExpectedErrors errors = ExpectedErrors.from("conflicting key value violates exclusion constraint", - "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", - "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", - " bit string too long", "can only be updated to DEFAULT", "division by zero", - "You might need to add explicit type casts.", "invalid regular expression", - "View columns that are not columns of their base relation are not updatable"); errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced // table multiple times errors.add("new row violates check option for view"); - List columns = randomTable.getRandomNonEmptyColumnSubset(); PostgresCommon.addCommonInsertUpdateErrors(errors); - - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - PostgresColumn column = columns.get(i); - sb.append(column.getName()); - sb.append(" = "); - if (!Randomly.getBoolean()) { - PostgresExpression constant = PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), - column.getType()); - sb.append(PostgresVisitor.asString(constant)); - } else if (Randomly.getBoolean()) { - sb.append("DEFAULT"); - } else { - sb.append("("); - PostgresExpression expr = PostgresExpressionGenerator.generateExpression(globalState, - randomTable.getColumns(), column.getType()); - // caused by casts - sb.append(PostgresVisitor.asString(expr)); - sb.append(")"); - } - } + updateColumns(columns); errors.add("invalid input syntax for "); errors.add("operator does not exist: text = boolean"); errors.add("violates check constraint"); @@ -73,4 +59,22 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors, true); } + @Override + protected void updateValue(PostgresColumn column) { + if (!Randomly.getBoolean()) { + PostgresExpression constant = PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(PostgresVisitor.asString(constant)); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append("("); + PostgresExpression expr = PostgresExpressionGenerator.generateExpression(globalState, + randomTable.getColumns(), column.getType()); + // caused by casts + sb.append(PostgresVisitor.asString(expr)); + sb.append(")"); + } + } + } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 791b7f5eb..5a17ad339 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -4,7 +4,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; @@ -14,12 +14,10 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3UpdateGenerator { +public class SQLite3UpdateGenerator extends AbstractUpdateGenerator { - private final StringBuilder sb = new StringBuilder(); - private final Randomly r; - private final ExpectedErrors errors = new ExpectedErrors(); private final SQLite3GlobalState globalState; + private final Randomly r; public SQLite3UpdateGenerator(SQLite3GlobalState globalState, Randomly r) { this.globalState = globalState; @@ -34,10 +32,11 @@ public static SQLQueryAdapter updateRow(SQLite3GlobalState globalState) { public static SQLQueryAdapter updateRow(SQLite3GlobalState globalState, SQLite3Table table) { SQLite3UpdateGenerator generator = new SQLite3UpdateGenerator(globalState, globalState.getRandomly()); - return generator.update(table); + return generator.generate(table); } - private SQLQueryAdapter update(SQLite3Table table) { + private SQLQueryAdapter generate(SQLite3Table table) { + List columnsToUpdate = Randomly.nonEmptySubsetPotentialDuplicates(table.getColumns()); sb.append("UPDATE "); if (Randomly.getBoolean()) { sb.append("OR IGNORE "); @@ -55,7 +54,6 @@ private SQLQueryAdapter update(SQLite3Table table) { sb.append(table.getName()); sb.append(" SET "); - List columnsToUpdate = Randomly.nonEmptySubsetPotentialDuplicates(table.getColumns()); if (Randomly.getBoolean()) { sb.append("("); sb.append(columnsToUpdate.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); @@ -66,19 +64,12 @@ private SQLQueryAdapter update(SQLite3Table table) { if (i != 0) { sb.append(", "); } - getToUpdateValue(columnsToUpdate, i); + updateValue(columnsToUpdate.get(i)); } sb.append(")"); // row values } else { - for (int i = 0; i < columnsToUpdate.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columnsToUpdate.get(i).getName()); - sb.append(" = "); - getToUpdateValue(columnsToUpdate, i); - } + updateColumns(columnsToUpdate); } if (Randomly.getBoolean()) { @@ -111,8 +102,9 @@ private SQLQueryAdapter update(SQLite3Table table) { } - private void getToUpdateValue(List columnsToUpdate, int i) { - if (columnsToUpdate.get(i).isIntegerPrimaryKey()) { + @Override + protected void updateValue(SQLite3Column column) { + if (column.isIntegerPrimaryKey()) { sb.append(SQLite3Visitor.asString(SQLite3Constant.createIntConstant(r.getInteger()))); } else { sb.append(SQLite3Visitor.asString(SQLite3ExpressionGenerator.getRandomLiteralValue(globalState))); diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 4912899fe..241ee3321 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -4,7 +4,7 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; @@ -13,32 +13,27 @@ import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.visitor.TiDBVisitor; -public final class TiDBUpdateGenerator { +public final class TiDBUpdateGenerator extends AbstractUpdateGenerator { - private TiDBUpdateGenerator() { + private final TiDBGlobalState globalState; + private TiDBExpressionGenerator gen; + + private TiDBUpdateGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - ExpectedErrors errors = new ExpectedErrors(); + return new TiDBUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() throws SQLException { TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); - StringBuilder sb = new StringBuilder("UPDATE "); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - if (Randomly.getBoolean()) { - sb.append(gen.generateConstant()); - } else { - sb.append(TiDBVisitor.asString(gen.generateExpression())); - TiDBErrors.addExpressionErrors(errors); - } - } + updateColumns(columns); if (Randomly.getBoolean()) { sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); @@ -49,4 +44,14 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(TiDBColumn column) { + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant()); + } else { + sb.append(TiDBVisitor.asString(gen.generateExpression())); + TiDBErrors.addExpressionErrors(errors); + } + } + } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java index d417ca96a..695301595 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -4,7 +4,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLErrors; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; @@ -13,35 +13,27 @@ import sqlancer.yugabyte.ycql.YCQLToStringVisitor; import sqlancer.yugabyte.ycql.ast.YCQLExpression; -public final class YCQLUpdateGenerator { +public final class YCQLUpdateGenerator extends AbstractUpdateGenerator { - private YCQLUpdateGenerator() { + private final YCQLGlobalState globalState; + private YCQLExpressionGenerator gen; + + private YCQLUpdateGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); + return new YCQLUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); - YCQLExpressionGenerator gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - Node expr; - if (Randomly.getBooleanWithSmallProbability()) { - expr = gen.generateExpression(); - YCQLErrors.addExpressionErrors(errors); - } else { - expr = gen.generateConstant(); - } - sb.append(YCQLToStringVisitor.asString(expr)); - } - + updateColumns(columns); errors.add("Invalid Arguments"); errors.add("Invalid CQL Statement"); errors.add("Invalid SQL Statement"); @@ -53,4 +45,16 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(YCQLColumn column) { + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + YCQLErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(YCQLToStringVisitor.asString(expr)); + } + } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index 06e762440..bc7b00d79 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -1,9 +1,10 @@ package sqlancer.yugabyte.ysql.gen; +import java.util.Arrays; import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; @@ -13,51 +14,37 @@ import sqlancer.yugabyte.ysql.YSQLVisitor; import sqlancer.yugabyte.ysql.ast.YSQLExpression; -public final class YSQLUpdateGenerator { +public final class YSQLUpdateGenerator extends AbstractUpdateGenerator { - private YSQLUpdateGenerator() { + private final YSQLGlobalState globalState; + private YSQLTable randomTable; + + private YSQLUpdateGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; + errors.addAll(Arrays.asList("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", + " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable")); } public static SQLQueryAdapter create(YSQLGlobalState globalState) { - YSQLTable randomTable = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); - StringBuilder sb = new StringBuilder(); + return new YSQLUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + randomTable = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); + List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); sb.append(randomTable.getName()); sb.append(" SET "); - ExpectedErrors errors = ExpectedErrors.from("conflicting key value violates exclusion constraint", - "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", - "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", - " bit string too long", "can only be updated to DEFAULT", "division by zero", - "You might need to add explicit type casts.", "invalid regular expression", - "View columns that are not columns of their base relation are not updatable"); errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced // table multiple times errors.add("new row violates check option for view"); - List columns = randomTable.getRandomNonEmptyColumnSubset(); YSQLErrors.addCommonInsertUpdateErrors(errors); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - YSQLColumn column = columns.get(i); - sb.append(column.getName()); - sb.append(" = "); - if (!Randomly.getBoolean()) { - YSQLExpression constant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), - column.getType()); - sb.append(YSQLVisitor.asString(constant)); - } else if (Randomly.getBoolean()) { - sb.append("DEFAULT"); - } else { - sb.append("("); - YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), - column.getType()); - // caused by casts - sb.append(YSQLVisitor.asString(expr)); - sb.append(")"); - } - } + updateColumns(columns); errors.add("invalid input syntax for "); errors.add("operator does not exist: text = boolean"); errors.add("violates check constraint"); @@ -74,4 +61,22 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors, true); } + @Override + protected void updateValue(YSQLColumn column) { + if (!Randomly.getBoolean()) { + YSQLExpression constant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(YSQLVisitor.asString(constant)); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append("("); + YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), + column.getType()); + // caused by casts + sb.append(YSQLVisitor.asString(expr)); + sb.append(")"); + } + } + } From f6e7c608a5c42c1034ed71845d34931f8add826b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 14:34:46 +0000 Subject: [PATCH 0714/1284] build(deps): bump maven-dependency-plugin from 3.3.0 to 3.4.0 (#667) Bumps [maven-dependency-plugin](https://github.com/apache/maven-dependency-plugin) from 3.3.0 to 3.4.0. - [Release notes](https://github.com/apache/maven-dependency-plugin/releases) - [Commits](https://github.com/apache/maven-dependency-plugin/compare/maven-dependency-plugin-3.3.0...maven-dependency-plugin-3.4.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-dependency-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1651566d3..1e6ea73dc 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.3.0 + 3.4.0 copy-dependencies From bf08f1d7ed2f1f34d152bff2968577b3dc6f8c07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 14:58:00 +0000 Subject: [PATCH 0715/1284] build(deps): bump plexus-compiler-api from 2.12.1 to 2.13.0 (#666) Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.12.1 to 2.13.0. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.12.1...plexus-compiler-2.13.0) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1e6ea73dc..320e24897 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ org.codehaus.plexus plexus-compiler-api - 2.12.1 + 2.13.0 From c5ab213a4f997d66fdd6b0adf7f5ab07d1b83a33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Dec 2022 03:14:19 +0800 Subject: [PATCH 0716/1284] build(deps): bump plexus-compiler-eclipse from 2.12.1 to 2.13.0 (#665) Bumps plexus-compiler-eclipse from 2.12.1 to 2.13.0. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 320e24897..b61126106 100644 --- a/pom.xml +++ b/pom.xml @@ -123,7 +123,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.12.1 + 2.13.0 org.eclipse.jdt From 30f8ac57b22a711ec75d4159c3c36096fcf8478b Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 20 Dec 2022 20:35:15 +0800 Subject: [PATCH 0717/1284] doc: QPG mutator list (#671) --- docs/QueryPlanGuidance.md | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/QueryPlanGuidance.md diff --git a/docs/QueryPlanGuidance.md b/docs/QueryPlanGuidance.md new file mode 100644 index 000000000..bb467461b --- /dev/null +++ b/docs/QueryPlanGuidance.md @@ -0,0 +1,66 @@ +# Query Plan Guidance +Query Plan Guidance (QPG) is a test case generation method that attempts to explore unseen query plans. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. Here, we document all mutators in which we choose the most promising one that may help covering more unique query plans to execute. + +# Mutators +All mutators are listed below and implemented in the enumeration variables `Action` in the `XXDBProvider.java` file of each DBMS. +The `Mutator` column includes the items in the `Action` enumeration variable. +The `Example` column includes an example of a realistic statement generated by this mutator. +The `Description` column includes an explanation of what the mutator does. +The `More unique query plans...` column explains why applying this mutator may help covering more unique query plans. + + +|DBMS |Mutator |Example |Description |More unique query plans may be covered because of | +|-----------|---------------------|--------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|--------------------------------------------------------| +|SQLite |PRAGMA |PRAGMA automatic_index true; |It modifies database options. |different options that decide how to execute statements.| +|SQLite |CREATE_INDEX |CREATE INDEX i0 ON t0 WHERE c0 ISNULL; |It adds a new index on a table. |subsequent differnt logic of querying data. | +|SQLite |CREATE_VIEW |CREATE VIEW v0(c0) AS SELECT DISTINCT ABS(t0.c2) FROM t0; |It adds a new view from existing tables. |more possible execution logics on the view. | +|SQLite |CREATE_TABLE |CREATE TABLE t0 (c0 INT CHECK ((c0) BETWEEN (1) AND (10)) ); |It adds a new table. |more possible execution logics on the table. | +|SQLite |CREATE_VIRTUALTABLE |CREATE VIRTUAL TABLE vt1 USING fts5(c0 UNINDEXED); |It adds a new table with fts5 feature. |more possible execution logics on the table with fts5. | +|SQLite |CREATE_RTREETABLE |CREATE VIRTUAL TABLE rt0 USING rtree_i32(c0, c1, c2, c3, c4); |It adds a new table with rtree feature. |more possible execution logics on the table with rtree. | +|SQLite |INSERT |INSERT INTO t0(c0, c1) VALUES ('lrd+a*', NULL); |It inserts a new row to a table. |subsequent different logic of querying data. | +|SQLite |DELETE |DELETE FROM t0 WHERE (c0>3); |It deletes specific rows from a table. |subsequent different logic of querying data. | +|SQLite |ALTER |ALTER TABLE t0 ADD COLUMN c39 REAL; |It changes the schema of a table. |more possible execution logics on the changed table. | +|SQLite |UPDATE |UPDATE t0 SET (c2, c0)=(-944, 'L((xA') WHERE t0.c1; |It updates specific data of a table. |subsequent different logic of querying data. | +|SQLite |DROP_INDEX |DROP INDEX i0; |It drops an index. |subsequent different logic of querying data. | +|SQLite |DROP_TABLE |DROP TABLE t0; |it drops an table. |subsequent different logic of querying data. | +|SQLite |DROP_VIEW |DROP VIEW v0; |It drops a view. |subsequent different logic of querying data. | +|SQLite |VACUUM |VACUUM main; |It rebuilds the database file. |subsequent different logic of querying data. | +|SQLite |REINDEX |REINDEX t0; |It drops and recreates indexes from scratch. |subsequent different logic of querying data. | +|SQLite |ANALYZE |ANALYZE t0; |It gathers statistics about tables to help make better query planning choices.|subsequent different logic of querying data. | +|SQLite |EXPLAIN |EXPLAIN SELECT * FROM t0; |It obtains query plan of a query. |subsequent different logic of querying data. | +|SQLite |CHECK_RTREE_TABLE |SELECT rtreecheck('rt0'); |It runs an integrity check on a table. |subsequent different logic of querying data. | +|SQLite |VIRTUAL_TABLE_ACTION |INSERT INTO vt0(vt0) VALUES('rebuild'); |It changes the options of a virtual table. |subsequent different logic of querying data. | +|SQLite |MANIPULATE_STAT_TABLE|INSERT INTO sqlite_stat1 VALUES('rt0', 't1', '2'); |It changes the table that stores statistics of all tables. |subsequent different logic of querying data. | +|SQLite |TRANSACTION_START |BEGIN TRANSACTION; |All statements after this will not be committed. |subsequent different logic of querying data. | +|SQLite |ROLLBACK_TRANSACTION |ROLLBACK TRANSACTION; |All statements after last BEGIN are dropped. |subsequent different logic of querying data. | +|SQLite |COMMIT |COMMIT; |All statements after last BEGIN are committed |subsequent different logic of querying data. | +|TiDB |CREATE_TABLE |CREATE TABLE t1(c0 INT); |It adds a new table. |more possible execution logics on the table. | +|TiDB |CREATE_INDEX |CREATE INDEX i0 ON t0(c0(250) ASC) KEY_BLOCK_SIZE 1564693810209727437; |It adds a new index on a table. |subsequent differnt logic of querying data. | +|TiDB |VIEW_GENERATOR |CREATE VIEW v0(c0, c1) AS SELECT t1.c0, ((t1.c0)REGEXP('8')) FROM t1; |It adds a new view from existing tables. |more possible execution logics on the view. | +|TiDB |INSERT |INSERT INTO t0(c0) VALUES (-16387); |It inserts a new row to a table. |subsequent different logic of querying data. | +|TiDB |ALTER_TABLE |ALTER TABLE t1 ADD PRIMARY KEY(c0); |It changes the schema of a table. |more possible execution logics on the changed table. | +|TiDB |TRUNCATE |TRUNCATE t0; |It drops all rows of a table. |subsequent different logic of querying data. | +|TiDB |UPDATE |UPDATE t0 SET c0='S' WHERE t0.c0; |It updates specific data of a table. |subsequent different logic of querying data. | +|TiDB |DELETE |DELETE FROM t0 ORDER BY CAST(t0.c0 AS CHAR) DESC; |It deletes specific rows from a table. |subsequent different logic of querying data. | +|TiDB |SET |set @@tidb_max_chunk_size=8864; |It modifies database options. |different options that decide how to execute statements.| +|TiDB |ADMIN_CHECKSUM_TABLE |ADMIN CHECKSUM TABLE t0; |it calculate the checksum for a table. |subsequent different logic of querying data. | +|TiDB |ANALYZE_TABLE |ANALYZE TABLE t1 WITH 174 BUCKETS; |It gathers statistics about tables to help make better query planning choices.|subsequent different logic of querying data. | +|TiDB |DROP_TABLE |DROP TABLE t0; |it drops an table. |subsequent different logic of querying data. | +|TiDB |DROP_VIEW |DROP VIEW v0; |It drops a view. |subsequent different logic of querying data. | +|CockroachDB|CREATE_TABLE |CREATE TABLE t1 (c0 INT4, c1 VARBIT(44) UNIQUE DEFAULT (B'000'), CONSTRAINT "primary" PRIMARY KEY(c1 ASC, c0 ASC));|It adds a new table. |more possible execution logics on the table. | +|CockroachDB|CREATE_INDEX |CREATE INDEX ON t0(rowid); |It adds a new index on a table. |subsequent differnt logic of querying data. | +|CockroachDB|CREATE_VIEW |CREATE VIEW v0(c0) AS SELECT DISTINCT MIN(TIMETZ '1970-01-11T12:19:44') FROM t0; |It adds a new view from existing tables. |more possible execution logics on the view. | +|CockroachDB|CREATE_STATISTICS |CREATE STATISTICS s0 FROM t2; |It gathers statistics about tables to help make better query planning choices.|subsequent different logic of querying data. | +|CockroachDB|INSERT |INSERT INTO t1 (rowid, c0) VALUES(NULL, true) ON CONFLICT (c0) DO NOTHING ; |It inserts a new row to a table. |subsequent different logic of querying data. | +|CockroachDB|UPDATE |UPDATE t0@{FORCE_INDEX=t0_pkey} SET c0=t0.c0; |It updates specific data of a table. |subsequent different logic of querying data. | +|CockroachDB|SET_SESSION |SET SESSION BYTEA_OUTPUT=escape; |It changes session configurations. |different options that decide how to execute statements.| +|CockroachDB|SET_CLUSTER_SETTING |SET CLUSTER SETTING sql.query_cache.enabled=true; |It changes cluster configurations. |different options that decide how to execute statements.| +|CockroachDB|DELETE |DELETE from t0; |It deletes specific rows from a table. |subsequent different logic of querying data. | +|CockroachDB|TRUNCATE |TRUNCATE TABLE t1 CASCADE; |It drops all rows of a table. |subsequent different logic of querying data. | +|CockroachDB|DROP_TABLE |DROP TABLE t0; |it drops an table. |subsequent different logic of querying data. | +|CockroachDB|DROP_VIEW |DROP VIEW v0; |It drops a view. |subsequent different logic of querying data. | +|CockroachDB|COMMENT_ON |COMMENT ON INDEX t0_c0_key IS '|?'; |It changes schema of a table. |subsequent different logic of querying data. | +|CockroachDB|SHOW |SHOW LOCALITY; |It lists detailed information of active queries. |subsequent different logic of querying data. | +|CockroachDB|EXPLAIN |EXPLAIN SELECT * FROM t0; |It obtains query plan of a query. |subsequent different logic of querying data. | +|CockroachDB|SCRUB |EXPERIMENTAL SCRUB table t0; |It checks data corruption of a table. |subsequent different logic of querying data. | +|CockroachDB|SPLIT |ALTER INDEX t0@t0_c0_key SPLIT AT VALUES (NULL); |It changes the indexes. |subsequent different logic of querying data. | From 7ca7d36a395bc751c460419bc63f08499add3c4c Mon Sep 17 00:00:00 2001 From: Albert Zhang <72067599+albertZhangTJ@users.noreply.github.com> Date: Thu, 22 Dec 2022 22:43:24 +0800 Subject: [PATCH 0718/1284] Add regex support for ExpectedError (#672) * Add Regex support in ExpectedErrors * Add Regex support in ExpectedErrors * Add Regex support in ExpectedErrors * Add unit test cases for regex in ExpectedError * Revert "Add unit test cases for regex in ExpectedError" This reverts commit cc9fce209a5e59f76ed1f72f1857565f90849dc6. * Add unit test cases for regex in ExpectedError * Removed config file from PR * Removed config file from PR * Remove unused imports * Fix PMD violation * Remove usingXXX fields in ExpectedError * Remove test cases for usingXXX fields in ExpectedError * Remove test cases for usingXXX fields in ExpectedError Co-authored-by: Manuel Rigger --- src/sqlancer/common/query/ExpectedErrors.java | 53 ++++++++++++++----- test/sqlancer/TestExpectedErrors.java | 29 ++++++++-- 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index 20a7d97c8..1016969c9 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -1,8 +1,11 @@ package sqlancer.common.query; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.regex.Pattern; /** * This class represents the errors that executing a statement might result in. For example, an INSERT statement might @@ -12,6 +15,7 @@ public class ExpectedErrors { private final Set errors = new HashSet<>(); + private final List regexes = new ArrayList<>(); public ExpectedErrors add(String error) { if (error == null) { @@ -21,6 +25,35 @@ public ExpectedErrors add(String error) { return this; } + public ExpectedErrors addRegex(Pattern errorPattern) { + if (errorPattern == null) { + throw new IllegalArgumentException(); + } + regexes.add(errorPattern); + return this; + } + + public ExpectedErrors addAll(Collection list) { + errors.addAll(list); + return this; + } + + public ExpectedErrors addAllRegexes(Collection list) { + if (list == null) { + throw new IllegalArgumentException(); + } + regexes.addAll(list); + return this; + } + + public static ExpectedErrors from(String... errors) { + ExpectedErrors expectedErrors = new ExpectedErrors(); + for (String error : errors) { + expectedErrors.add(error); + } + return expectedErrors; + } + /** * Checks whether the error message (e.g., returned by the DBMS under test) contains any of the added error * messages. @@ -34,25 +67,17 @@ public boolean errorIsExpected(String error) { if (error == null) { throw new IllegalArgumentException(); } - for (String s : errors) { + for (String s : this.errors) { if (error.contains(s)) { return true; } } - return false; - } - - public ExpectedErrors addAll(Collection list) { - errors.addAll(list); - return this; - } - - public static ExpectedErrors from(String... errors) { - ExpectedErrors expectedErrors = new ExpectedErrors(); - for (String error : errors) { - expectedErrors.add(error); + for (Pattern p : this.regexes) { + if (p.matcher(error).find()) { + return true; + } } - return expectedErrors; + return false; } } diff --git a/test/sqlancer/TestExpectedErrors.java b/test/sqlancer/TestExpectedErrors.java index 69a0f53e6..2781df89b 100644 --- a/test/sqlancer/TestExpectedErrors.java +++ b/test/sqlancer/TestExpectedErrors.java @@ -1,5 +1,7 @@ package sqlancer; +import java.util.regex.Pattern; + import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -16,7 +18,7 @@ public void testEmpty() { } @Test - public void testSimple() { + public void testStringSimple() { ExpectedErrors errors = new ExpectedErrors(); errors.add("a"); errors.add("b"); @@ -25,16 +27,37 @@ public void testSimple() { assertTrue(errors.errorIsExpected("b")); assertTrue(errors.errorIsExpected("c")); assertTrue(errors.errorIsExpected("aa")); - assertFalse(errors.errorIsExpected("d")); + + } + + @Test + public void testRegexSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addRegex(Pattern.compile("a\\d")); + errors.addRegex(Pattern.compile("b\\D")); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } @Test - public void testRealistic() { + public void testStringRealistic() { ExpectedErrors errors = new ExpectedErrors(); errors.add("violated"); assertTrue(errors.errorIsExpected("UNIQUE constraint was violated!")); assertTrue(errors.errorIsExpected("PRIMARY KEY constraint was violated!")); } + @Test + public void testRegexRealistic() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addRegex(Pattern.compile(".violated.")); + assertTrue(errors.errorIsExpected("UNIQUE constraint was violated!")); + assertTrue(errors.errorIsExpected("PRIMARY KEY constraint was violated!")); + } + } From f41520312d8fba0dd31389b8d3d049cba5a54f38 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 24 Dec 2022 15:38:43 +0800 Subject: [PATCH 0719/1284] fix null pointers (#673) --- src/sqlancer/cockroachdb/CockroachDBProvider.java | 2 +- src/sqlancer/tidb/TiDBProvider.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 9bc72b614..c9e0d24a9 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -299,7 +299,7 @@ public String getQueryPlan(String selectStr, CockroachDBGlobalState globalState) e.printStackTrace(); } } - SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, null); + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery); boolean afterProjection = false; // Remove the concrete expression after each Projection operator try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs != null) { diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index fbb905eea..c4f13a207 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -183,7 +183,7 @@ public String getQueryPlan(String selectStr, TiDBGlobalState globalState) throws } } - SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN " + selectStr, null); + SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN " + selectStr); try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs != null) { while (rs.next()) { From 2a5ca983c4ded3b1042b405950637265f5388d12 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 24 Dec 2022 16:13:02 +0800 Subject: [PATCH 0720/1284] More tests for QPG (#674) * add tests for QPG * add actions for QPG --- .github/workflows/main.yml | 48 +++++++++++++++++- test/sqlancer/dbms/TestSQLiteQPG.java | 20 -------- .../qpg/cockroachdb/TestCockroachDBQPG.java | 24 +++++++++ .../cockroachdb/TestCockroachDBQueryPlan.java | 47 ++++++++++++++++++ test/sqlancer/qpg/sqlite/TestSQLiteQPG.java | 23 +++++++++ .../qpg/sqlite/TestSQLiteQueryPlan.java | 41 ++++++++++++++++ test/sqlancer/qpg/tidb/TestTiDBQPG.java | 24 +++++++++ test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java | 49 +++++++++++++++++++ 8 files changed, 255 insertions(+), 21 deletions(-) delete mode 100644 test/sqlancer/dbms/TestSQLiteQPG.java create mode 100644 test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java create mode 100644 test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java create mode 100644 test/sqlancer/qpg/sqlite/TestSQLiteQPG.java create mode 100644 test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java create mode 100644 test/sqlancer/qpg/tidb/TestTiDBQPG.java create mode 100644 test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b498033d..b810c925b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -116,7 +116,30 @@ jobs: run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - + + cockroachdb-qpg: + name: QPG Tests (CockroachDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up CockroachDB + run: | + wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.0-beta.4.linux-amd64.tgz | tar xvz + cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach start-single-node --insecure & + sleep 10 + - name: Create SQLancer user + run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + - name: Run Tests + run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test + databend: name: DBMS Tests (Databend) runs-on: ubuntu-latest @@ -313,6 +336,29 @@ jobs: - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test + tidb-qpg: + name: QPG Tests (TiDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up TiDB + run: | + docker pull pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + sleep 10 + - name: Create SQLancer user + run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + - name: Run Tests + run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test + yugabyte: name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest diff --git a/test/sqlancer/dbms/TestSQLiteQPG.java b/test/sqlancer/dbms/TestSQLiteQPG.java deleted file mode 100644 index 37be6f78d..000000000 --- a/test/sqlancer/dbms/TestSQLiteQPG.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestSQLiteQPG { - - @Test - public void testSqliteQPG() { - assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", "1000", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", - "--database-prefix", "pqsdb" /* Workaround for connections not being closed */, "--qpg-enable", "true", - "sqlite3", "--oracle", "NoREC", "--test-fts", "false", "--test-rtree", "false", - "--test-check-constraints", "false", "--test-in-operator", "false" })); - } - -} diff --git a/test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java new file mode 100644 index 000000000..b3ba57a10 --- /dev/null +++ b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java @@ -0,0 +1,24 @@ +package sqlancer.qpg.cockroachdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestCockroachDBQPG { + + @Test + public void testCockroachDBQPG() { + String cockroachDB = System.getenv("COCKROACHDB_AVAILABLE"); + boolean cockroachDBIsAvailable = cockroachDB != null && cockroachDB.equalsIgnoreCase("true"); + assumeTrue(cockroachDBIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, + "cockroachdb", "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java new file mode 100644 index 000000000..c858ee190 --- /dev/null +++ b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java @@ -0,0 +1,47 @@ +package sqlancer.qpg.cockroachdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.cockroachdb.CockroachDBOptions; +import sqlancer.cockroachdb.CockroachDBProvider; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; + +public class TestCockroachDBQueryPlan { + + @Test + void testCockroachDBQueryPlan() throws Exception { + String cockroachDB = System.getenv("COCKROACHDB_AVAILABLE"); + boolean cockroachDBIsAvailable = cockroachDB != null && cockroachDB.equalsIgnoreCase("true"); + assumeTrue(cockroachDBIsAvailable); + + String databaseName = "cockroachdb"; + CockroachDBProvider provider = new CockroachDBProvider(); + CockroachDBGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + CockroachDBOptions cockroachdbOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(cockroachdbOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals("left-join (cross);scan t2;select;scan t1;filters;filters (true);", queryPlan); + } + +} diff --git a/test/sqlancer/qpg/sqlite/TestSQLiteQPG.java b/test/sqlancer/qpg/sqlite/TestSQLiteQPG.java new file mode 100644 index 000000000..488037237 --- /dev/null +++ b/test/sqlancer/qpg/sqlite/TestSQLiteQPG.java @@ -0,0 +1,23 @@ +package sqlancer.qpg.sqlite; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestSQLiteQPG { + + @Test + public void testSqliteQPG() { + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", + "pqsdb" /* Workaround for connections not being closed */, "--qpg-enable", "true", "sqlite3", + "--oracle", "NoREC", "--test-fts", "false", "--test-rtree", "false", "--test-check-constraints", + "false", "--test-in-operator", "false" })); + } + +} diff --git a/test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java b/test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java new file mode 100644 index 000000000..7672f7edf --- /dev/null +++ b/test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java @@ -0,0 +1,41 @@ +package sqlancer.qpg.sqlite; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3Options; +import sqlancer.sqlite3.SQLite3Provider; + +public class TestSQLiteQueryPlan { + + @Test + void testSQLiteQueryPlan() throws Exception { + String databaseName = "sqlite"; + SQLite3Provider provider = new SQLite3Provider(); + SQLite3GlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + SQLite3Options sqlite3Option = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(sqlite3Option); + state.setDatabaseName(databaseName); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals("SCAN t1;SCAN t2;RIGHT-JOIN t2;SCAN t2;", queryPlan); + } + +} diff --git a/test/sqlancer/qpg/tidb/TestTiDBQPG.java b/test/sqlancer/qpg/tidb/TestTiDBQPG.java new file mode 100644 index 000000000..66a43a41a --- /dev/null +++ b/test/sqlancer/qpg/tidb/TestTiDBQPG.java @@ -0,0 +1,24 @@ +package sqlancer.qpg.tidb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestTiDBQPG { + + @Test + public void testTiDBQPG() { + String tiDB = System.getenv("TIDB_AVAILABLE"); + boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); + assumeTrue(tiDBIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "tidb", + "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java b/test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java new file mode 100644 index 000000000..d07bebe61 --- /dev/null +++ b/test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java @@ -0,0 +1,49 @@ +package sqlancer.qpg.tidb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBOptions; +import sqlancer.tidb.TiDBProvider; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; + +public class TestTiDBQueryPlan { + + @Test + void testTiDBQueryPlan() throws Exception { + String tiDB = System.getenv("TIDB_AVAILABLE"); + boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); + assumeTrue(tiDBIsAvailable); + + String databaseName = "tidb"; + TiDBProvider provider = new TiDBProvider(); + TiDBGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + TiDBOptions TiDBOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(TiDBOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals( + "HashJoin_7;TableReader_10(Build);Selection_9;TableFullScan_8;TableReader_12(Probe);TableFullScan_11;", + queryPlan); + } + +} From 7bd657d52af8d04a530f8b8671efa968749603a0 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 3 Jan 2023 07:36:30 +0800 Subject: [PATCH 0721/1284] Update CockroachDB version in Action (#675) --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b810c925b..2cf55af4e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,11 +109,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.0-beta.4.linux-amd64.tgz | tar xvz - cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-alpha.1.linux-amd64.tgz | tar xvz + cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -132,11 +132,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.0-beta.4.linux-amd64.tgz | tar xvz - cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-alpha.1.linux-amd64.tgz | tar xvz + cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test From 91c4f9baa8c11eaa0a3b305eaae3ea1646486b65 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 4 Jan 2023 19:59:08 +0800 Subject: [PATCH 0722/1284] tidb: support USE_INDEX_MERGE(TableName) Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBHintGenerator.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index 06105ed4c..b1a343c67 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -96,7 +96,15 @@ private void generate() { twoTablesHint("SHUFFLE_JOIN", table); break; case USE_INDEX_MERGE: - twoTablesHint("USE_INDEX_MERGE", table); + if (Randomly.getBoolean()) { + if (table.hasIndexes()) { + tablesHint("USE_INDEX_MERGE"); + } else { + throw new IgnoreMeException(); + } + } else { + twoTablesHint("USE_INDEX_MERGE", table); + } break; case NO_INDEX_MERGE: sb.append("NO_INDEX_MERGE()"); From 06677bcf2f967d9c299622968c34c07e504c711d Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 5 Jan 2023 00:24:26 +0800 Subject: [PATCH 0723/1284] tidb: ignore fast ananlyze error (#683) * tidb: ignore fast ananlyze error Signed-off-by: Weizhen Wang * tidb: ignore fast ananlyze error Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index d84b117c2..8bcea30fb 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -13,6 +14,7 @@ private TiDBAnalyzeTableGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { + ExpectedErrors errors = new ExpectedErrors(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); StringBuilder sb = new StringBuilder("ANALYZE "); @@ -30,7 +32,8 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(Randomly.getNotCachedInteger(1, 1024)); sb.append(" BUCKETS"); } - return new SQLQueryAdapter(sb.toString()); + errors.add("Fast analyze hasn't reached General Availability and only support analyze version 1 currently"); + return new SQLQueryAdapter(sb.toString(), errors); } } From 7f1efbd4ba672f387ea2034b07bcbaf3b23e69d2 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 4 Jan 2023 19:49:19 +0100 Subject: [PATCH 0724/1284] Rewrite ClickHouse query generation (#681) * Rewrite ClickHouse query generation * Minor fix that triggered false positive * rename * Fix formatting --- src/sqlancer/clickhouse/ClickHouseErrors.java | 10 ++ .../clickhouse/ast/ClickHouseAggregate.java | 11 +- .../gen/ClickHouseColumnBuilder.java | 11 +- .../gen/ClickHouseExpressionGenerator.java | 144 +++++++++++++----- .../gen/ClickHouseTableGenerator.java | 12 +- .../oracle/norec/ClickHouseNoRECOracle.java | 79 +++++----- .../tlp/ClickHouseTLPAggregateOracle.java | 59 +++---- .../oracle/tlp/ClickHouseTLPBase.java | 21 +-- .../tlp/ClickHouseTLPGroupByOracle.java | 9 +- .../oracle/tlp/ClickHouseTLPHavingOracle.java | 40 ++--- .../oracle/tlp/ClickHouseTLPWhereOracle.java | 9 +- .../ast/ClickHouseToStringVisitorTest.java | 3 +- test/sqlancer/dbms/TestClickHouse.java | 97 ++++++++++-- 13 files changed, 323 insertions(+), 182 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index d2b6c700a..0548a04e9 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -62,7 +62,17 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("ILLEGAL_DIVISION"); errors.add("DECIMAL_OVERFLOW"); errors.add("Cannot convert out of range floating point value to integer type"); + errors.add("Unexpected inf or nan to integer conversion"); + errors.add("No such name in Block::erase"); // https://github.com/ClickHouse/ClickHouse/issues/42769 errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/42768 + errors.add("cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)"); + errors.add("AMBIGUOUS_IDENTIFIER"); + errors.add("CYCLIC_ALIASES"); + errors.add("Positional argument numeric constant expression is not representable as"); + errors.add("Positional argument must be constant with numeric type"); + errors.add(" is out of bounds. Expected in range"); + errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); + errors.add("Different order of columns in UNION subquery"); // https://github.com/ClickHouse/ClickHouse/issues/44866 } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java index e868f82ca..a0483bd57 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java @@ -12,13 +12,12 @@ public class ClickHouseAggregate extends ClickHouseExpression { private final ClickHouseAggregate.ClickHouseAggregateFunction func; - private final List expr; + private final ClickHouseExpression expr; public enum ClickHouseAggregateFunction { AVG(ClickHouseDataType.Int8, ClickHouseDataType.Int16, ClickHouseDataType.Int32, ClickHouseDataType.Int64, ClickHouseDataType.UInt8, ClickHouseDataType.UInt16, ClickHouseDataType.UInt32, ClickHouseDataType.UInt64, ClickHouseDataType.Float32, ClickHouseDataType.Float64), - BOOL_AND(ClickHouseDataType.UInt8), BOOL_OR(ClickHouseDataType.UInt8), COUNT(ClickHouseDataType.Int8, ClickHouseDataType.Int16, ClickHouseDataType.Int32, ClickHouseDataType.Int64, ClickHouseDataType.UInt8, ClickHouseDataType.UInt16, ClickHouseDataType.UInt32, ClickHouseDataType.UInt64, ClickHouseDataType.Float32, ClickHouseDataType.Float64, @@ -42,8 +41,8 @@ public static ClickHouseAggregateFunction getRandom(ClickHouseDataType type) { return Randomly.fromOptions(values()); } - public List getTypes(ClickHouseDataType returnType) { - return Arrays.asList(returnType); + public ClickHouseDataType getType(ClickHouseDataType returnType) { + return returnType; } public boolean supportsReturnType(ClickHouseDataType returnType) { @@ -66,7 +65,7 @@ public ClickHouseSchema.ClickHouseLancerDataType getRandomReturnType() { } - public ClickHouseAggregate(List expr, ClickHouseAggregateFunction func) { + public ClickHouseAggregate(ClickHouseExpression expr, ClickHouseAggregateFunction func) { this.expr = expr; this.func = func; } @@ -75,7 +74,7 @@ public ClickHouseAggregate.ClickHouseAggregateFunction getFunc() { return func; } - public List getExpr() { + public ClickHouseExpression getExpr() { return expr; } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java index 65eb65623..2584ff0b3 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java @@ -61,12 +61,11 @@ public String createColumn(String columnName, ClickHouseProvider.ClickHouseGloba if (allowMaterialized) { sb.append(" MATERIALIZED ("); sb.append( - ClickHouseVisitor - .asString(new ClickHouseExpressionGenerator(globalState) - .setColumns( - columns.stream().filter(p -> !p.getName().contentEquals(columnName)) - .collect(Collectors.toList())) - .generateExpression(dataType))); + ClickHouseVisitor.asString( + new ClickHouseExpressionGenerator(globalState).generateExpressionWithColumns( + columns.stream().filter(p -> !p.getName().contentEquals(columnName)) + .map(p -> p.asColumnReference(null)).collect(Collectors.toList()), + 2))); sb.append(")"); } break; diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index cd8ad8ff3..03750fee0 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -53,48 +53,134 @@ private enum Expression { UNARY_POSTFIX } - public ClickHouseExpression generateExpressionWithColumns(List columns, int depth) { - if (columns.isEmpty() || depth > 0 && Randomly.getBoolean()) { + public ClickHouseExpression generateExpressionWithColumns(List columns, + int remainingDepth) { + if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { return generateConstant(null); } - if (depth >= 2 || Randomly.getBoolean()) { + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { return columns.get((int) Randomly.getNotCachedInteger(0, columns.size() - 1)); } ColumnLike expr = Randomly.fromOptions(ColumnLike.values()); switch (expr) { case UNARY_PREFIX: - return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, depth + 1), + return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, remainingDepth - 1), ClickHouseUnaryPrefixOperator.MINUS); case BINARY_ARITHMETIC: - return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, depth + 1), - generateExpressionWithColumns(columns, depth + 1), + return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); case UNARY_FUNCTION: - return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, depth + 1), + return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); case BINARY_FUNCTION: - return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, depth + 1), - generateExpressionWithColumns(columns, depth + 1), + return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); default: throw new AssertionError(expr); } } + public ClickHouseExpression generateAggregateExpressionWithColumns(List columns, + int remainingDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return new ClickHouseAggregate(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseAggregate.ClickHouseAggregateFunction.getRandom()); + } + if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(null); + } + + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + return columns.get((int) Randomly.getNotCachedInteger(0, columns.size() - 1)); + } + + ColumnLike expr = Randomly.fromOptions(ColumnLike.values()); + switch (expr) { + case UNARY_PREFIX: + return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseUnaryPrefixOperator.MINUS); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); + default: + throw new AssertionError(expr); + } + } + + public ClickHouseExpression generateExpressionWithExpression(List expression, + int remainingDepth) { + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + if (Randomly.getBoolean()) { + return expression.get((int) Randomly.getNotCachedInteger(0, expression.size() - 1)); + } else { + return generateConstant(null); + } + } + + Expression type = Randomly.fromOptions(Expression.values()); + switch (type) { + case UNARY_PREFIX: + return new ClickHouseUnaryPrefixOperation(generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new ClickHouseUnaryPostfixOperation(generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.getRandom(), false); + case BINARY_COMPARISON: + return new ClickHouseBinaryComparisonOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.getRandomOperator()); + case BINARY_LOGICAL: + return new ClickHouseBinaryLogicalOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryLogicalOperation.ClickHouseBinaryLogicalOperator.getRandom()); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); + default: + throw new AssertionError(type); + } + } + @Override protected ClickHouseExpression generateExpression(ClickHouseLancerDataType type, int depth) { - if (allowAggregateFunctions && Randomly.getBoolean()) { - return generateAggregate(); + if (allowAggregateFunctions && Randomly.getBooleanWithRatherLowProbability()) { + ClickHouseLancerDataType aggType = ClickHouseLancerDataType.getRandom(); + return new ClickHouseAggregate(generateExpression(aggType, depth + 1), + ClickHouseAggregate.ClickHouseAggregateFunction.getRandom()); } - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { return generateLeafNode(type); } Expression expr = Randomly.fromOptions(Expression.values()); ClickHouseLancerDataType leftLeafType = ClickHouseLancerDataType.getRandom(); ClickHouseLancerDataType rightLeafType = ClickHouseLancerDataType.getRandom(); - if (Randomly.getBoolean()) { + if (Randomly.getBooleanWithRatherLowProbability()) { rightLeafType = leftLeafType; } @@ -133,8 +219,8 @@ protected ClickHouseExpression.ClickHouseJoinOnClause generateJoinClause(ClickHo ClickHouseTableReference rightTable) { List leftColumns = leftTable.getColumnReferences(); List rightColumns = rightTable.getColumnReferences(); - ClickHouseExpression leftExpr = generateExpressionWithColumns(leftColumns, 0); - ClickHouseExpression rightExpr = generateExpressionWithColumns(rightColumns, 0); + ClickHouseExpression leftExpr = generateExpressionWithColumns(leftColumns, 2); + ClickHouseExpression rightExpr = generateExpressionWithColumns(rightColumns, 2); return new ClickHouseExpression.ClickHouseJoinOnClause(leftExpr, rightExpr); } @@ -223,20 +309,17 @@ public ClickHouseExpression generateConstant(ClickHouseLancerDataType genType) { } public ClickHouseExpression getHavingClause() { - allowAggregateFunctions = true; - return generateExpression(new ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + return generateAggregate(); } public ClickHouseAggregate generateArgsForAggregate(ClickHouseDataType dataType, ClickHouseAggregate.ClickHouseAggregateFunction agg) { - List types = agg.getTypes(dataType); - List args = new ArrayList<>(); - for (ClickHouseDataType argType : types) { - this.allowAggregateFunctions = false; - args.add(generateExpression(new ClickHouseLancerDataType(argType))); - this.allowAggregateFunctions = true; - } - return new ClickHouseAggregate(args, agg); + ClickHouseDataType type = agg.getType(dataType); + this.allowAggregateFunctions = false; + ClickHouseExpression arg = generateExpression(new ClickHouseLancerDataType(type)); + this.allowAggregateFunctions = true; + + return new ClickHouseAggregate(arg, agg); } public ClickHouseExpressionGenerator allowAggregates(boolean value) { @@ -245,19 +328,12 @@ public ClickHouseExpressionGenerator allowAggregates(boolean value) { } public ClickHouseExpression generateAggregate() { - return getAggregate(ClickHouseLancerDataType.getRandom().getType()); - } - - private ClickHouseExpression getAggregate(ClickHouseDataType dataType) { - List aggregates = ClickHouseAggregate.ClickHouseAggregateFunction - .getAggregates(dataType); - ClickHouseAggregate.ClickHouseAggregateFunction agg = Randomly.fromList(aggregates); - return generateArgsForAggregate(dataType, agg); + return generateAggregateExpressionWithColumns(columnRefs, 3); } @Override public ClickHouseExpression generatePredicate() { - return generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + return generateExpressionWithColumns(columnRefs, 3); } @Override diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index f3e5baecc..19ce78d27 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -4,8 +4,6 @@ import java.util.List; import java.util.stream.Collectors; -import com.clickhouse.client.ClickHouseDataType; - import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; @@ -82,7 +80,7 @@ public void start() { if (Randomly.getBoolean()) { sb.append(" ORDER BY "); ClickHouseExpression expr = gen.generateExpressionWithColumns( - columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); sb.append(ClickHouseToStringVisitor.asString(expr)); } else { sb.append(" ORDER BY tuple() "); @@ -91,13 +89,13 @@ public void start() { if (Randomly.getBoolean()) { sb.append(" PARTITION BY "); ClickHouseExpression expr = gen.generateExpressionWithColumns( - columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); sb.append(ClickHouseToStringVisitor.asString(expr)); } if (Randomly.getBoolean()) { sb.append(" SAMPLE BY "); ClickHouseExpression expr = gen.generateExpressionWithColumns( - columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); sb.append(ClickHouseToStringVisitor.asString(expr)); } // TODO: PRIMARY KEY @@ -111,8 +109,8 @@ private void addColumnsConstraint(ClickHouseExpressionGenerator gen) { sb.append(" CONSTRAINT "); sb.append(ClickHouseCommon.createConstraintName(i)); sb.append(" CHECK "); - ClickHouseExpression expr = gen - .generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 2); sb.append(ClickHouseToStringVisitor.asString(expr)); } } diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java index cda529cbe..9818d198b 100644 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -1,24 +1,20 @@ package sqlancer.clickhouse.oracle.norec; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.SQLConnection; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; +import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; import sqlancer.clickhouse.ast.ClickHouseTableReference; @@ -31,27 +27,33 @@ public class ClickHouseNoRECOracle extends NoRECBase implements TestOracle { - private final ClickHouseSchema s; + private final ClickHouseSchema schema; public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { super(globalState); - this.s = globalState.getSchema(); + this.schema = globalState.getSchema(); ClickHouseErrors.addExpectedExpressionErrors(errors); } @Override public void check() throws SQLException { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state); + List tables = schema.getRandomTableNonEmptyTables().getTables(); + ClickHouseTableReference table = new ClickHouseTableReference( + tables.get((int) Randomly.getNotCachedInteger(0, tables.size() - 1)), "left"); + List columns = table.getColumnReferences(); - ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); - ClickHouseTable leftTable = randomTables.getTables().remove(0); - ClickHouseTableReference table = new ClickHouseTableReference(leftTable, "left"); - List columns = randomTables.getColumns(); - columns.addAll(leftTable.getColumns()); - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); - ClickHouseExpression randomWhereCondition = gen.generateExpression(ClickHouseLancerDataType.getRandom()); - List joins = gen.getRandomJoinClauses(table, randomTables.getTables()); - int secondCount = getSecondQuery(table, randomWhereCondition, joins); - int firstCount = getFirstQueryCount(con, table, columns, randomWhereCondition, joins); + List joinStatements = new ArrayList<>(); + if (state.getClickHouseOptions().testJoins && Randomly.getBoolean()) { + joinStatements = gen.getRandomJoinClauses(table, tables); + columns.addAll(joinStatements.stream().flatMap(j -> j.getRightTable().getColumnReferences().stream()) + .collect(Collectors.toList())); + } + gen.addColumns(columns); + + ClickHouseExpression randomWhereCondition = gen.generateExpressionWithColumns(columns, 5); + int secondCount = getSecondQuery(table, randomWhereCondition, joinStatements); + int firstCount = getFirstQueryCount(table, columns, randomWhereCondition, joinStatements); if (firstCount == -1 || secondCount == -1) { throw new IgnoreMeException(); } @@ -91,34 +93,35 @@ private int getSecondQuery(ClickHouseExpression table, ClickHouseExpression wher return secondCount; } - private int getFirstQueryCount(SQLConnection con, ClickHouseExpression tableList, List columns, + private int getFirstQueryCount(ClickHouseExpression tableList, List columns, ClickHouseExpression randomWhereCondition, List joins) throws SQLException { ClickHouseSelect select = new ClickHouseSelect(); - List allColumns = columns.stream().map(c -> c.asColumnReference(null)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); + List filteredColumns = Randomly.extractNrRandomColumns(columns, + (int) Randomly.getNotCachedInteger(1, columns.size())); + select.setFetchColumns( + filteredColumns.stream().map(c -> (ClickHouseExpression) c).collect(Collectors.toList())); select.setFromClause(tableList); select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( - new ClickHouseExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } select.setJoinClauses(joins); int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = ClickHouseToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); + optimizedQueryString = ClickHouseToStringVisitor.asString(select); + SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); + SQLancerResultSet rs; + + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(optimizedQueryString, e); + } + if (rs == null) { + return -1; } + + while (rs.next()) { + firstCount++; + } + rs.close(); return firstCount; } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 2646878a2..d14356a57 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -3,23 +3,16 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.List; - -import com.clickhouse.client.ClickHouseDataType; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; -import sqlancer.clickhouse.ast.ClickHouseExpression; -import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.ast.ClickHouseTableReference; -import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; -import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; public class ClickHouseTLPAggregateOracle extends ClickHouseTLPBase { @@ -30,55 +23,41 @@ public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState sta @Override public void check() throws SQLException { - ClickHouseSchema s = state.getSchema(); - ClickHouseSchema.ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); - ClickHouseSchema.ClickHouseTable table = randomTables.getTables().remove(0); - ClickHouseTableReference tableRef = new ClickHouseTableReference(table, table.getName()); - List columns = randomTables.getColumns(); - columns.addAll(table.getColumns()); - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); - List joins = gen.getRandomJoinClauses(tableRef, randomTables.getTables()); + super.check(); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); + } - gen = new ClickHouseExpressionGenerator(state).setColumns(randomTables.getColumns()); - ClickHouseSelect select = new ClickHouseSelect(); ClickHouseAggregate.ClickHouseAggregateFunction windowFunction = Randomly.fromOptions( ClickHouseAggregate.ClickHouseAggregateFunction.MIN, ClickHouseAggregate.ClickHouseAggregateFunction.MAX, ClickHouseAggregate.ClickHouseAggregateFunction.SUM); - ClickHouseAggregate aggregate = new ClickHouseAggregate( - gen.generateExpressions(ClickHouseSchema.ClickHouseLancerDataType.getRandom(), 1), windowFunction); + + ClickHouseAggregate aggregate = new ClickHouseAggregate(gen.generateExpressionWithColumns(columns, 6), + windowFunction); select.setFetchColumns(Arrays.asList(aggregate)); - ClickHouseExpression from = tableRef; - select.setFromClause(from); - select.setJoinClauses(joins); - if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); - } + String originalQuery = ClickHouseVisitor.asString(select); originalQuery += " SETTINGS aggregate_functions_null_for_empty = 1"; - ClickHouseExpression whereClause = gen - .generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); - ClickHouseUnaryPrefixOperation negatedClause = new ClickHouseUnaryPrefixOperation(whereClause, - ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.NOT); - ClickHouseUnaryPostfixOperation notNullClause = new ClickHouseUnaryPostfixOperation(whereClause, - ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.IS_NULL, false); - select.setFetchColumns(Arrays.asList(new ClickHouseAliasOperation(aggregate, "aggr"))); - select.setFromClause(from); - select.setWhereClause(whereClause); + + select.setWhereClause(predicate); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setGroupByClause(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setGroupByClause(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } String metamorphicText = "SELECT " + aggregate.getFunc().toString() + "(aggr) FROM ("; metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; - select.setWhereClause(negatedClause); + select.setWhereClause(negatedPredicate); metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; - select.setWhereClause(notNullClause); + select.setWhereClause(isNullPredicate); metamorphicText += ClickHouseVisitor.asString(select); metamorphicText += ")"; metamorphicText += " SETTINGS aggregate_functions_null_for_empty = 1"; diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 690994ec0..9374442ff 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -13,7 +13,6 @@ import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; @@ -29,7 +28,7 @@ public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase { ClickHouseSchema schema; - ClickHouseTables targetTables; + List columns; ClickHouseExpressionGenerator gen; ClickHouseSelect select; @@ -42,26 +41,30 @@ public ClickHouseTLPBase(ClickHouseGlobalState state) { public void check() throws SQLException { gen = new ClickHouseExpressionGenerator(state); schema = state.getSchema(); - initializeTernaryPredicateVariants(); select = new ClickHouseSelect(); List tables = schema.getRandomTableNonEmptyTables().getTables(); ClickHouseTableReference table = new ClickHouseTableReference( tables.get((int) Randomly.getNotCachedInteger(0, tables.size())), Randomly.getBoolean() ? "left" : null); select.setFromClause(table); - List columns = table.getColumnReferences(); + columns = table.getColumnReferences(); if (state.getClickHouseOptions().testJoins && Randomly.getBoolean()) { List joinStatements = gen.getRandomJoinClauses(table, tables); - select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); + columns.addAll(joinStatements.stream().flatMap(j -> j.getRightTable().getColumnReferences().stream()) + .collect(Collectors.toList())); + select.setJoinClauses(joinStatements); } gen.addColumns(columns); - range(0, Randomly.smallNumber()).mapToObj(i -> gen.generateExpressionWithColumns(columns, 0)) - .collect(Collectors.toList()); - select.setFetchColumns(generateFetchColumns(columns)); + int small = Randomly.smallNumber(); + List from = range(0, 1 + small) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList()); + select.setFetchColumns(from); select.setWhereClause(null); + initializeTernaryPredicateVariants(); // Smoke check - ComparatorHelper.getResultSetFirstColumnAsString(ClickHouseVisitor.asString(select), errors, state); + String query = ClickHouseVisitor.asString(select); + ComparatorHelper.getResultSetFirstColumnAsString(query, errors, state); } List generateFetchColumns(List columns) { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index 8aff8fd77..25bfff57f 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -3,10 +3,14 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.ComparatorHelper; +import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.ClickHouseExpression; public class ClickHouseTLPGroupByOracle extends ClickHouseTLPBase { @@ -17,7 +21,10 @@ public ClickHouseTLPGroupByOracle(ClickHouseProvider.ClickHouseGlobalState state @Override public void check() throws SQLException { super.check(); - select.setGroupByClause(select.getFetchColumns()); + List groupByColumns = IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList()); + + select.setGroupByClause(groupByColumns); select.setWhereClause(null); String originalQueryString = ClickHouseVisitor.asString(select); diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index d71e17624..a6f3e0cdb 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -11,15 +11,12 @@ import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; +import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; public class ClickHouseTLPHavingOracle extends ClickHouseTLPBase { @@ -30,33 +27,15 @@ public ClickHouseTLPHavingOracle(ClickHouseProvider.ClickHouseGlobalState state) @Override public void check() throws SQLException { - ClickHouseSchema s = state.getSchema(); - ClickHouseSchema.ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); - ClickHouseSchema.ClickHouseTable table = randomTables.getTables().remove(0); - ClickHouseTableReference tableRef = new ClickHouseTableReference(table, table.getName()); - List columns = randomTables.getColumns(); - columns.addAll(table.getColumns()); - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); - List joins = gen.getRandomJoinClauses(tableRef, randomTables.getTables()); - List colRefs = joins.stream() - .flatMap(j -> j.getRightTable().getColumnReferences().stream()).collect(Collectors.toList()); - - ClickHouseExpressionGenerator aggrGen = new ClickHouseExpressionGenerator(state).allowAggregates(true) - .setColumns(columns); - ClickHouseSelect select = new ClickHouseSelect(); - select.setFetchColumns(aggrGen.generateExpressions(Randomly.smallNumber() + 1)); - ClickHouseTableReference from = new ClickHouseTableReference(table, table.getName()); - select.setJoinClauses(joins); + super.check(); + select.setFetchColumns(IntStream.range(0, Randomly.smallNumber() + 1) + .mapToObj(i -> gen.generateAggregateExpressionWithColumns(columns, 3)).collect(Collectors.toList())); select.setSelectType(ClickHouseSelect.SelectType.ALL); - select.setFromClause(from); // TODO order by? - List groupByColumns = IntStream.range(0, Randomly.smallNumber()) - .mapToObj(i -> gen.generateExpressionWithColumns(colRefs, 0)).collect(Collectors.toList()); + List groupByColumns = IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 6)).collect(Collectors.toList()); - if (groupByColumns.isEmpty()) { - throw new IgnoreMeException(); - } select.setGroupByClause(groupByColumns); select.setHavingClause(null); String originalQueryString = ClickHouseVisitor.asString(select); @@ -64,7 +43,12 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - ClickHouseExpression predicate = aggrGen.getHavingClause(); + List aggregateExprs = select.getFetchColumns().stream() + .filter(p -> p instanceof ClickHouseAggregate).collect(Collectors.toList()); + if (aggregateExprs.isEmpty()) { + throw new IgnoreMeException(); + } + ClickHouseExpression predicate = gen.generateExpressionWithExpression(aggregateExprs, 6); select.setHavingClause(predicate); String firstQueryString = ClickHouseVisitor.asString(select); select.setHavingClause(new ClickHouseUnaryPrefixOperation(predicate, diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java index 1976ca957..92195bedf 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java @@ -3,6 +3,8 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.ComparatorHelper; import sqlancer.Randomly; @@ -21,15 +23,18 @@ public ClickHouseTLPWhereOracle(ClickHouseProvider.ClickHouseGlobalState state) public void check() throws SQLException { super.check(); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } String originalQueryString = ClickHouseVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } + select.setWhereClause(predicate); String firstQueryString = ClickHouseVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java index 7231aced8..ba9628037 100644 --- a/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java +++ b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java @@ -164,8 +164,7 @@ void selectSumAGroupByBTest() { ClickHouseColumnReference a_ref = a_col.asColumnReference(null); ClickHouseColumnReference b_ref = b_col.asColumnReference(null); ClickHouseSelect selectAB = new ClickHouseSelect(); - ClickHouseAggregate sum_a = new ClickHouseAggregate(Arrays.asList(a_ref), - ClickHouseAggregate.ClickHouseAggregateFunction.SUM); + ClickHouseAggregate sum_a = new ClickHouseAggregate(a_ref, ClickHouseAggregate.ClickHouseAggregateFunction.SUM); selectAB.setFetchColumns(Arrays.asList(sum_a)); selectAB.setFromClause(table_ref); selectAB.setGroupByClause(Arrays.asList(b_ref)); diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 073fdcbd3..ae861dfd9 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -16,8 +16,8 @@ public void testClickHouseTLPWhereGroupBy() { assumeTrue(clickHouseIsAvailable); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPWhere", - "--oracle", "TLPGroupBy")); + "5", "--username", "default", "--password", "", "--database-prefix", "T1_", "clickhouse", + "--oracle", "TLPWhere", "--oracle", "TLPGroupBy")); } @Test @@ -25,8 +25,10 @@ public void testClickHouseTLPWhere() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, - "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPWhere")); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T2_", "clickhouse", + "--oracle", "TLPWhere")); } @Test @@ -37,7 +39,7 @@ public void testClickHouseTLPHaving() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", - "--password", "", "clickhouse", "--oracle", "TLPHaving")); + "--password", "", "--database-prefix", "T3_", "clickhouse", "--oracle", "TLPHaving")); } @Test @@ -48,7 +50,7 @@ public void testClickHouseTLPGroupBy() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", - "--password", "", "clickhouse", "--oracle", "TLPGroupBy")); + "--password", "", "--database-prefix", "T4_", "clickhouse", "--oracle", "TLPGroupBy")); } @Test @@ -59,7 +61,7 @@ public void testClickHouseTLPDistinct() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", - "--password", "", "clickhouse", "--oracle", "TLPDistinct")); + "--password", "", "--database-prefix", "T5_", "clickhouse", "--oracle", "TLPDistinct")); } @Test @@ -70,7 +72,7 @@ public void testClickHouseTLPAggregate() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", - "--password", "", "clickhouse", "--oracle", "TLPAggregate")); + "--password", "", "--database-prefix", "T6_", "clickhouse", "--oracle", "TLPAggregate")); } @Test @@ -81,7 +83,84 @@ public void testClickHouseNoREC() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", - "--password", "", "clickhouse", "--oracle", "NoREC")); + "--password", "", "--database-prefix", "T7_", "clickhouse", "--oracle", "NoREC")); + } + + @Test + public void testClickHouseTLPWhereGroupByWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T8_", "clickhouse", + "--oracle", "TLPWhere", "--oracle", "TLPGroupBy")); + } + + @Test + public void testClickHouseTLPWhereWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T9_", "clickhouse", + "--oracle", "TLPWhere")); + } + + @Test + public void testClickHouseTLPHavingWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "--database-prefix", "T10_", "clickhouse", "--oracle", "TLPHaving")); + } + + @Test + public void testClickHouseTLPGroupByWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T11_", "clickhouse", "--oracle", "TLPGroupBy")); + } + + @Test + public void testClickHouseTLPDistinctWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T12_", "clickhouse", "--oracle", "TLPDistinct")); + } + + @Test + public void testClickHouseTLPAggregateWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T13_", "clickhouse", "--oracle", "TLPAggregate")); + } + + @Test + public void testClickHouseNoRECWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "--database-prefix", "T14_", "clickhouse", "--oracle", "NoREC")); } } From 427d6a45f74973afd3f34d7044bff17a5c52524c Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Fri, 6 Jan 2023 23:11:13 +0100 Subject: [PATCH 0725/1284] Disable SEMI JOIN in ClickHouse (#685) * Disable SEMI JOIN * fix * disable NATURAL --- src/sqlancer/clickhouse/ClickHouseErrors.java | 1 + src/sqlancer/clickhouse/ClickHouseToStringVisitor.java | 6 ------ src/sqlancer/clickhouse/ast/ClickHouseExpression.java | 4 +++- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 0548a04e9..2e03b17fd 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -73,6 +73,7 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add(" is out of bounds. Expected in range"); errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); errors.add("Different order of columns in UNION subquery"); // https://github.com/ClickHouse/ClickHouse/issues/44866 + errors.add("Unexpected inf or nan to integer conversion"); } } diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index cbc622560..37f4d3ab6 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -151,12 +151,6 @@ public void visit(ClickHouseExpression.ClickHouseJoin join) { } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.FULL_OUTER) { sb.append(" FULL OUTER JOIN "); visit(join.getRightTable()); - } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_SEMI) { - sb.append(" LEFT SEMI JOIN "); - visit(join.getRightTable()); - } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_SEMI) { - sb.append(" RIGHT SEMI JOIN "); - visit(join.getRightTable()); } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_ANTI) { sb.append(" LEFT ANTI JOIN "); visit(join.getRightTable()); diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index 34eda72a5..f5cd2ed1e 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -63,8 +63,10 @@ public String getOperatorRepresentation() { public static class ClickHouseJoin extends ClickHouseExpression { // TODO: support ANY, ALL, ASOF modifiers + // LEFT_SEMI, RIGHT_SEMI are not deterministic as ClickHouse allows to read columns from + // whitelist table as well public enum JoinType { - INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, RIGHT_SEMI, LEFT_ANTI, RIGHT_ANTI; + INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_ANTI, RIGHT_ANTI; } private final ClickHouseTableReference leftTable; From be4d62f8efce0525f627eb71ed5569dd9368f7fe Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:10:18 +0100 Subject: [PATCH 0726/1284] Fix comparator for GroupBy Oracle in ClickHouse (#691) --- .../clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index 25bfff57f..e8881cb84 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -37,8 +37,8 @@ public void check() throws SQLException { select.setWhereClause(isNullPredicate); String thirdQueryString = ClickHouseVisitor.asString(select); List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, - secondQueryString, thirdQueryString, combinedString, false, state, errors); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } From 834f646f428e4a937d5cf3d6b75b686bdec80bc7 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Fri, 13 Jan 2023 13:47:54 +0100 Subject: [PATCH 0727/1284] Update ClickHouse expected errors (#687) * Update errors * Update ClickHouseErrors.java --- src/sqlancer/clickhouse/ClickHouseErrors.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 2e03b17fd..5a115ebd5 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -38,7 +38,6 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Sorting key cannot contain constants"); errors.add("There is no supertype for types"); errors.add("argument of function"); - errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 errors.add("but its arguments considered equal according to constraints"); errors.add("does not return a value of type UInt8"); errors.add("does not return a value of type UInt8"); @@ -64,7 +63,7 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Cannot convert out of range floating point value to integer type"); errors.add("Unexpected inf or nan to integer conversion"); errors.add("No such name in Block::erase"); // https://github.com/ClickHouse/ClickHouse/issues/42769 - errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/42768 + errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/43003 errors.add("cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)"); errors.add("AMBIGUOUS_IDENTIFIER"); errors.add("CYCLIC_ALIASES"); @@ -74,6 +73,8 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); errors.add("Different order of columns in UNION subquery"); // https://github.com/ClickHouse/ClickHouse/issues/44866 errors.add("Unexpected inf or nan to integer conversion"); + errors.add("Unsigned type must not contain"); + errors.add("Unexpected inf or nan to integer conversion"); } } From 1c053cef10b3a1b9b66d8a31892ca8949073d068 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 15 Jan 2023 23:33:45 +0800 Subject: [PATCH 0728/1284] tidb: support tidb_cost_model_version (#692) Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 4 ++-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cf55af4e..c51c32270 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -328,8 +328,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + docker pull pingcap/tidb:nightly + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:nightly sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index b48ea31cc..222947807 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -13,7 +13,7 @@ private TiDBSetGenerator() { } private enum Action { - + TIDB_COST_MODEL_VERSION("tidb_cost_model_version", (r) -> Randomly.fromOptions(1, 2)), // // SQL_MODE("sql_mode", (r) -> Randomly.fromOptions("TRADITIONAL", "ANSI", "POSTGRESQL", "ORACLE")), TIDB_OPT_AGG_PUSH_DOWN("tidb_opt_agg_push_down", (r) -> Randomly.fromOptions(0, 1)), // TIDB_BUILD_STATS_CONCURRENCY("tidb_build_stats_concurrency", (r) -> Randomly.getNotCachedInteger(0, 500)), // From e7d91ad032492ca050b62b01db733a03a622f7f1 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 16 Jan 2023 09:42:56 +0800 Subject: [PATCH 0729/1284] Revert "tidb: support tidb_cost_model_version (#692)" (#694) This reverts commit 1c053cef10b3a1b9b66d8a31892ca8949073d068. --- .github/workflows/main.yml | 4 ++-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c51c32270..2cf55af4e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -328,8 +328,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:nightly - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:nightly + docker pull pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 222947807..b48ea31cc 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -13,7 +13,7 @@ private TiDBSetGenerator() { } private enum Action { - TIDB_COST_MODEL_VERSION("tidb_cost_model_version", (r) -> Randomly.fromOptions(1, 2)), // + // SQL_MODE("sql_mode", (r) -> Randomly.fromOptions("TRADITIONAL", "ANSI", "POSTGRESQL", "ORACLE")), TIDB_OPT_AGG_PUSH_DOWN("tidb_opt_agg_push_down", (r) -> Randomly.fromOptions(0, 1)), // TIDB_BUILD_STATS_CONCURRENCY("tidb_build_stats_concurrency", (r) -> Randomly.getNotCachedInteger(0, 500)), // From ea7de8bf76c0a6b3fc50659b6ed2b9e3f2f62e20 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 16 Jan 2023 22:52:13 +0800 Subject: [PATCH 0730/1284] tidb: not use DEFAULT if columns have not default value (#693) * tidb: not use DEFAULT if columns have not default value Signed-off-by: Weizhen Wang * tidb: not use DEFAULT if columns have not default value Signed-off-by: Weizhen Wang * tidb: not use DEFAULT if columns have not default value Signed-off-by: Weizhen Wang * tidb: not use DEFAULT if columns have not default value Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBExpressionGenerator.java | 6 +++++- src/sqlancer/tidb/TiDBSchema.java | 13 +++++++++++-- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index a8fb75da0..a1db6fe2a 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -66,7 +66,11 @@ protected TiDBExpression generateExpression(int depth) { if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } - return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(generateColumn())); + TiDBColumn column = Randomly.fromList(columns); + if (column.hasDefault()) { + return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(new TiDBColumnReference(column))); + } + throw new IgnoreMeException(); case UNARY_POSTFIX: return new TiDBUnaryPostfixOperation(generateExpression(depth + 1), TiDBUnaryPostfixOperator.getRandom()); case UNARY_PREFIX: diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 676475d4a..04bde5e33 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -160,11 +160,14 @@ public static class TiDBColumn extends AbstractTableColumn { @@ -343,7 +350,9 @@ private static List getTableColumns(SQLConnection con, String tableN String dataType = rs.getString("Type"); boolean isNullable = rs.getString("Null").contentEquals("YES"); boolean isPrimaryKey = rs.getString("Key").contains("PRI"); - TiDBColumn c = new TiDBColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); + boolean hasDefault = rs.getString("Default") != null; + TiDBColumn c = new TiDBColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable, + hasDefault); columns.add(c); } } diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 0ce8190e7..4349b69ca 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -38,7 +38,7 @@ public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException allowPrimaryKey = Randomly.getBoolean(); primaryKeyAsTableConstraints = allowPrimaryKey && Randomly.getBoolean(); for (int i = 0; i < nrColumns; i++) { - TiDBColumn fakeColumn = new TiDBColumn("c" + i, null, false, false); + TiDBColumn fakeColumn = new TiDBColumn("c" + i, null, false, false, false); columns.add(fakeColumn); } TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(columns); From 67751dc3d83b880cd8c7ed53416b8b157a2ba2ea Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 18 Jan 2023 15:21:46 +0100 Subject: [PATCH 0731/1284] Ignore AMBIGUOUS_IDENTIFIER (#696) * Ignore AMBIGUOUS_IDENTIFIER * Ignore MULTIPLE_EXPRESSIONS_FOR_ALIAS --- src/sqlancer/clickhouse/ClickHouseErrors.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 5a115ebd5..385ae2e27 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -75,6 +75,11 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Unexpected inf or nan to integer conversion"); errors.add("Unsigned type must not contain"); errors.add("Unexpected inf or nan to integer conversion"); + + // The way we generate JOINs we can have ambiguous left table column without alias + // We may not count it as an issue, but it makes no sense to add more complex AST generation logic + errors.add("MULTIPLE_EXPRESSIONS_FOR_ALIAS"); + errors.add("AMBIGUOUS_IDENTIFIER"); // https://github.com/ClickHouse/ClickHouse/issues/45389 } } From 9275f1ddd1d3bb33c1a10f07f41ecf9b552fdfbd Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Thu, 26 Jan 2023 09:51:42 +0800 Subject: [PATCH 0732/1284] update cockroachdb in CI (#703) --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cf55af4e..5988c56fd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,11 +109,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-alpha.1.linux-amd64.tgz | tar xvz - cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.3.linux-amd64.tgz | tar xvz + cd cockroach-v22.2.3.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -132,11 +132,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-alpha.1.linux-amd64.tgz | tar xvz - cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.3.linux-amd64.tgz | tar xvz + cd cockroach-v22.2.3.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test From 39c48cde9ff0ab29b950a8d3e35930b0949f36a0 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Sat, 28 Jan 2023 03:02:01 +0100 Subject: [PATCH 0733/1284] Use master build of ClickHouse and update expected errors (#702) * Remove skipped error * Use master ClickHouse build in tests * Fix one more time * Get fatal logs if ClickHouse crashed * Fix tab from vi * fix not found errors * Increase socket timeout --------- Co-authored-by: Manuel Rigger --- .github/workflows/main.yml | 11 +++++++++-- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 -- src/sqlancer/clickhouse/ClickHouseProvider.java | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5988c56fd..1587aee78 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,6 +72,7 @@ jobs: - name: Run Tests run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test + clickhouse: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest @@ -87,11 +88,17 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up ClickHouse run: | - docker pull clickhouse/clickhouse-server:latest - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:latest + docker pull clickhouse/clickhouse-server:head + docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:head sleep 5 - name: Run Tests run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test + - name: Show fatal errors + run: docker exec clickhouse-server grep Fatal /var/log/clickhouse-server/clickhouse-server.log || echo No Fatal Errors found + - name: Teardown ClickHouse server + run: | + docker stop clickhouse-server + docker rm clickhouse-server cockroachdb: diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 385ae2e27..97a5e300c 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -40,7 +40,6 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("argument of function"); errors.add("but its arguments considered equal according to constraints"); errors.add("does not return a value of type UInt8"); - errors.add("does not return a value of type UInt8"); errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 errors.add("invalid character class range"); @@ -71,7 +70,6 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Positional argument must be constant with numeric type"); errors.add(" is out of bounds. Expected in range"); errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); - errors.add("Different order of columns in UNION subquery"); // https://github.com/ClickHouse/ClickHouse/issues/44866 errors.add("Unexpected inf or nan to integer conversion"); errors.add("Unsigned type must not contain"); errors.add("Unexpected inf or nan to integer conversion"); diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index e3a1fb21c..5c0c371c6 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -142,7 +142,8 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ e.printStackTrace(); } con.close(); - con = DriverManager.getConnection(String.format("jdbc:clickhouse://%s:%d/%s", host, port, databaseName), + con = DriverManager.getConnection( + String.format("jdbc:clickhouse://%s:%d/%s?socket_timeout=300000", host, port, databaseName), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); return new SQLConnection(con); } From 354d591cfcd37fa1de85ec77ec933d5d975e947a Mon Sep 17 00:00:00 2001 From: ZuoTiJia <49109132+ZuoTiJia@users.noreply.github.com> Date: Sat, 28 Jan 2023 20:59:50 +0800 Subject: [PATCH 0734/1284] Support CnosDB (#697) * Add Cnosdb (#1) Add Cnosdb * cnosdb: fix check style error * cnosdb: fix compiler error * cnosdb: remove unused * cnosdb: remove some pqs impl * cnosdb: remove dead code * add query execute count * refactor expectedError * cnosdb: remove unsuccessful count --- pom.xml | 5 + src/check_names.py | 1 + src/sqlancer/cnosdb/CnosDBBugs.java | 13 + .../cnosdb/CnosDBComparatorHelper.java | 145 +++++ .../cnosdb/CnosDBCompoundDataType.java | 20 + src/sqlancer/cnosdb/CnosDBExpectedError.java | 80 +++ src/sqlancer/cnosdb/CnosDBGlobalState.java | 28 + .../cnosdb/CnosDBLoggableFactory.java | 55 ++ src/sqlancer/cnosdb/CnosDBOptions.java | 64 +++ src/sqlancer/cnosdb/CnosDBProvider.java | 123 +++++ src/sqlancer/cnosdb/CnosDBSchema.java | 243 ++++++++ .../cnosdb/CnosDBToStringVisitor.java | 278 ++++++++++ src/sqlancer/cnosdb/CnosDBVisitor.java | 102 ++++ src/sqlancer/cnosdb/ast/CnosDBAggregate.java | 113 ++++ src/sqlancer/cnosdb/ast/CnosDBAlias.java | 35 ++ .../cnosdb/ast/CnosDBBetweenOperation.java | 34 ++ .../ast/CnosDBBinaryArithmeticOperation.java | 69 +++ .../ast/CnosDBBinaryComparisonOperation.java | 57 ++ .../ast/CnosDBBinaryLogicalOperation.java | 33 ++ .../cnosdb/ast/CnosDBCastOperation.java | 60 ++ .../cnosdb/ast/CnosDBColumnValue.java | 27 + .../cnosdb/ast/CnosDBConcatOperation.java | 22 + src/sqlancer/cnosdb/ast/CnosDBConstant.java | 520 ++++++++++++++++++ src/sqlancer/cnosdb/ast/CnosDBExpression.java | 14 + src/sqlancer/cnosdb/ast/CnosDBFunction.java | 30 + .../ast/CnosDBFunctionWithUnknownResult.java | 104 ++++ .../cnosdb/ast/CnosDBInOperation.java | 35 ++ src/sqlancer/cnosdb/ast/CnosDBJoin.java | 46 ++ .../cnosdb/ast/CnosDBLikeOperation.java | 22 + .../cnosdb/ast/CnosDBOrderByTerm.java | 37 ++ .../cnosdb/ast/CnosDBPostfixOperation.java | 97 ++++ .../cnosdb/ast/CnosDBPostfixText.java | 29 + .../cnosdb/ast/CnosDBPrefixOperation.java | 73 +++ src/sqlancer/cnosdb/ast/CnosDBSelect.java | 102 ++++ src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java | 28 + src/sqlancer/cnosdb/client/CnosDBClient.java | 110 ++++ .../cnosdb/client/CnosDBConnection.java | 27 + .../cnosdb/client/CnosDBException.java | 9 + .../cnosdb/client/CnosDBResultSet.java | 52 ++ src/sqlancer/cnosdb/gen/CnosDBCommon.java | 31 ++ .../cnosdb/gen/CnosDBExpressionGenerator.java | 461 ++++++++++++++++ .../cnosdb/gen/CnosDBInsertGenerator.java | 59 ++ .../cnosdb/gen/CnosDBTableGenerator.java | 77 +++ .../cnosdb/oracle/CnosDBNoRECBase.java | 23 + .../cnosdb/oracle/CnosDBNoRECOracle.java | 171 ++++++ .../oracle/tlp/CnosDBTLPAggregateOracle.java | 176 ++++++ .../cnosdb/oracle/tlp/CnosDBTLPBase.java | 112 ++++ .../oracle/tlp/CnosDBTLPHavingOracle.java | 65 +++ .../oracle/tlp/CnosDBTLPWhereOracle.java | 46 ++ .../cnosdb/query/CnosDBOtherQuery.java | 30 + .../cnosdb/query/CnosDBQueryAdapter.java | 41 ++ .../cnosdb/query/CnosDBQueryProvider.java | 6 + .../cnosdb/query/CnosDBSelectQuery.java | 38 ++ 53 files changed, 4278 insertions(+) create mode 100644 src/sqlancer/cnosdb/CnosDBBugs.java create mode 100644 src/sqlancer/cnosdb/CnosDBComparatorHelper.java create mode 100644 src/sqlancer/cnosdb/CnosDBCompoundDataType.java create mode 100644 src/sqlancer/cnosdb/CnosDBExpectedError.java create mode 100644 src/sqlancer/cnosdb/CnosDBGlobalState.java create mode 100644 src/sqlancer/cnosdb/CnosDBLoggableFactory.java create mode 100644 src/sqlancer/cnosdb/CnosDBOptions.java create mode 100644 src/sqlancer/cnosdb/CnosDBProvider.java create mode 100644 src/sqlancer/cnosdb/CnosDBSchema.java create mode 100644 src/sqlancer/cnosdb/CnosDBToStringVisitor.java create mode 100644 src/sqlancer/cnosdb/CnosDBVisitor.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBAggregate.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBAlias.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBCastOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBColumnValue.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBConstant.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBExpression.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunction.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBInOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBJoin.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixText.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBSelect.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java create mode 100644 src/sqlancer/cnosdb/client/CnosDBClient.java create mode 100644 src/sqlancer/cnosdb/client/CnosDBConnection.java create mode 100644 src/sqlancer/cnosdb/client/CnosDBException.java create mode 100644 src/sqlancer/cnosdb/client/CnosDBResultSet.java create mode 100644 src/sqlancer/cnosdb/gen/CnosDBCommon.java create mode 100644 src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java create mode 100644 src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java create mode 100644 src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java create mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java create mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java create mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java create mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java create mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java create mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java create mode 100644 src/sqlancer/cnosdb/query/CnosDBOtherQuery.java create mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java create mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryProvider.java create mode 100644 src/sqlancer/cnosdb/query/CnosDBSelectQuery.java diff --git a/pom.xml b/pom.xml index b61126106..46211aac2 100644 --- a/pom.xml +++ b/pom.xml @@ -358,6 +358,11 @@ 2.7.1 runtime + + org.apache.commons + commons-csv + 1.9.0 + diff --git a/src/check_names.py b/src/check_names.py index bca448b19..72191ac7d 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -26,3 +26,4 @@ def verify_prefix(prefix, files): verify_prefix('Postgres', get_java_files("sqlancer/postgres/")) verify_prefix('SQLite3', get_java_files("sqlancer/sqlite3/")) verify_prefix('TiDB', get_java_files("sqlancer/tidb/")) +verify_prefix('CnosDB', get_java_files("sqlancer/cnosdb/")) diff --git a/src/sqlancer/cnosdb/CnosDBBugs.java b/src/sqlancer/cnosdb/CnosDBBugs.java new file mode 100644 index 000000000..4e6eb96e9 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBBugs.java @@ -0,0 +1,13 @@ +package sqlancer.cnosdb; + +public final class CnosDBBugs { + + // https://github.com/cnosdb/cnosdb/issues/786 + public static final boolean BUG786 = true; + + // https://github.com/apache/arrow-rs/issues/3547 + public static final boolean BUG3547 = true; + + private CnosDBBugs() { + } +} diff --git a/src/sqlancer/cnosdb/CnosDBComparatorHelper.java b/src/sqlancer/cnosdb/CnosDBComparatorHelper.java new file mode 100644 index 000000000..46b6ba615 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBComparatorHelper.java @@ -0,0 +1,145 @@ +package sqlancer.cnosdb; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.cnosdb.query.CnosDBSelectQuery; +import sqlancer.common.query.ExpectedErrors; + +public final class CnosDBComparatorHelper { + + private CnosDBComparatorHelper() { + } + + public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, + CnosDBGlobalState state) throws Exception { + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, errors); + List result = new ArrayList<>(); + CnosDBResultSet resultSet; + try { + q.executeAndGet(state); + resultSet = q.getResultSet(); + if (resultSet == null) { + throw new AssertionError(q); + } + while (resultSet.next()) { + result.add(resultSet.getString(1)); + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw e; + } + if (e instanceof NumberFormatException) { + throw new IgnoreMeException(); + } + if (e.getMessage() == null) { + throw new AssertionError(queryString, e); + } + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + throw new AssertionError(queryString, e); + } + + return result; + } + + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, + String originalQueryString, List combinedString, CnosDBGlobalState state) { + if (resultSet.size() != secondResultSet.size()) { + String queryFormatString = "-- %s;\n-- cardinality: %d"; + String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); + String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), + secondResultSet.size()); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", + resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); + throw new AssertionError(assertionMessage); + } + + Set firstHashSet = new HashSet<>(resultSet); + Set secondHashSet = new HashSet<>(secondResultSet); + + if (!firstHashSet.equals(secondHashSet)) { + Set firstResultSetMisses = new HashSet<>(firstHashSet); + firstResultSetMisses.removeAll(secondHashSet); + Set secondResultSetMisses = new HashSet<>(secondHashSet); + secondResultSetMisses.removeAll(firstHashSet); + String queryFormatString = "-- %s;\n-- misses: %s"; + String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); + String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), + secondResultSetMisses); + // update the SELECT queries to be logged at the bottom of the error log file + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", + firstQueryString, secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, + String originalQueryString, List combinedString, CnosDBGlobalState state, + UnaryOperator canonicalizationRule) { + // Overloaded version of assumeResultSetsAreEqual that takes a canonicalization function which is applied to + // both result sets before their comparison. + List canonicalizedResultSet = resultSet.stream().map(canonicalizationRule).collect(Collectors.toList()); + List canonicalizedSecondResultSet = secondResultSet.stream().map(canonicalizationRule) + .collect(Collectors.toList()); + assumeResultSetsAreEqual(canonicalizedResultSet, canonicalizedSecondResultSet, originalQueryString, + combinedString, state); + } + + public static List getCombinedResultSet(String firstQueryString, String secondQueryString, + String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, + ExpectedErrors errors) throws Exception { + List secondResultSet; + if (asUnion) { + String unionString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL " + + thirdQueryString; + combinedString.add(unionString); + secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); + } else { + secondResultSet = new ArrayList<>(); + secondResultSet.addAll(getResultSetFirstColumnAsString(firstQueryString, errors, state)); + secondResultSet.addAll(getResultSetFirstColumnAsString(secondQueryString, errors, state)); + secondResultSet.addAll(getResultSetFirstColumnAsString(thirdQueryString, errors, state)); + combinedString.add(firstQueryString); + combinedString.add(secondQueryString); + combinedString.add(thirdQueryString); + } + return secondResultSet; + } + + public static List getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString, + String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, + ExpectedErrors errors) throws Exception { + String unionString; + if (asUnion) { + unionString = firstQueryString + " UNION " + secondQueryString + " UNION " + thirdQueryString; + } else { + unionString = "SELECT DISTINCT * FROM (" + firstQueryString + " UNION ALL " + secondQueryString + + " UNION ALL " + thirdQueryString + ")"; + } + List secondResultSet; + combinedString.add(unionString); + secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); + return secondResultSet; + } +} diff --git a/src/sqlancer/cnosdb/CnosDBCompoundDataType.java b/src/sqlancer/cnosdb/CnosDBCompoundDataType.java new file mode 100644 index 000000000..034f0fc90 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBCompoundDataType.java @@ -0,0 +1,20 @@ +package sqlancer.cnosdb; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public final class CnosDBCompoundDataType { + + private final CnosDBDataType dataType; + + private CnosDBCompoundDataType(CnosDBDataType dataType) { + this.dataType = dataType; + } + + public static CnosDBCompoundDataType create(CnosDBDataType type) { + return new CnosDBCompoundDataType(type); + } + + public CnosDBDataType getDataType() { + return dataType; + } +} diff --git a/src/sqlancer/cnosdb/CnosDBExpectedError.java b/src/sqlancer/cnosdb/CnosDBExpectedError.java new file mode 100644 index 000000000..516e44ce4 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBExpectedError.java @@ -0,0 +1,80 @@ +package sqlancer.cnosdb; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class CnosDBExpectedError { + private static final List ERRORS = new ArrayList<>(); + + private CnosDBExpectedError() { + } + + static { + ERRORS.add("have the same name. Consider aliasing"); + ERRORS.add( + "error: Optimizer rule 'projection_push_down' failed due to unexpected error: Schema error: Schema contains duplicate qualified field name"); + ERRORS.add("Projection references non-aggregate values:"); + ERRORS.add("External err: Schema error: No field named"); + ERRORS.add( + "Optimizer rule 'common_sub_expression_eliminate' failed due to unexpected error: Schema error: No field named"); + ERRORS.add("Binary"); + ERRORS.add("Invalid pattern in LIKE expression"); + ERRORS.add("If the projection contains the time column, it must contain the field column."); + ERRORS.add("Schema error: No field named"); + ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:"); + ERRORS.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error"); + ERRORS.add("Schema error: No field named "); + ERRORS.add("err: External err: Schema error: No field named"); + ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error"); + ERRORS.add("Csv error: CSV Writer does not support List"); + ERRORS.add("This feature is not implemented: cross join."); + ERRORS.add("Execution error: field position must be greater than zero"); + ERRORS.add("First argument of `DATE_PART` must be non-null scalar Utf8"); + ERRORS.add("Cannot create filter with non-boolean predicate 'NULL' returning Null"); + ERRORS.add("requested character too large for encoding."); + ERRORS.add("Can not find compatible types to compare Boolean with [Utf8]."); + ERRORS.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT"); + ERRORS.add("HAVING clause references non-aggregate values:"); + ERRORS.add("Cannot create filter with non-boolean predicate"); + ERRORS.add("negative substring length not allowed"); + ERRORS.add("The function Sum does not support inputs of type Boolean."); + ERRORS.add("The function Avg does not support inputs of type Boolean."); + ERRORS.add("Percentile value must be between 0.0 and 1.0 inclusive"); + ERRORS.add("Date part '' not supported"); + ERRORS.add("Min/Max accumulator not implemented for type Boolean."); + ERRORS.add("meta need get_series_id_by_filter"); + ERRORS.add("Arrow: Cast error:"); + ERRORS.add("Arrow error: Cast error:"); + ERRORS.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:"); + ERRORS.add("Arrow error: Divide by zero error"); + ERRORS.add("desired percentile argument must be float literal"); + ERRORS.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)"); + ERRORS.add("Execution error: Date part"); + ERRORS.add("Physical plan does not support logical expression MIN(Boolean"); + ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64"); + ERRORS.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64."); + ERRORS.add("Data type UInt64 not supported for binary operation '#' on dyn arrays."); + ERRORS.add("Arrow: Divide by zero error"); + ERRORS.add("The function ApproxPercentileCont does not support inputs of type Null."); + ERRORS.add("can't be evaluated because there isn't a common type to coerce the types to"); + ERRORS.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug"); + ERRORS.add("The function ApproxMedian does not support inputs of type Null."); + ERRORS.add("null character not permitted."); + ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Null."); + ERRORS.add("This feature is not implemented"); + ERRORS.add("The function Avg does not support inputs of type Null."); + ERRORS.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]"); + ERRORS.add( + "Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed."); + ERRORS.add("Coercion from"); + } + + public static ExpectedErrors expectedErrors() { + ExpectedErrors res = new ExpectedErrors(); + res.addAll(ERRORS); + return res; + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBGlobalState.java b/src/sqlancer/cnosdb/CnosDBGlobalState.java new file mode 100644 index 000000000..9f34e03a5 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBGlobalState.java @@ -0,0 +1,28 @@ +package sqlancer.cnosdb; + +import sqlancer.ExecutionTimer; +import sqlancer.GlobalState; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.common.query.Query; + +public class CnosDBGlobalState extends GlobalState { + + @Override + protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { + boolean logExecutionTime = getOptions().logExecutionTime(); + if (success && getOptions().printSucceedingStatements()) { + System.out.println(q.getQueryString()); + } + if (logExecutionTime) { + getLogger().writeCurrent(" -- " + timer.end().asString()); + } + if (q.couldAffectSchema()) { + updateSchema(); + } + } + + @Override + public CnosDBSchema readSchema() throws Exception { + return CnosDBSchema.fromConnection(getConnection()); + } +} diff --git a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java new file mode 100644 index 000000000..a3f361b14 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java @@ -0,0 +1,55 @@ +package sqlancer.cnosdb; + +import java.io.PrintWriter; +import java.io.StringWriter; + +import sqlancer.cnosdb.query.CnosDBOtherQuery; +import sqlancer.cnosdb.query.CnosDBQueryAdapter; +import sqlancer.common.log.Loggable; +import sqlancer.common.log.LoggableFactory; +import sqlancer.common.log.LoggedString; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; + +public class CnosDBLoggableFactory extends LoggableFactory { + + @Override + protected Loggable createLoggable(String input, String suffix) { + String completeString = input; + if (!input.endsWith(";")) { + completeString += ";"; + } + if (suffix != null && suffix.length() != 0) { + completeString += suffix; + } + return new LoggedString(completeString); + } + + @Override + public CnosDBQueryAdapter getQueryForStateToReproduce(String queryString) { + return new CnosDBOtherQuery(queryString, CnosDBExpectedError.expectedErrors()); + } + + @Override + public CnosDBQueryAdapter commentOutQuery(Query query) { + String queryString = query.getLogString(); + String newQueryString = "-- " + queryString; + ExpectedErrors errors = new ExpectedErrors(); + return new CnosDBOtherQuery(newQueryString, errors); + } + + @Override + protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { + String sb = "-- Time: " + time + "\n" + "-- Database: " + databaseName + "\n" + "-- Database version: " + + databaseVersion + "\n" + "-- seed value: " + seedValue + "\n"; + return new LoggedString(sb); + } + + @Override + public Loggable convertStacktraceToLoggable(Throwable throwable) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + throwable.printStackTrace(pw); + return new LoggedString("--" + sw.toString().replace("\n", "\n--")); + } +} diff --git a/src/sqlancer/cnosdb/CnosDBOptions.java b/src/sqlancer/cnosdb/CnosDBOptions.java new file mode 100644 index 000000000..9a6da00c4 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBOptions.java @@ -0,0 +1,64 @@ +package sqlancer.cnosdb; + +import java.util.ArrayList; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.cnosdb.CnosDBOptions.CnosDBOracleFactory; +import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPAggregateOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPHavingOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPWhereOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; + +@Parameters(separators = "=", commandDescription = "CnosDB (default port: " + CnosDBOptions.DEFAULT_PORT + + ", default host: " + CnosDBOptions.DEFAULT_HOST + ")") +public class CnosDBOptions implements DBMSSpecificOptions { + + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 31001; + + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for CnosDB") + public List oracle = List.of(CnosDBOracleFactory.QUERY_PARTITIONING); + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the CnosDB", arity = 1) + public String connectionURL = String.format("http://%s:%d", CnosDBOptions.DEFAULT_HOST, CnosDBOptions.DEFAULT_PORT); + + @Override + public List getTestOracleFactory() { + return oracle; + } + + public enum CnosDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + return new CnosDBNoRECOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + return new CnosDBTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + List> oracles = new ArrayList<>(); + oracles.add(new CnosDBTLPWhereOracle(globalState)); + oracles.add(new CnosDBTLPHavingOracle(globalState)); + oracles.add(new CnosDBTLPAggregateOracle(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + } + + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBProvider.java b/src/sqlancer/cnosdb/CnosDBProvider.java new file mode 100644 index 000000000..8b69c53b3 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBProvider.java @@ -0,0 +1,123 @@ +package sqlancer.cnosdb; + +import java.util.Objects; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.ProviderAdapter; +import sqlancer.Randomly; +import sqlancer.StatementExecutor; +import sqlancer.cnosdb.client.CnosDBClient; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.cnosdb.gen.CnosDBInsertGenerator; +import sqlancer.cnosdb.gen.CnosDBTableGenerator; +import sqlancer.cnosdb.query.CnosDBOtherQuery; +import sqlancer.cnosdb.query.CnosDBQueryProvider; +import sqlancer.common.log.LoggableFactory; + +@AutoService(DatabaseProvider.class) +public class CnosDBProvider extends ProviderAdapter { + + protected String username; + protected String password; + protected String host; + protected int port; + protected String databaseName; + + public CnosDBProvider() { + super(CnosDBGlobalState.class, CnosDBOptions.class); + } + + protected CnosDBProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + protected static int mapActions(CnosDBGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + if (Objects.requireNonNull(a) == Action.INSERT) { + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + } else { + throw new AssertionError(a); + } + return nrPerformed; + + } + + @Override + protected void checkViewsAreValid(CnosDBGlobalState globalState) { + } + + @Override + public void generateDatabase(CnosDBGlobalState globalState) throws Exception { + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + prepareTables(globalState); + + } + + @Override + public CnosDBConnection createDatabase(CnosDBGlobalState globalState) throws Exception { + + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + databaseName = globalState.getDatabaseName(); + CnosDBClient client = new CnosDBClient(host, port, username, password, databaseName); + CnosDBConnection connection = new CnosDBConnection(client); + client.execute("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + client.execute("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + + return connection; + } + + protected void createTables(CnosDBGlobalState globalState, int numTables) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + String tableName = String.format("m%d", globalState.getSchema().getDatabaseTables().size()); + CnosDBOtherQuery createTable = CnosDBTableGenerator.generate(tableName); + globalState.executeStatement(createTable); + } + } + + protected void prepareTables(CnosDBGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + CnosDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public String getDBMSName() { + return "CnosDB".toLowerCase(); + } + + @Override + public LoggableFactory getLoggableFactory() { + return new CnosDBLoggableFactory(); + } + + public enum Action implements AbstractAction { + INSERT(CnosDBInsertGenerator::insert); + + private final CnosDBQueryProvider sqlQueryProvider; + + Action(CnosDBQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public CnosDBOtherQuery getQuery(CnosDBGlobalState state) throws Exception { + return new CnosDBOtherQuery(sqlQueryProvider.getQuery(state).getQueryString(), + CnosDBExpectedError.expectedErrors()); + } + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBSchema.java b/src/sqlancer/cnosdb/CnosDBSchema.java new file mode 100644 index 000000000..35d9d8887 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBSchema.java @@ -0,0 +1,243 @@ +package sqlancer.cnosdb; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import sqlancer.Randomly; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; + +public class CnosDBSchema extends AbstractSchema { + + private final String databaseName; + + public CnosDBSchema(List databaseTables, String databaseName) { + super(databaseTables); + this.databaseName = databaseName; + } + + public static CnosDBDataType getColumnType(String typeString) { + switch (typeString.toLowerCase()) { + case "bigint": + return CnosDBDataType.INT; + case "boolean": + return CnosDBDataType.BOOLEAN; + case "string": + return CnosDBDataType.STRING; + case "double": + return CnosDBDataType.DOUBLE; + case "bigint unsigned": + case "unsigned": + return CnosDBDataType.UINT; + case "timestamp": + return CnosDBDataType.TIMESTAMP; + default: + throw new AssertionError(typeString); + } + } + + public static CnosDBSchema fromConnection(CnosDBConnection con) throws Exception { + CnosDBResultSet tablesRes = con.getClient().executeQuery("SHOW TABLES"); + + List tables = new ArrayList<>(); + while (tablesRes.next()) { + String tableName = tablesRes.getString(1); + List columns = getTableColumns(con, tableName); + tables.add(new CnosDBTable(tableName, columns)); + } + + return new CnosDBSchema(tables, con.getClient().getDatabase()); + } + + protected static List getTableColumns(CnosDBConnection con, String tableName) throws Exception { + CnosDBResultSet columnsRes = con.getClient().executeQuery("DESCRIBE TABLE " + tableName); + List columns = new ArrayList<>(); + CnosDBTable table = new CnosDBTable(tableName, columns); + while (columnsRes.next()) { + String columnName = columnsRes.getString(1); + String columnType = columnsRes.getString(3).toLowerCase(); + CnosDBDataType dataType = CnosDBSchema.getColumnType(columnsRes.getString(2)); + CnosDBColumn column; + if (columnType.contentEquals("time")) { + column = new CnosDBTimeColumn(); + } else if (columnType.contentEquals("tag")) { + column = new CnosDBTagColumn(columnName); + } else { + column = new CnosDBFieldColumn(columnName, dataType); + } + column.setTable(table); + columns.add(column); + } + + return columns; + } + + public CnosDBTables getRandomTableNonEmptyTables() { + return new CnosDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public String getDatabaseName() { + return databaseName; + } + + public enum CnosDBDataType { + INT, BOOLEAN, STRING, DOUBLE, UINT, TIMESTAMP; + + public static CnosDBDataType getRandomType() { + return Randomly.fromOptions(values()); + } + + public static CnosDBDataType getRandomTypeWithoutTimeStamp() { + List dataTypes = new ArrayList<>(Arrays.asList(values())); + dataTypes.remove(TIMESTAMP); + return Randomly.fromList(dataTypes); + } + } + + public static class CnosDBColumn extends AbstractTableColumn { + + public CnosDBColumn(String name, CnosDBDataType columnType) { + super(name, null, columnType); + } + + public static CnosDBColumn createDummy(String name) { + return new CnosDBColumn(name, CnosDBDataType.INT); + } + + } + + public static class CnosDBTagColumn extends CnosDBColumn { + public CnosDBTagColumn(String name) { + super(name, CnosDBDataType.STRING); + } + } + + public static class CnosDBTimeColumn extends CnosDBColumn { + public CnosDBTimeColumn() { + super("TIME", CnosDBDataType.TIMESTAMP); + } + } + + public static class CnosDBFieldColumn extends CnosDBColumn { + public CnosDBFieldColumn(String name, CnosDBDataType columnType) { + super(name, columnType); + assert columnType != CnosDBDataType.TIMESTAMP; + } + } + + public static class CnosDBTables extends AbstractTables { + + public CnosDBTables(List tables) { + super(tables); + } + + public CnosDBRowValue getRandomRowValue(CnosDBConnection con) { + return null; + } + + public List getRandomColumnsWithOnlyOneField() { + ArrayList res = new ArrayList<>(); + this.getTables().forEach(table -> res.addAll(table.getRandomColumnsWithOnlyOneField())); + return res; + } + + } + + public static class CnosDBRowValue extends AbstractRowValue { + + protected CnosDBRowValue(CnosDBTables tables, Map values) { + super(tables, values); + } + + } + + public static class CnosDBTable extends AbstractTable { + + public CnosDBTable(String tableName, List columns) { + super(tableName, columns, null, false); + } + + @Override + public List getColumns() { + List res = super.getColumns(); + boolean hasTime = false; + for (CnosDBColumn column : res) { + if (column instanceof CnosDBTimeColumn) { + hasTime = true; + break; + } + } + assert hasTime; + + return res; + } + + public List getRandomColumnsWithOnlyOneField() { + ArrayList res = new ArrayList<>(); + boolean hasField = false; + for (CnosDBColumn column : getColumns()) { + if (column instanceof CnosDBTagColumn && Randomly.getBoolean()) { + res.add(column); + } else if (column instanceof CnosDBFieldColumn && !hasField) { + res.add(column); + hasField = true; + } + } + return res; + } + + // SELECT COUNT(*) FROM table; + @Override + public long getNrRows(CnosDBGlobalState globalState) { + long res; + try { + CnosDBResultSet tableCountRes = globalState.getConnection().getClient() + .executeQuery("SELECT COUNT(time) FROM " + this.name); + tableCountRes.next(); + res = tableCountRes.getLong(1); + } catch (Exception e) { + res = 0; + } + return res; + } + + @Override + public List getRandomNonEmptyColumnSubset() { + List selectedColumns = new ArrayList<>(); + ArrayList remainingColumns = new ArrayList<>(this.getColumns()); + + remainingColumns.removeIf(column -> column instanceof CnosDBTimeColumn); + CnosDBTimeColumn timeColumn = new CnosDBTimeColumn(); + timeColumn.setTable(this); + selectedColumns.add(timeColumn); + + remainingColumns.stream().filter(column -> column instanceof CnosDBTagColumn).findFirst().ifPresent(tag -> { + selectedColumns.add(tag); + remainingColumns.remove(tag); + }); + + remainingColumns.stream().filter(column -> column instanceof CnosDBFieldColumn).findFirst() + .ifPresent(field -> { + selectedColumns.add(field); + remainingColumns.remove(field); + }); + + int nr = Math.min(Randomly.smallNumber() + 1, remainingColumns.size()); + for (int i = 0; i < nr; i++) { + selectedColumns + .add(remainingColumns.remove((int) Randomly.getNotCachedInteger(0, remainingColumns.size()))); + } + return selectedColumns; + } + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java new file mode 100644 index 000000000..64f2fd8b8 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java @@ -0,0 +1,278 @@ +package sqlancer.cnosdb; + +import sqlancer.Randomly; +import sqlancer.cnosdb.ast.CnosDBAggregate; +import sqlancer.cnosdb.ast.CnosDBBetweenOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; +import sqlancer.cnosdb.ast.CnosDBCastOperation; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBFunction; +import sqlancer.cnosdb.ast.CnosDBInOperation; +import sqlancer.cnosdb.ast.CnosDBJoin; +import sqlancer.cnosdb.ast.CnosDBLikeOperation; +import sqlancer.cnosdb.ast.CnosDBOrderByTerm; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation; +import sqlancer.cnosdb.ast.CnosDBPostfixText; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; +import sqlancer.cnosdb.ast.CnosDBSimilarTo; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.ToStringVisitor; + +public final class CnosDBToStringVisitor extends ToStringVisitor implements CnosDBVisitor { + + @Override + public void visitSpecific(CnosDBExpression expr) { + CnosDBVisitor.super.visit(expr); + } + + @Override + public void visit(CnosDBConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(CnosDBPostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" "); + sb.append(op.getOperatorTextRepresentation()); + } + + @Override + public void visit(CnosDBColumnValue c) { + sb.append(c.getColumn().getFullQualifiedName()); + } + + @Override + public void visit(CnosDBPrefixOperation op) { + sb.append(op.getTextRepresentation()); + sb.append(" ("); + visit(op.getExpression()); + sb.append(")"); + } + + @Override + public void visit(CnosDBFromTable from) { + sb.append(from.getTable().getName()); + } + + @Override + public void visit(CnosDBSubquery subquery) { + sb.append("("); + visit(subquery.getSelect()); + sb.append(") AS "); + sb.append(subquery.getName()); + } + + @Override + public void visit(CnosDBSelect s) { + sb.append("SELECT "); + switch (s.getSelectOption()) { + case DISTINCT: + sb.append("DISTINCT "); + if (s.getDistinctOnClause() != null) { + sb.append("ON ("); + visit(s.getDistinctOnClause()); + sb.append(") "); + } + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + if (s.getFetchColumns() == null) { + sb.append("*"); + } else { + visit(s.getFetchColumns()); + } + sb.append(" FROM "); + visit(s.getFromList()); + + for (CnosDBJoin j : s.getJoinClauses()) { + sb.append(" "); + switch (j.getType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } + sb.append("JOIN"); + break; + case LEFT: + sb.append("LEFT OUTER JOIN"); + break; + case RIGHT: + sb.append("RIGHT OUTER JOIN"); + break; + case FULL: + sb.append("FULL OUTER JOIN"); + break; + // case CROSS: + // sb.append("CROSS JOIN"); + // break; + default: + throw new AssertionError(j.getType()); + } + sb.append(" "); + visit(j.getTableReference()); + // if (j.getType() != CnosDBJoinType.CROSS) { + sb.append(" ON "); + visit(j.getOnClause()); + // } + } + + if (s.getWhereClause() != null) { + sb.append(" WHERE "); + visit(s.getWhereClause()); + } + if (s.getGroupByExpressions().size() > 0) { + sb.append(" GROUP BY "); + visit(s.getGroupByExpressions()); + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + + } + if (!s.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(s.getOrderByExpressions()); + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(CnosDBOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder()); + } + + @Override + public void visit(CnosDBFunction f) { + sb.append(f.getFunctionName()); + sb.append("("); + int i = 0; + for (CnosDBExpression arg : f.getArguments()) { + if (i++ != 0) { + sb.append(", "); + } + visit(arg); + } + sb.append(")"); + } + + @Override + public void visit(CnosDBCastOperation cast) { + sb.append("CAST( "); + visit(cast.getExpression()); + sb.append(" AS "); + appendType(cast); + sb.append(")"); + } + + private void appendType(CnosDBCastOperation cast) { + CnosDBCompoundDataType compoundType = cast.getCompoundType(); + switch (compoundType.getDataType()) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: + sb.append("BIGINT"); + break; + case STRING: + sb.append(Randomly.fromOptions("STRING")); + break; + case DOUBLE: + sb.append("DOUBLE"); + break; + case UINT: + sb.append("BIGINT UNSIGNED"); + break; + case TIMESTAMP: + sb.append("TIMESTAMP"); + break; + + default: + throw new AssertionError(cast.getType()); + } + } + + @Override + public void visit(CnosDBBetweenOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(") BETWEEN ("); + visit(op.getLeft()); + sb.append(") AND ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(CnosDBInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN ("); + visit(op.getListElements()); + sb.append(")"); + } + + @Override + public void visit(CnosDBPostfixText op) { + visit(op.getExpr()); + sb.append(op.getText()); + } + + @Override + public void visit(CnosDBAggregate op) { + sb.append(op.getFunction()); + sb.append("("); + visit(op.getArgs()); + sb.append(")"); + } + + @Override + public void visit(CnosDBSimilarTo op) { + sb.append("("); + visit(op.getString()); + sb.append(" SIMILAR TO "); + visit(op.getSimilarTo()); + sb.append(")"); + } + + @Override + public void visit(CnosDBBinaryLogicalOperation op) { + super.visit((BinaryOperation) op); + } + + @Override + public void visit(CnosDBLikeOperation op) { + super.visit((BinaryOperation) op); + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBVisitor.java b/src/sqlancer/cnosdb/CnosDBVisitor.java new file mode 100644 index 000000000..7c1af7224 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBVisitor.java @@ -0,0 +1,102 @@ +package sqlancer.cnosdb; + +import sqlancer.cnosdb.ast.CnosDBAggregate; +import sqlancer.cnosdb.ast.CnosDBBetweenOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; +import sqlancer.cnosdb.ast.CnosDBCastOperation; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBFunction; +import sqlancer.cnosdb.ast.CnosDBInOperation; +import sqlancer.cnosdb.ast.CnosDBLikeOperation; +import sqlancer.cnosdb.ast.CnosDBOrderByTerm; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation; +import sqlancer.cnosdb.ast.CnosDBPostfixText; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; +import sqlancer.cnosdb.ast.CnosDBSimilarTo; + +public interface CnosDBVisitor { + + static String asString(CnosDBExpression expr) { + CnosDBToStringVisitor visitor = new CnosDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + void visit(CnosDBConstant constant); + + void visit(CnosDBPostfixOperation op); + + void visit(CnosDBColumnValue c); + + void visit(CnosDBPrefixOperation op); + + void visit(CnosDBSelect op); + + void visit(CnosDBOrderByTerm op); + + void visit(CnosDBFunction f); + + void visit(CnosDBCastOperation cast); + + void visit(CnosDBBetweenOperation op); + + void visit(CnosDBInOperation op); + + void visit(CnosDBPostfixText op); + + void visit(CnosDBAggregate op); + + void visit(CnosDBFromTable from); + + void visit(CnosDBSubquery subquery); + + void visit(CnosDBBinaryLogicalOperation op); + + void visit(CnosDBLikeOperation op); + + void visit(CnosDBSimilarTo op); + + default void visit(CnosDBExpression expression) { + if (expression instanceof CnosDBConstant) { + visit((CnosDBConstant) expression); + } else if (expression instanceof CnosDBPostfixOperation) { + visit((CnosDBPostfixOperation) expression); + } else if (expression instanceof CnosDBColumnValue) { + visit((CnosDBColumnValue) expression); + } else if (expression instanceof CnosDBPrefixOperation) { + visit((CnosDBPrefixOperation) expression); + } else if (expression instanceof CnosDBSelect) { + visit((CnosDBSelect) expression); + } else if (expression instanceof CnosDBOrderByTerm) { + visit((CnosDBOrderByTerm) expression); + } else if (expression instanceof CnosDBFunction) { + visit((CnosDBFunction) expression); + } else if (expression instanceof CnosDBCastOperation) { + visit((CnosDBCastOperation) expression); + } else if (expression instanceof CnosDBBetweenOperation) { + visit((CnosDBBetweenOperation) expression); + } else if (expression instanceof CnosDBInOperation) { + visit((CnosDBInOperation) expression); + } else if (expression instanceof CnosDBAggregate) { + visit((CnosDBAggregate) expression); + } else if (expression instanceof CnosDBPostfixText) { + visit((CnosDBPostfixText) expression); + } else if (expression instanceof CnosDBSimilarTo) { + visit((CnosDBSimilarTo) expression); + } else if (expression instanceof CnosDBFromTable) { + visit((CnosDBFromTable) expression); + } else if (expression instanceof CnosDBSubquery) { + visit((CnosDBSubquery) expression); + } else if (expression instanceof CnosDBLikeOperation) { + visit((CnosDBLikeOperation) expression); + } else { + throw new AssertionError(expression); + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAggregate.java b/src/sqlancer/cnosdb/ast/CnosDBAggregate.java new file mode 100644 index 000000000..df30717b4 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBAggregate.java @@ -0,0 +1,113 @@ +package sqlancer.cnosdb.ast; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBBugs; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; +import sqlancer.common.ast.FunctionNode; + +public class CnosDBAggregate extends FunctionNode + implements CnosDBExpression { + + public CnosDBAggregate(List args, CnosDBAggregateFunction func) { + super(func, args); + } + + public enum CnosDBAggregateFunction { + AVG(CnosDBDataType.DOUBLE), + MAX(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, + CnosDBDataType.UINT), + MIN(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, + CnosDBDataType.UINT), + COUNT(CnosDBDataType.INT) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.getRandomType() }; + } + }, + SUM(CnosDBDataType.INT, CnosDBDataType.DOUBLE, CnosDBDataType.UINT), APPROX_MEDIAN(CnosDBDataType.DOUBLE), + + VAR(CnosDBDataType.DOUBLE), VAR_SAMP(CnosDBDataType.DOUBLE), VAR_POP(CnosDBDataType.DOUBLE), + STDDEV(CnosDBDataType.DOUBLE), STDDEV_SAMP(CnosDBDataType.DOUBLE), STDDEV_POP(CnosDBDataType.DOUBLE), + COVAR(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + COVAR_SAMP(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.INT }; + } + }, + CORR(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + COVAR_POP(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + + APPROX_PERCENTILE_CONT(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + APPROX_PERCENTILE_CONT_WITH_WEIGHT(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + APPROX_DISTINCT(CnosDBDataType.UINT), GROUPING(CnosDBDataType.INT), ARRAY_AGG(CnosDBDataType.STRING); + + private final CnosDBDataType[] supportedReturnTypes; + + CnosDBAggregateFunction(CnosDBDataType... supportedReturnTypes) { + this.supportedReturnTypes = supportedReturnTypes.clone(); + } + + public static List getAggregates(CnosDBDataType type) { + List res = Stream.of(values()).filter(p -> p.supportsReturnType(type)) + .collect(Collectors.toList()); + if (CnosDBBugs.BUG786) { + res.removeAll(List.of(VAR, VAR_POP, VAR_SAMP, STDDEV, STDDEV_POP, STDDEV_SAMP, CORR, COVAR, COVAR_POP, + COVAR_SAMP, APPROX_PERCENTILE_CONT_WITH_WEIGHT, APPROX_DISTINCT, APPROX_PERCENTILE_CONT, + APPROX_PERCENTILE_CONT_WITH_WEIGHT, GROUPING, ARRAY_AGG)); + } + + return res; + } + + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { returnType }; + } + + public boolean supportsReturnType(CnosDBDataType returnType) { + return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType) + || supportedReturnTypes.length == 0; + } + + public CnosDBDataType getRandomReturnType() { + if (supportedReturnTypes.length == 0) { + return Randomly.fromOptions(CnosDBDataType.getRandomType()); + } else { + return Randomly.fromOptions(supportedReturnTypes); + } + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAlias.java b/src/sqlancer/cnosdb/ast/CnosDBAlias.java new file mode 100644 index 000000000..86bba199f --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBAlias.java @@ -0,0 +1,35 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.common.visitor.UnaryOperation; + +public class CnosDBAlias implements UnaryOperation, CnosDBExpression { + + private final CnosDBExpression expr; + private final String alias; + + public CnosDBAlias(CnosDBExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + @Override + public CnosDBExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return " as " + alias; + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java new file mode 100644 index 000000000..d0addced1 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java @@ -0,0 +1,34 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public final class CnosDBBetweenOperation implements CnosDBExpression { + + private final CnosDBExpression expr; + private final CnosDBExpression left; + private final CnosDBExpression right; + + public CnosDBBetweenOperation(CnosDBExpression expr, CnosDBExpression left, CnosDBExpression right) { + this.expr = expr; + this.left = left; + this.right = right; + } + + public CnosDBExpression getExpr() { + return expr; + } + + public CnosDBExpression getLeft() { + return left; + } + + public CnosDBExpression getRight() { + return right; + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java new file mode 100644 index 000000000..acf3e93d5 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java @@ -0,0 +1,69 @@ +package sqlancer.cnosdb.ast; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; +import sqlancer.common.ast.BinaryOperatorNode; + +public class CnosDBBinaryArithmeticOperation extends BinaryOperatorNode + implements CnosDBExpression { + + public CnosDBBinaryArithmeticOperation(CnosDBExpression left, CnosDBExpression right, CnosDBBinaryOperator op) { + super(left, right, op); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.INT; + } + + public enum CnosDBBinaryOperator implements BinaryOperatorNode.Operator { + + ADDITION("+") { + }, + SUBTRACTION("-") { + }, + MULTIPLICATION("*") { + }, + DIVISION("/") { + + }, + MODULO("%") { + }, + EXPONENTIATION("^") { + }; + + private final String textRepresentation; + + CnosDBBinaryOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static CnosDBBinaryOperator getRandom(CnosDBDataType dataType) { + List ops = new ArrayList<>(Arrays.asList(values())); + switch (dataType) { + case DOUBLE: + case UINT: + case STRING: + ops.remove(EXPONENTIATION); + ops.remove(MODULO); + break; + default: + break; + } + + return Randomly.fromList(ops); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java new file mode 100644 index 000000000..af38849c9 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java @@ -0,0 +1,57 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator; +import sqlancer.common.ast.BinaryOperatorNode; + +public class CnosDBBinaryComparisonOperation + extends BinaryOperatorNode implements CnosDBExpression { + + public CnosDBBinaryComparisonOperation(CnosDBExpression left, CnosDBExpression right, + CnosDBBinaryComparisonOperator op) { + super(left, right, op); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + public enum CnosDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("=") { + }, + IS_DISTINCT("IS DISTINCT FROM") { + }, + IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { + }, + NOT_EQUALS("!=") { + }, + LESS("<") { + }, + LESS_EQUALS("<=") { + }, + GREATER(">") { + }, + GREATER_EQUALS(">=") { + + }; + + private final String textRepresentation; + + CnosDBBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static CnosDBBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(CnosDBBinaryComparisonOperator.values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java new file mode 100644 index 000000000..bad8a3b75 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java @@ -0,0 +1,33 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; +import sqlancer.common.ast.BinaryOperatorNode; + +public class CnosDBBinaryLogicalOperation extends BinaryOperatorNode + implements CnosDBExpression { + + public CnosDBBinaryLogicalOperation(CnosDBExpression left, CnosDBExpression right, BinaryLogicalOperator op) { + super(left, right, op); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + public enum BinaryLogicalOperator implements BinaryOperatorNode.Operator { + AND, OR; + + public static BinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java b/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java new file mode 100644 index 000000000..41db62d81 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java @@ -0,0 +1,60 @@ +package sqlancer.cnosdb.ast; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.cnosdb.CnosDBCompoundDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBCastOperation implements CnosDBExpression { + + private final CnosDBExpression expression; + private final CnosDBCompoundDataType type; + + public CnosDBCastOperation(CnosDBExpression expression, CnosDBCompoundDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + public static List canCastTo(CnosDBDataType dataType) { + List options = new ArrayList<>(Arrays.asList(CnosDBDataType.values())); + + switch (dataType) { + case UINT: + case BOOLEAN: + case DOUBLE: + options.remove(CnosDBDataType.TIMESTAMP); + break; + case TIMESTAMP: + options.remove(CnosDBDataType.BOOLEAN); + options.remove(CnosDBDataType.UINT); + options.remove(CnosDBDataType.DOUBLE); + break; + default: + break; + } + return options; + } + + @Override + public CnosDBDataType getExpressionType() { + return type.getDataType(); + } + + public CnosDBExpression getExpression() { + return expression; + } + + public CnosDBDataType getType() { + return type.getDataType(); + } + + public CnosDBCompoundDataType getCompoundType() { + return type; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java b/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java new file mode 100644 index 000000000..f90b6120f --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java @@ -0,0 +1,27 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBColumnValue implements CnosDBExpression { + + private final CnosDBColumn c; + + public CnosDBColumnValue(CnosDBColumn c) { + this.c = c; + } + + public static CnosDBColumnValue create(CnosDBColumn c) { + return new CnosDBColumnValue(c); + } + + @Override + public CnosDBDataType getExpressionType() { + return c.getType(); + } + + public CnosDBColumn getColumn() { + return c; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java b/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java new file mode 100644 index 000000000..6821f83b8 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java @@ -0,0 +1,22 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.common.ast.BinaryNode; + +public class CnosDBConcatOperation extends BinaryNode implements CnosDBExpression { + + public CnosDBConcatOperation(CnosDBExpression left, CnosDBExpression right) { + super(left, right); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.STRING; + } + + @Override + public String getOperatorRepresentation() { + return "||"; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConstant.java b/src/sqlancer/cnosdb/ast/CnosDBConstant.java new file mode 100644 index 000000000..42ecd3908 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBConstant.java @@ -0,0 +1,520 @@ +package sqlancer.cnosdb.ast; + +import java.math.BigDecimal; +import java.text.SimpleDateFormat; +import java.util.Date; + +import sqlancer.IgnoreMeException; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public abstract class CnosDBConstant implements CnosDBExpression { + + public static CnosDBConstant createNullConstant() { + return new CnosDBNullConstant(); + } + + public static CnosDBConstant createIntConstant(long val) { + return new IntConstant(val, false); + } + + public static CnosDBConstant createBooleanConstant(boolean val) { + return new BooleanConstant(val); + } + + public static CnosDBConstant createFalse() { + return createBooleanConstant(false); + } + + public static CnosDBConstant createTrue() { + return createBooleanConstant(true); + } + + public static CnosDBConstant createStringConstant(String string) { + return new StringConstant(string); + } + + public static CnosDBConstant createDoubleConstant(double val) { + return new DoubleConstant(val); + } + + public static CnosDBConstant createUintConstant(long val) { + return new IntConstant(val, true); + } + + public static CnosDBConstant createTimeStampConstant(long val) { + return new TimeStampConstant(val); + } + + public abstract String getTextRepresentation(); + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isString() { + return false; + } + + public boolean isNull() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asDouble() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isBoolean() { + return false; + } + + public abstract CnosDBConstant isEquals(CnosDBConstant rightVal); + + public boolean isInt() { + return false; + } + + protected abstract CnosDBConstant isLessThan(CnosDBConstant rightVal); + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract CnosDBConstant cast(CnosDBDataType type); + + public static class BooleanConstant extends CnosDBConstant { + + private final boolean value; + + public BooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return value ? "TRUE" : "FALSE"; + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return CnosDBConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(value == rightVal.cast(CnosDBDataType.BOOLEAN).asBoolean()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isString()) { + return isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); + } else { + assert rightVal.isBoolean(); + return CnosDBConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + switch (type) { + case BOOLEAN: + return this; + case INT: + return CnosDBConstant.createIntConstant(value ? 1 : 0); + case UINT: + return CnosDBConstant.createUintConstant(value ? 1 : 0); + case STRING: + return CnosDBConstant.createStringConstant(value ? "true" : "false"); + default: + return null; + } + } + + } + + public static class CnosDBNullConstant extends CnosDBConstant { + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + return CnosDBConstant.createNullConstant(); + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + return CnosDBConstant.createNullConstant(); + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + return CnosDBConstant.createNullConstant(); + } + } + + public static class StringConstant extends CnosDBConstant { + + private final String value; + + public StringConstant(String value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'", value.replace("'", "''")); + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(CnosDBDataType.INT).isEquals(rightVal.cast(CnosDBDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal.cast(CnosDBDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(CnosDBDataType.INT).isLessThan(rightVal.cast(CnosDBDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + if (type == CnosDBDataType.STRING) { + return this; + } + String s = value.trim(); + switch (type) { + case BOOLEAN: + try { + return CnosDBConstant.createBooleanConstant(Long.parseLong(s) != 0); + } catch (NumberFormatException ignored) { + } + switch (s.toUpperCase()) { + case "T": + case "TR": + case "TRU": + case "TRUE": + case "1": + case "YES": + case "YE": + case "Y": + case "ON": + return CnosDBConstant.createTrue(); + case "F": + case "FA": + case "FAL": + case "FALS": + case "FALSE": + case "N": + case "NO": + case "OF": + case "OFF": + default: + return CnosDBConstant.createFalse(); + } + case INT: + try { + return CnosDBConstant.createIntConstant(Long.parseLong(s)); + } catch (NumberFormatException e) { + return CnosDBConstant.createIntConstant(-1); + } + case UINT: + try { + return CnosDBConstant.createUintConstant(Long.parseUnsignedLong(s)); + } catch (NumberFormatException e) { + return CnosDBConstant.createUintConstant(0); + } + case DOUBLE: + try { + return CnosDBConstant.createDoubleConstant(Double.parseDouble(s)); + } catch (NumberFormatException e) { + return CnosDBConstant.createDoubleConstant(0.0); + } + + default: + return null; + } + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.STRING; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public String asString() { + return value; + } + + } + + public static class IntConstant extends CnosDBConstant { + + private final long val; + private final boolean unsigned; + + public IntConstant(long val, boolean unsigned) { + this.val = val; + this.unsigned = unsigned; + } + + @Override + public String getTextRepresentation() { + if (unsigned) { + return Long.toUnsignedString(val); + } else { + return String.valueOf(val); + } + } + + @Override + public CnosDBDataType getExpressionType() { + if (unsigned) { + return CnosDBDataType.UINT; + } + return CnosDBDataType.INT; + } + + @Override + public long asInt() { + return val; + } + + @Override + public double asDouble() { + return val; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); + } else if (rightVal.isInt()) { + return CnosDBConstant.createBooleanConstant(val == rightVal.asInt()); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.INT).asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); + } else if (rightVal.isBoolean()) { + throw new AssertionError(rightVal); + } else if (rightVal.getExpressionType() == CnosDBDataType.UINT) { + return CnosDBConstant.createBooleanConstant(Long.compareUnsigned(val, rightVal.asInt()) < 0); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.INT).asInt()); + } else { + throw new IgnoreMeException(); + } + + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + switch (type) { + case BOOLEAN: + return CnosDBConstant.createBooleanConstant(val != 0); + case INT: + return CnosDBConstant.createIntConstant(val); + case STRING: + return CnosDBConstant.createStringConstant(String.valueOf(val)); + case UINT: + return CnosDBConstant.createUintConstant(val); + case DOUBLE: + return CnosDBConstant.createDoubleConstant(val); + default: + return null; + } + } + } + + public static class TimeStampConstant extends CnosDBConstant { + final long val; + + TimeStampConstant(long time) { + val = time; + } + + @Override + public String getTextRepresentation() { + return "CAST (" + val + " AS TIMESTAMP)"; + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return createNullConstant(); + } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { + return createBooleanConstant(val == rightVal.asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { + return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + switch (type) { + case INT: + return createIntConstant(val); + case STRING: + final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + return CnosDBConstant.createStringConstant(dateFormat.format(new Date(val))); + default: + return null; + } + } + + @Override + public long asInt() { + return val; + } + + } + + public static class DoubleConstant extends CnosDBConstant { + + private final double val; + + public DoubleConstant(double val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + BigDecimal bigDecimal = new BigDecimal(val); + return bigDecimal.toPlainString(); + } else { + return String.valueOf(0.0); + } + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.DOUBLE; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal); + } else { + return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); + } + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); + } else { + return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); + } + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + return null; + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBExpression.java b/src/sqlancer/cnosdb/ast/CnosDBExpression.java new file mode 100644 index 000000000..63997a0f5 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBExpression.java @@ -0,0 +1,14 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public interface CnosDBExpression { + + default CnosDBDataType getExpressionType() { + return null; + } + + default CnosDBConstant getExpectedValue() { + throw new AssertionError("Not impl"); + } +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunction.java b/src/sqlancer/cnosdb/ast/CnosDBFunction.java new file mode 100644 index 000000000..7a35d703e --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBFunction.java @@ -0,0 +1,30 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBFunction implements CnosDBExpression { + + private final String func; + private final CnosDBExpression[] args; + private final CnosDBDataType returnType; + + public CnosDBFunction(CnosDBFunctionWithUnknownResult f, CnosDBDataType returnType, CnosDBExpression... args) { + this.func = f.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public String getFunctionName() { + return func; + } + + public CnosDBExpression[] getArguments() { + return args.clone(); + } + + @Override + public CnosDBDataType getExpressionType() { + return returnType; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java b/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java new file mode 100644 index 000000000..485f2309d --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java @@ -0,0 +1,104 @@ +package sqlancer.cnosdb.ast; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.cnosdb.CnosDBBugs; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; + +public enum CnosDBFunctionWithUnknownResult { + + // String functions + ASCII("ascii", CnosDBDataType.INT, CnosDBDataType.STRING), + BTRIM("btrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + CHAR_LENGTH("char_length", CnosDBDataType.INT, CnosDBDataType.STRING), + CHARACTER_LENGTH("character_length", CnosDBDataType.INT, CnosDBDataType.STRING), + CONCAT("concat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + CONCAT_WS("concat_ws", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + CHR("chr", CnosDBDataType.STRING, CnosDBDataType.INT), + BIT_LENGTH("bit_length", CnosDBDataType.INT, CnosDBDataType.STRING), + INITCAP("initcap", CnosDBDataType.STRING, CnosDBDataType.STRING), + + LEFT("left", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + LENGTH("length", CnosDBDataType.UINT, CnosDBDataType.STRING), + LOWER("lower", CnosDBDataType.STRING, CnosDBDataType.STRING), + UPPER("upper", CnosDBDataType.STRING, CnosDBDataType.STRING), + LPAD3("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), + LPAD2("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + RPAD3("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), + RPAD2("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + LTRIM("ltrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + OCTET_LENGTH("octet_length", CnosDBDataType.INT, CnosDBDataType.STRING), + // REPEAT("repeat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + REPLACE("replace", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + REVERSE("reverse", CnosDBDataType.STRING, CnosDBDataType.STRING), + RIGHT("right", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + RTRIM("rtrim", CnosDBDataType.STRING, CnosDBDataType.STRING), + SPLIT_PART("split_part", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + STARTS_WITH("starts_with", CnosDBDataType.BOOLEAN, CnosDBDataType.STRING, CnosDBDataType.STRING), + STRPOS("strpos", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.STRING), + SUBSTR("substr", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.INT), + TRANSLATE("translate", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + MD5("md5", CnosDBDataType.STRING, CnosDBDataType.STRING), + // mathematical functions + ABS("abs", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + CEIL("ceil", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + EXP("exp", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), LN("ln", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + LOG2("log2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + LOG10("log10", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + POWER("power", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + ROUND("round", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + TRUNC("trunc", CnosDBDataType.DOUBLE, CnosDBDataType.INT), + FLOOR("floor", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + SIGNUM("signum", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + ACOS("acos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + ASIN("asin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + ATAN2("atan2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + COS("cos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), SIN("sin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + SQRT("sqrt", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + TAN("tan", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + DATE_PART("date_part", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP), + TO_TIMESTAMP("to_timestamp", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), + TO_TIMESTAMP_MILLIS("to_timestamp_millis", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), + TO_TIMESTAMP_MICROS("to_timestamp_micros", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), + TO_TIMESTAMP_SECONDS("to_timestamp_seconds", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT); + + private final String functionName; + private final CnosDBDataType returnType; + private final CnosDBDataType[] argTypes; + + CnosDBFunctionWithUnknownResult(String functionName, CnosDBDataType returnType, CnosDBDataType... indexType) { + this.functionName = functionName; + this.returnType = returnType; + this.argTypes = indexType.clone(); + + } + + public static List getSupportedFunctions(CnosDBDataType type) { + List res = Stream.of(values()) + .filter(function -> function.isCompatibleWithReturnType(type)).collect(Collectors.toList()); + if (CnosDBBugs.BUG3547) { + res.removeAll(List.of(TO_TIMESTAMP, TO_TIMESTAMP_MICROS, TO_TIMESTAMP_MILLIS, TO_TIMESTAMP_SECONDS)); + } + return res; + } + + public boolean isCompatibleWithReturnType(CnosDBDataType t) { + return t == returnType; + } + + public CnosDBExpression[] getArguments(CnosDBDataType ignore, CnosDBExpressionGenerator gen, int depth) { + CnosDBExpression[] args = new CnosDBExpression[argTypes.length]; + for (int i = 0; i < args.length; i++) { + args[i] = gen.generateExpression(depth, argTypes[i]); + } + return args; + } + + public String getName() { + return functionName; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBInOperation.java b/src/sqlancer/cnosdb/ast/CnosDBInOperation.java new file mode 100644 index 000000000..c0ffd34ed --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBInOperation.java @@ -0,0 +1,35 @@ +package sqlancer.cnosdb.ast; + +import java.util.List; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBInOperation implements CnosDBExpression { + + private final CnosDBExpression expr; + private final List listElements; + private final boolean isTrue; + + public CnosDBInOperation(CnosDBExpression expr, List listElements, boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public CnosDBExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + public boolean isTrue() { + return isTrue; + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBJoin.java b/src/sqlancer/cnosdb/ast/CnosDBJoin.java new file mode 100644 index 000000000..eea88466f --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBJoin.java @@ -0,0 +1,46 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBJoin implements CnosDBExpression { + + private final CnosDBExpression tableReference; + private final CnosDBExpression onClause; + private final CnosDBJoinType type; + + public CnosDBJoin(CnosDBExpression tableReference, CnosDBExpression onClause, CnosDBJoinType type) { + this.tableReference = tableReference; + this.onClause = onClause; + this.type = type; + } + + public CnosDBExpression getTableReference() { + return tableReference; + } + + public CnosDBExpression getOnClause() { + return onClause; + } + + public CnosDBJoinType getType() { + return type; + } + + @Override + public CnosDBDataType getExpressionType() { + throw new AssertionError(); + } + + public enum CnosDBJoinType { + INNER, LEFT, RIGHT, FULL; + // now not support + // CROSS; + + public static CnosDBJoinType getRandom() { + return Randomly.fromOptions(values()); + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java b/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java new file mode 100644 index 000000000..616cd39ee --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java @@ -0,0 +1,22 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.common.ast.BinaryNode; + +public class CnosDBLikeOperation extends BinaryNode implements CnosDBExpression { + + public CnosDBLikeOperation(CnosDBExpression left, CnosDBExpression right) { + super(left, right); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + @Override + public String getOperatorRepresentation() { + return "LIKE"; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java b/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java new file mode 100644 index 000000000..de5812d76 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java @@ -0,0 +1,37 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBOrderByTerm implements CnosDBExpression { + + private final CnosDBOrder order; + private final CnosDBExpression expr; + + public CnosDBOrderByTerm(CnosDBExpression expr, CnosDBOrder order) { + this.expr = expr; + this.order = order; + } + + public CnosDBOrder getOrder() { + return order; + } + + public CnosDBExpression getExpr() { + return expr; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + + public enum CnosDBOrder { + ASC, DESC; + + public static CnosDBOrder getRandomOrder() { + return Randomly.fromOptions(CnosDBOrder.values()); + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java new file mode 100644 index 000000000..f37621f44 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java @@ -0,0 +1,97 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.common.ast.BinaryOperatorNode.Operator; + +public class CnosDBPostfixOperation implements CnosDBExpression { + + private final CnosDBExpression expr; + private final String operatorTextRepresentation; + + public CnosDBPostfixOperation(CnosDBExpression expr, PostfixOperator op) { + this.expr = expr; + this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); + } + + public static CnosDBExpression create(CnosDBExpression expr, PostfixOperator op) { + return new CnosDBPostfixOperation(expr, op); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + public String getOperatorTextRepresentation() { + return operatorTextRepresentation; + } + + public CnosDBExpression getExpression() { + return expr; + } + + public enum PostfixOperator implements Operator { + IS_NULL("IS NULL"/* , "ISNULL" */) { + @Override + public CnosDBDataType[] getInputDataTypes() { + return CnosDBDataType.values(); + } + + }, + IS_UNKNOWN("IS UNKNOWN") { + @Override + public CnosDBDataType[] getInputDataTypes() { + return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; + } + }, + + IS_NOT_NULL("IS NOT NULL"/* "NOTNULL" */) { + + @Override + public CnosDBDataType[] getInputDataTypes() { + return CnosDBDataType.values(); + } + + }, + IS_NOT_UNKNOWN("IS NOT UNKNOWN") { + + @Override + public CnosDBDataType[] getInputDataTypes() { + return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; + } + }, + IS_TRUE("IS TRUE") { + @Override + public CnosDBDataType[] getInputDataTypes() { + return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; + } + + }, + IS_FALSE("IS FALSE") { + @Override + public CnosDBDataType[] getInputDataTypes() { + return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; + } + + }; + + private final String[] textRepresentations; + + PostfixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public static PostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract CnosDBDataType[] getInputDataTypes(); + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java new file mode 100644 index 000000000..241fab89a --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java @@ -0,0 +1,29 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBPostfixText implements CnosDBExpression { + + private final CnosDBExpression expr; + private final String text; + private final CnosDBDataType type; + + public CnosDBPostfixText(CnosDBExpression expr, String text, CnosDBDataType type) { + this.expr = expr; + this.text = text; + this.type = type; + } + + public CnosDBExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + @Override + public CnosDBDataType getExpressionType() { + return type; + } +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java new file mode 100644 index 000000000..db37f0089 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java @@ -0,0 +1,73 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.common.ast.BinaryOperatorNode.Operator; + +public class CnosDBPrefixOperation implements CnosDBExpression { + + private final CnosDBExpression expr; + private final PrefixOperator op; + + public CnosDBPrefixOperation(CnosDBExpression expr, PrefixOperator op) { + this.expr = expr; + this.op = op; + } + + @Override + public CnosDBDataType getExpressionType() { + return op.getExpressionType(); + } + + public CnosDBDataType[] getInputDataTypes() { + return op.dataTypes; + } + + public String getTextRepresentation() { + return op.textRepresentation; + } + + public CnosDBExpression getExpression() { + return expr; + } + + public enum PrefixOperator implements Operator { + NOT("NOT", CnosDBDataType.BOOLEAN) { + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + }, + UNARY_PLUS("+", CnosDBDataType.INT) { + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.INT; + } + + }, + UNARY_MINUS("-", CnosDBDataType.INT) { + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.INT; + } + + }; + + private final String textRepresentation; + private final CnosDBDataType[] dataTypes; + + PrefixOperator(String textRepresentation, CnosDBDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract CnosDBDataType getExpressionType(); + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSelect.java b/src/sqlancer/cnosdb/ast/CnosDBSelect.java new file mode 100644 index 000000000..0db657f19 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBSelect.java @@ -0,0 +1,102 @@ +package sqlancer.cnosdb.ast; + +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.common.ast.SelectBase; + +public class CnosDBSelect extends SelectBase implements CnosDBExpression { + + private SelectType selectOption = SelectType.ALL; + private List joinClauses = Collections.emptyList(); + private CnosDBExpression distinctOnClause; + + public void setSelectType(SelectType fromOptions) { + this.setSelectOption(fromOptions); + } + + public SelectType getSelectOption() { + return selectOption; + } + + public void setSelectOption(SelectType fromOptions) { + this.selectOption = fromOptions; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + + public List getJoinClauses() { + return joinClauses; + } + + public void setJoinClauses(List joinStatements) { + this.joinClauses = joinStatements; + + } + + public CnosDBExpression getDistinctOnClause() { + return distinctOnClause; + } + + public void setDistinctOnClause(CnosDBExpression distinctOnClause) { + if (selectOption != SelectType.DISTINCT) { + throw new IllegalArgumentException(); + } + this.distinctOnClause = distinctOnClause; + } + + public enum SelectType { + DISTINCT, ALL; + + public static SelectType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public static class CnosDBFromTable implements CnosDBExpression { + private final CnosDBTable t; + + public CnosDBFromTable(CnosDBTable t) { + this.t = t; + } + + public CnosDBTable getTable() { + return t; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + } + + public static class CnosDBSubquery implements CnosDBExpression { + private final CnosDBSelect s; + private final String name; + + public CnosDBSubquery(CnosDBSelect s, String name) { + this.s = s; + this.name = name; + } + + public CnosDBSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java b/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java new file mode 100644 index 000000000..9e3467ada --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java @@ -0,0 +1,28 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBSimilarTo implements CnosDBExpression { + + private final CnosDBExpression string; + private final CnosDBExpression similarTo; + + public CnosDBSimilarTo(CnosDBExpression string, CnosDBExpression similarTo) { + this.string = string; + this.similarTo = similarTo; + } + + public CnosDBExpression getString() { + return string; + } + + public CnosDBExpression getSimilarTo() { + return similarTo; + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/cnosdb/client/CnosDBClient.java b/src/sqlancer/cnosdb/client/CnosDBClient.java new file mode 100644 index 000000000..ccc9dcc16 --- /dev/null +++ b/src/sqlancer/cnosdb/client/CnosDBClient.java @@ -0,0 +1,110 @@ +package sqlancer.cnosdb.client; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.codec.binary.Base64; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; + +import com.arangodb.internal.util.IOUtils; + +public class CnosDBClient { + private final String userName; + private final String password; + private final String host; + private final int port; + + private final String database; + private final CloseableHttpClient client; + + public CnosDBClient(String host, int port, String userName, String password, String database) { + this.host = host; + this.port = port; + this.userName = userName; + this.password = password; + this.database = database; + this.client = HttpClientBuilder.create().build(); + } + + private String url() { + return "http://" + host + ":" + port + "/api/v1/"; + } + + public String ping() throws Exception { + HttpGet httpGet = new HttpGet(this.url() + "ping"); + httpGet.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); + CloseableHttpResponse resp = client.execute(httpGet); + + String content = IOUtils.toString(resp.getEntity().getContent()); + resp.close(); + return content; + } + + public CnosDBResultSet executeQuery(String query) throws Exception { + HttpUriRequest request = createRequest(query); + CloseableHttpResponse resp = client.execute(request); + String text = IOUtils.toString(resp.getEntity().getContent()); + if (resp.getStatusLine().getStatusCode() != 200) { + resp.close(); + throw new CnosDBException(database + ":" + query + ";\n" + text); + } + resp.close(); + InputStream stream = new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); + + return new CnosDBResultSet(new InputStreamReader(stream)); + } + + public boolean execute(String query) throws Exception { + HttpUriRequest request = createRequest(query); + CloseableHttpResponse resp = client.execute(request); + if (resp.getStatusLine().getStatusCode() != 200) { + String res = IOUtils.toString(resp.getEntity().getContent()); + resp.close(); + throw new CnosDBException(query + res); + } + resp.close(); + return true; + } + + public void close() throws IOException { + client.close(); + } + + public String getDatabase() { + return this.database; + } + + private String getAuth() { + String auth = userName + ":" + password; + byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.ISO_8859_1)); + return "Basic " + new String(encodedAuth); + + } + + private HttpUriRequest createRequest(String query) throws URISyntaxException, UnsupportedEncodingException { + + URIBuilder builder = new URIBuilder(this.url() + "sql"); + builder.setParameter("db", database); + builder.setParameter("pretty", "true"); + HttpPost httpPost = new HttpPost(builder.build()); + + httpPost.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); + StringEntity stringEntity = new StringEntity(query); + httpPost.setEntity(stringEntity); + return httpPost; + } + +} diff --git a/src/sqlancer/cnosdb/client/CnosDBConnection.java b/src/sqlancer/cnosdb/client/CnosDBConnection.java new file mode 100644 index 000000000..9277f203b --- /dev/null +++ b/src/sqlancer/cnosdb/client/CnosDBConnection.java @@ -0,0 +1,27 @@ +package sqlancer.cnosdb.client; + +import java.io.IOException; + +import sqlancer.SQLancerDBConnection; + +public class CnosDBConnection implements SQLancerDBConnection { + private final CnosDBClient client; + + public CnosDBConnection(CnosDBClient client) { + this.client = client; + } + + @Override + public String getDatabaseVersion() throws Exception { + return client.ping(); + } + + public CnosDBClient getClient() { + return client; + } + + @Override + public void close() throws IOException { + client.close(); + } +} diff --git a/src/sqlancer/cnosdb/client/CnosDBException.java b/src/sqlancer/cnosdb/client/CnosDBException.java new file mode 100644 index 000000000..a1055e90b --- /dev/null +++ b/src/sqlancer/cnosdb/client/CnosDBException.java @@ -0,0 +1,9 @@ +package sqlancer.cnosdb.client; + +public class CnosDBException extends RuntimeException { + private static final long serialVersionUID = 1L; + + CnosDBException(String message) { + super(message); + } +} diff --git a/src/sqlancer/cnosdb/client/CnosDBResultSet.java b/src/sqlancer/cnosdb/client/CnosDBResultSet.java new file mode 100644 index 000000000..877b6ba5d --- /dev/null +++ b/src/sqlancer/cnosdb/client/CnosDBResultSet.java @@ -0,0 +1,52 @@ +package sqlancer.cnosdb.client; + +import java.io.Reader; +import java.sql.SQLException; +import java.util.Iterator; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; + +import sqlancer.IgnoreMeException; + +public class CnosDBResultSet { + private final Iterator records; + private CSVRecord next; + + public CnosDBResultSet(Reader in) throws Exception { + Iterable records = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).build() + .parse(in); + this.records = records.iterator(); + } + + public void close() { + } + + public boolean next() throws SQLException { + if (records.hasNext()) { + next = records.next(); + return true; + } + return false; + } + + public int getInt(int i) throws SQLException { + return Integer.parseInt(next.get(i - 1)); + } + + public String getString(int i) throws SQLException { + return next.get(i - 1); + } + + public long getLong(int i) throws SQLException { + if (next.get(i - 1).isEmpty()) { + throw new IgnoreMeException(); + } + return Long.parseLong(next.get(i - 1)); + } + + // public boolean getBool(int i) throws Exception { + // return Boolean.parseBoolean(getString(i)); + // } + +} diff --git a/src/sqlancer/cnosdb/gen/CnosDBCommon.java b/src/sqlancer/cnosdb/gen/CnosDBCommon.java new file mode 100644 index 000000000..6c7b0bba7 --- /dev/null +++ b/src/sqlancer/cnosdb/gen/CnosDBCommon.java @@ -0,0 +1,31 @@ +package sqlancer.cnosdb.gen; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public final class CnosDBCommon { + + private CnosDBCommon() { + } + + public static void appendDataType(CnosDBDataType type, StringBuilder sb) throws AssertionError { + switch (type) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: + sb.append("BIGINT"); + break; + case STRING: + sb.append("STRING"); + break; + case DOUBLE: + sb.append("DOUBLE"); + break; + case UINT: + sb.append("BIGINT UNSIGNED"); + break; + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java new file mode 100644 index 000000000..121f78254 --- /dev/null +++ b/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java @@ -0,0 +1,461 @@ +package sqlancer.cnosdb.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBCompoundDataType; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBAggregate; +import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; +import sqlancer.cnosdb.ast.CnosDBBetweenOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; +import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; +import sqlancer.cnosdb.ast.CnosDBCastOperation; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBConcatOperation; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBFunction; +import sqlancer.cnosdb.ast.CnosDBFunctionWithUnknownResult; +import sqlancer.cnosdb.ast.CnosDBInOperation; +import sqlancer.cnosdb.ast.CnosDBLikeOperation; +import sqlancer.cnosdb.ast.CnosDBOrderByTerm; +import sqlancer.cnosdb.ast.CnosDBOrderByTerm.CnosDBOrder; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; +import sqlancer.cnosdb.ast.CnosDBSimilarTo; +import sqlancer.common.gen.ExpressionGenerator; + +public class CnosDBExpressionGenerator implements ExpressionGenerator { + + private final int maxDepth; + + private final Randomly r; + + private List columns; + + private boolean allowAggregateFunctions; + + public CnosDBExpressionGenerator(CnosDBGlobalState globalState) { + this.r = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + } + + public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, CnosDBDataType type) { + return new CnosDBExpressionGenerator(globalState).generateExpression(0, type); + } + + private static CnosDBCompoundDataType getCompoundDataType(CnosDBDataType type) { + return CnosDBCompoundDataType.create(type); + } + + public static CnosDBExpression generateConstant(Randomly r, CnosDBDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return CnosDBConstant.createNullConstant(); + } + switch (type) { + case INT: + return CnosDBConstant.createIntConstant(r.getInteger()); + case UINT: + return CnosDBConstant.createUintConstant(r.getPositiveInteger()); + case TIMESTAMP: + return CnosDBConstant.createTimeStampConstant(r.getPositiveIntegerNotNull()); + case BOOLEAN: + return CnosDBConstant.createBooleanConstant(Randomly.getBoolean()); + case STRING: + return CnosDBConstant.createStringConstant(r.getString()); + case DOUBLE: + return CnosDBConstant.createDoubleConstant(r.getDouble()); + default: + throw new AssertionError(type); + } + } + + public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns, + CnosDBDataType type) { + return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); + } + + public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns) { + return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0); + } + + public CnosDBExpressionGenerator setColumns(List columns) { + this.columns = columns; + return this; + } + + public CnosDBExpression generateExpression(int depth) { + return generateExpression(depth, CnosDBDataType.getRandomType()); + } + + public List generateOrderBy() { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new CnosDBOrderByTerm(CnosDBColumnValue.create(Randomly.fromList(columns)), + CnosDBOrder.getRandomOrder())); + } + return orderBys; + } + + private CnosDBExpression generateFunctionWithUnknownResult(int depth, CnosDBDataType type) { + List supportedFunctions = CnosDBFunctionWithUnknownResult + .getSupportedFunctions(type); + if (supportedFunctions.isEmpty()) { + throw new IgnoreMeException(); + } + CnosDBFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); + return new CnosDBFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); + } + + private CnosDBExpression generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + PostfixOperator random = PostfixOperator.getRandom(); + return CnosDBPostfixOperation + .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); + case IN_OPERATION: + return inOperation(depth + 1); + case NOT: + return new CnosDBPrefixOperation(generateExpression(depth + 1, CnosDBDataType.BOOLEAN), PrefixOperator.NOT); + case BINARY_LOGICAL_OPERATOR: + CnosDBExpression first = generateExpression(depth + 1, CnosDBDataType.BOOLEAN); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + first = new CnosDBBinaryLogicalOperation(first, generateExpression(depth + 1, CnosDBDataType.BOOLEAN), + BinaryLogicalOperator.getRandom()); + } + return first; + case BINARY_COMPARISON: + CnosDBDataType dataType = getMeaningfulType(); + return generateComparison(depth, dataType); + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.BOOLEAN); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.BOOLEAN); + case LIKE: + return new CnosDBLikeOperation(generateExpression(depth + 1, CnosDBDataType.STRING), + generateExpression(depth + 1, CnosDBDataType.STRING)); + case BETWEEN: + CnosDBDataType type = getMeaningfulType(); + return new CnosDBBetweenOperation(generateExpression(depth + 1, type), generateExpression(depth + 1, type), + generateExpression(depth + 1, type)); + case SIMILAR_TO: + return new CnosDBSimilarTo(generateExpression(depth + 1, CnosDBDataType.STRING), + generateExpression(depth + 1, CnosDBDataType.STRING)); + default: + throw new AssertionError(); + } + } + + private CnosDBDataType getMeaningfulType() { + // make it more likely that the expression does not only consist of constant + // expressions + if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { + return CnosDBDataType.getRandomType(); + } else { + return Randomly.fromList(columns).getType(); + } + } + + private CnosDBExpression generateFunction(int depth, CnosDBDataType type) { + return generateFunctionWithUnknownResult(depth, type); + } + + private CnosDBExpression generateComparison(int depth, CnosDBDataType dataType) { + CnosDBExpression leftExpr = generateExpression(depth + 1, dataType); + CnosDBExpression rightExpr = generateExpression(depth + 1, dataType); + return getComparison(leftExpr, rightExpr); + } + + private CnosDBExpression getComparison(CnosDBExpression leftExpr, CnosDBExpression rightExpr) { + return new CnosDBBinaryComparisonOperation(leftExpr, rightExpr, + CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator.getRandom()); + } + + private CnosDBExpression inOperation(int depth) { + CnosDBDataType type = CnosDBDataType.getRandomType(); + CnosDBExpression leftExpr = generateExpression(depth + 1, type); + List rightExpr = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + rightExpr.add(generateConstant(new Randomly(), type)); + } + return new CnosDBInOperation(leftExpr, rightExpr, Randomly.getBoolean()); + } + + public CnosDBExpression generateExpression(int depth, CnosDBDataType originalType) { + return generateExpressionInternal(depth, originalType); + } + + private CnosDBExpression generateExpressionInternal(int depth, CnosDBDataType dataType) throws AssertionError { + if (allowAggregateFunctions && Randomly.getBoolean()) { + return getAggregate(dataType); + } + + if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { + // generic expression + if (Randomly.getBoolean() || depth > maxDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(r, dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(r, dataType); + } else { + return createColumnOfType(dataType); + } + } + } else { + if (Randomly.getBoolean()) { + return generateCastExpression(depth + 1, dataType); + } else { + return generateFunctionWithUnknownResult(depth, dataType); + } + } + } else { + switch (dataType) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case UINT: + return generateUIntExpression(depth); + case STRING: + return generateStringExpression(depth); + case DOUBLE: + return generateFloatExpression(depth); + case TIMESTAMP: + return generateTimeStampExpression(depth); + default: + throw new AssertionError(dataType); + } + } + } + + private CnosDBExpression generateStringExpression(int depth) { + StringExpression option; + List validOptions = new ArrayList<>(Arrays.asList(StringExpression.values())); + option = Randomly.fromList(validOptions); + + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.STRING); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.STRING); + case CONCAT: + return generateConcat(depth); + default: + throw new AssertionError(); + } + } + + private CnosDBExpression generateConcat(int depth) { + CnosDBExpression left = generateExpression(depth + 1, CnosDBDataType.STRING); + CnosDBExpression right = generateExpression(depth + 1); + return new CnosDBConcatOperation(left, right); + } + + private CnosDBExpression generateIntExpression(int depth) { + IntExpression option; + option = Randomly.fromOptions(IntExpression.values()); + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.INT); + case UNARY_OPERATION: + CnosDBExpression intExpression = generateExpression(depth + 1, CnosDBDataType.INT); + return new CnosDBPrefixOperation(intExpression, + Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.INT); + case BINARY_ARITHMETIC_EXPRESSION: + return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.INT), + generateExpression(depth + 1, CnosDBDataType.INT), + CnosDBBinaryOperator.getRandom(CnosDBDataType.INT)); + default: + throw new AssertionError(); + } + } + + private CnosDBExpression generateUIntExpression(int depth) { + UIntExpression option = Randomly.fromOptions(UIntExpression.values()); + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.UINT); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.UINT); + case BINARY_ARITHMETIC_EXPRESSION: + return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.UINT), + generateExpression(depth + 1, CnosDBDataType.UINT), + CnosDBBinaryOperator.getRandom(CnosDBDataType.UINT)); + default: + throw new AssertionError(); + } + + } + + private CnosDBExpression generateFloatExpression(int depth) { + FloatExpression option; + option = Randomly.fromOptions(FloatExpression.values()); + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.DOUBLE); + case UNARY_OPERATION: + CnosDBExpression floatExpression = generateExpression(depth + 1, CnosDBDataType.DOUBLE); + return new CnosDBPrefixOperation(floatExpression, + Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.DOUBLE); + case BINARY_ARITHMETIC_EXPRESSION: + return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.DOUBLE), + generateExpression(depth + 1, CnosDBDataType.DOUBLE), + CnosDBBinaryOperator.getRandom(CnosDBDataType.DOUBLE)); + case CONSTANT: + return generateConstant(r, CnosDBDataType.DOUBLE); + default: + throw new AssertionError(); + } + } + + private CnosDBExpression generateTimeStampExpression(int depth) { + if (Randomly.getBoolean()) { + return generateConstant(r, CnosDBDataType.TIMESTAMP); + } + TimestampExpression option; + option = Randomly.fromOptions(TimestampExpression.values()); + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.TIMESTAMP); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.TIMESTAMP); + default: + throw new AssertionError(); + } + } + + private CnosDBExpression generateCastExpression(int depth, CnosDBDataType castToType) { + CnosDBDataType castFromType = Randomly.fromList(CnosDBCastOperation.canCastTo(castToType)); + return new CnosDBCastOperation(generateExpression(depth + 1, castFromType), getCompoundDataType(castToType)); + } + + private CnosDBExpression createColumnOfType(CnosDBDataType type) { + List columns = filterColumns(type); + if (columns.isEmpty()) { + throw new IgnoreMeException(); + } + CnosDBColumn fromList = Randomly.fromList(columns); + return CnosDBColumnValue.create(fromList); + } + + final List filterColumns(CnosDBDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + public List generateExpressions(int nr) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(0)); + } + return expressions; + } + + public CnosDBExpression generateExpression(CnosDBDataType dataType) { + return generateExpression(0, dataType); + } + + public CnosDBExpression generateHavingClause() { + this.allowAggregateFunctions = true; + CnosDBExpression expression = generateExpression(CnosDBDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + + public CnosDBExpression generateAggregate() { + return getAggregate(CnosDBDataType.getRandomType()); + } + + private CnosDBExpression getAggregate(CnosDBDataType dataType) { + if (dataType == CnosDBDataType.BOOLEAN) { + List aggregates = CnosDBAggregateFunction.getAggregates(CnosDBDataType.INT); + CnosDBAggregateFunction agg = Randomly.fromList(aggregates); + return new CnosDBCastOperation(generateArgsForAggregate(dataType, agg), + CnosDBCompoundDataType.create(CnosDBDataType.BOOLEAN)); + } else { + List aggregates = CnosDBAggregateFunction.getAggregates(dataType); + CnosDBAggregateFunction agg = Randomly.fromList(aggregates); + return generateArgsForAggregate(dataType, agg); + } + } + + public CnosDBAggregate generateArgsForAggregate(CnosDBDataType dataType, CnosDBAggregateFunction agg) { + CnosDBDataType[] types = agg.getArgsTypes(dataType); + List args = new ArrayList<>(); + for (CnosDBDataType argType : types) { + args.add(createColumnOfType(argType)); + // args.add(generateExpression(argType)); + } + return new CnosDBAggregate(args, agg); + } + + public CnosDBExpressionGenerator allowAggregates(boolean value) { + allowAggregateFunctions = value; + return this; + } + + @Override + public CnosDBExpression generatePredicate() { + return generateExpression(CnosDBDataType.BOOLEAN); + } + + @Override + public CnosDBExpression negatePredicate(CnosDBExpression predicate) { + return new CnosDBPrefixOperation(predicate, PrefixOperator.NOT); + } + + @Override + public CnosDBExpression isNull(CnosDBExpression expr) { + return new CnosDBPostfixOperation(expr, PostfixOperator.IS_NULL); + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, CAST, LIKE, BETWEEN, IN_OPERATION, + SIMILAR_TO, + } + + private enum StringExpression { + CAST, FUNCTION, CONCAT + } + + private enum IntExpression { + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION + } + + private enum UIntExpression { + FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION + } + + private enum FloatExpression { + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, CONSTANT + } + + private enum TimestampExpression { + FUNCTION, CAST + } + +} diff --git a/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java new file mode 100644 index 000000000..0d575d3c7 --- /dev/null +++ b/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java @@ -0,0 +1,59 @@ +package sqlancer.cnosdb.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.cnosdb.CnosDBVisitor; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.query.CnosDBOtherQuery; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; + +public final class CnosDBInsertGenerator { + + private CnosDBInsertGenerator() { + } + + public static CnosDBOtherQuery insert(CnosDBGlobalState globalState) { + CnosDBTable table = globalState.getSchema().getRandomTable(); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("Column time cannot be null."); + StringBuilder sb = new StringBuilder(); + sb.append("INSERT "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES"); + + int n = Randomly.smallNumber() + 1; + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + insertRow(globalState, sb, columns); + } + + // error + return new CnosDBOtherQuery(sb.toString(), errors); + } + + private static void insertRow(CnosDBGlobalState globalState, StringBuilder sb, List columns) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i > 0) { + sb.append(", "); + } + CnosDBExpression generateConstant = CnosDBExpressionGenerator.generateConstant(globalState.getRandomly(), + columns.get(i).getType()); + sb.append(CnosDBVisitor.asString(generateConstant)); + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java new file mode 100644 index 000000000..c046ad3e9 --- /dev/null +++ b/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java @@ -0,0 +1,77 @@ +package sqlancer.cnosdb.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBFieldColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTagColumn; +import sqlancer.cnosdb.query.CnosDBOtherQuery; +import sqlancer.common.query.ExpectedErrors; + +public class CnosDBTableGenerator { + + protected final ExpectedErrors errors = new ExpectedErrors(); + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + private final List columnsToBeAdd = new ArrayList<>(); + private CnosDBTable table; + + public CnosDBTableGenerator(String tableName) { + this.tableName = tableName; + } + + public static CnosDBOtherQuery generate(String tableName) { + return new CnosDBTableGenerator(tableName).generate(); + } + + protected CnosDBOtherQuery generate() { + table = new CnosDBTable(tableName, columnsToBeAdd); + + sb.append("CREATE TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + + sb.append("("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String name = String.format("f%d", i); + createField(name); + sb.append(", "); + } + + sb.append("TAGS("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + String name = String.format("t%d", i); + createTag(name); + } + sb.append("))"); + return new CnosDBOtherQuery(sb.toString(), new ExpectedErrors()); + } + + private void createField(String name) throws AssertionError { + sb.append(name); + sb.append(" "); + CnosDBDataType type = CnosDBDataType.getRandomTypeWithoutTimeStamp(); + CnosDBCommon.appendDataType(type, sb); + CnosDBFieldColumn c = new CnosDBFieldColumn(name, type); + c.setTable(table); + sb.append(" "); + columnsToBeAdd.add(c); + } + + private void createTag(String name) { + sb.append(name); + CnosDBColumn column = new CnosDBTagColumn(name); + column.setTable(table); + columnsToBeAdd.add(column); + } +} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java new file mode 100644 index 000000000..472aa8f66 --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java @@ -0,0 +1,23 @@ +package sqlancer.cnosdb.oracle; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.common.oracle.TestOracle; + +public abstract class CnosDBNoRECBase implements TestOracle { + protected final CnosDBGlobalState state; + protected final Main.StateLogger logger; + protected final MainOptions options; + protected final CnosDBConnection con; + protected String optimizedQueryString; + protected String unoptimizedQueryString; + + public CnosDBNoRECBase(CnosDBGlobalState state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } +} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java new file mode 100644 index 000000000..df91e9a58 --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java @@ -0,0 +1,171 @@ +package sqlancer.cnosdb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBCompoundDataType; +import sqlancer.cnosdb.CnosDBExpectedError; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; +import sqlancer.cnosdb.CnosDBVisitor; +import sqlancer.cnosdb.ast.CnosDBCastOperation; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBJoin; +import sqlancer.cnosdb.ast.CnosDBJoin.CnosDBJoinType; +import sqlancer.cnosdb.ast.CnosDBPostfixText; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; +import sqlancer.cnosdb.ast.CnosDBSelect.SelectType; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPBase; +import sqlancer.cnosdb.query.CnosDBSelectQuery; +import sqlancer.common.oracle.TestOracle; + +public class CnosDBNoRECOracle extends CnosDBNoRECBase implements TestOracle { + + private final CnosDBSchema s; + + public CnosDBNoRECOracle(CnosDBGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + } + + public static List getJoinStatements(CnosDBGlobalState globalState, List columns, + List tables) { + List joinStatements = new ArrayList<>(); + CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); + CnosDBTable table = Randomly.fromList(tables); + tables.remove(table); + CnosDBJoinType options = CnosDBJoinType.getRandom(); + CnosDBJoin j = new CnosDBJoin(new CnosDBFromTable(table), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + CnosDBTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + CnosDBSubquery subquery = CnosDBTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); + CnosDBJoinType options = CnosDBJoinType.getRandom(); + CnosDBJoin j = new CnosDBJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public void check() throws Exception { + CnosDBTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + CnosDBExpression randomWhereCondition = getRandomWhereCondition(columns); + List tables = randomTables.getTables(); + + List joinStatements = getJoinStatements(state, columns, tables); + List fromTables = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); + int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); + int firstCount = getOptimizedQueryCount(fromTables, List.of(CnosDBColumn.createDummy("f0")), + randomWhereCondition, joinStatements); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, + firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + private CnosDBExpression getRandomWhereCondition(List columns) { + return new CnosDBExpressionGenerator(state).setColumns(columns).generateExpression(CnosDBDataType.BOOLEAN); + } + + private int getUnoptimizedQueryCount(List fromTables, CnosDBExpression randomWhereCondition, + List joinStatements) throws Exception { + CnosDBSelect select = new CnosDBSelect(); + CnosDBCastOperation isTrue = new CnosDBCastOperation(randomWhereCondition, + CnosDBCompoundDataType.create(CnosDBDataType.INT)); + CnosDBPostfixText asText = new CnosDBPostfixText(isTrue, " as count", CnosDBDataType.INT); + select.setFetchColumns(List.of(asText)); + select.setFromList(fromTables); + select.setSelectType(SelectType.ALL); + select.setJoinClauses(joinStatements); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + CnosDBVisitor.asString(select) + ") as res"; + if (options.logEachSelect()) { + logger.writeCurrent(unoptimizedQueryString); + } + CnosDBSelectQuery q = new CnosDBSelectQuery(unoptimizedQueryString, CnosDBExpectedError.expectedErrors()); + CnosDBResultSet rs; + try { + q.executeAndGet(state); + rs = q.getResultSet(); + } catch (Exception e) { + if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getOptimizedQueryCount(List randomTables, List columns, + CnosDBExpression randomWhereCondition, List joinStatements) { + CnosDBSelect select = new CnosDBSelect(); + CnosDBColumnValue allColumns = new CnosDBColumnValue(Randomly.fromList(columns)); + select.setFetchColumns(List.of(allColumns)); + select.setFromList(randomTables); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new CnosDBExpressionGenerator(state).setColumns(columns).generateOrderBy()); + } + select.setSelectType(SelectType.ALL); + select.setJoinClauses(joinStatements); + int firstCount = 0; + optimizedQueryString = CnosDBVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + CnosDBSelectQuery query = new CnosDBSelectQuery(optimizedQueryString, CnosDBExpectedError.expectedErrors()); + CnosDBResultSet rs; + try { + query.executeAndGet(state); + rs = query.getResultSet(); + while (rs.next()) { + firstCount++; + } + } catch (Exception e) { + if (query.getExpectedErrors().errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java new file mode 100644 index 000000000..6ff72558e --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java @@ -0,0 +1,176 @@ +package sqlancer.cnosdb.oracle.tlp; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBExpectedError; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBVisitor; +import sqlancer.cnosdb.ast.CnosDBAggregate; +import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; +import sqlancer.cnosdb.ast.CnosDBAlias; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBJoin; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.cnosdb.query.CnosDBSelectQuery; +import sqlancer.common.oracle.TestOracle; + +public class CnosDBTLPAggregateOracle extends CnosDBTLPBase implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public CnosDBTLPAggregateOracle(CnosDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() { + CnosDBAggregateFunction aggregateFunction = Randomly.fromOptions(CnosDBAggregateFunction.MAX, + CnosDBAggregateFunction.MIN, CnosDBAggregateFunction.SUM); + + CnosDBAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), + aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(fetchColumns); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + originalQuery = CnosDBVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + String queryFormatString = "-- %s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + String assertionMessage = String.format("%s: the results mismatch!\n%s\n%s", this.s.getDatabaseName(), + firstQueryString, secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + private String createMetamorphicUnionQuery(CnosDBSelect select, CnosDBAggregate aggregate, + List from) { + String metamorphicQuery; + CnosDBExpression whereClause = gen.generateExpression(CnosDBDataType.BOOLEAN); + CnosDBExpression negatedClause = new CnosDBPrefixOperation(whereClause, PrefixOperator.NOT); + CnosDBExpression notNullClause = new CnosDBPostfixOperation(whereClause, PostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + CnosDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); + CnosDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); + CnosDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += CnosDBVisitor.asString(leftSelect) + " UNION ALL " + CnosDBVisitor.asString(middleSelect) + + " UNION ALL " + CnosDBVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String resultString = null; + + CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, CnosDBExpectedError.expectedErrors()); + try { + q.executeAndGet(state); + CnosDBResultSet result = q.getResultSet(); + + if (result == null || !result.next()) { + throw new IgnoreMeException(); + } + + resultString = result.getString(1); + + } catch (Exception e) { + if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + } + + return resultString; + } + + private List mapped(CnosDBAggregate aggregate) { + switch (aggregate.getFunction()) { + case SUM: + case MAX: + case MIN: + return aliasArgs(List.of(aggregate)); + // now not support + // case COUNT: + // case AVG: + default: + throw new AssertionError(aggregate.getFunction()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (CnosDBExpression expr : originalAggregateArgs) { + args.add(new CnosDBAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(CnosDBAggregate aggregate) { + if (Objects.requireNonNull(aggregate.getFunction()) == CnosDBAggregateFunction.COUNT) { + return CnosDBAggregateFunction.SUM + "(agg0)"; + } + return aggregate.getFunction() + "(agg0)"; + } + + private CnosDBSelect getSelect(List aggregates, List from, + CnosDBExpression whereClause, List joinList) { + CnosDBSelect leftSelect = new CnosDBSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinClauses(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java new file mode 100644 index 000000000..a4dfd3061 --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java @@ -0,0 +1,112 @@ +package sqlancer.cnosdb.oracle.tlp; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBJoin; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; +import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; +import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; + +public class CnosDBTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + protected CnosDBSchema s; + protected CnosDBTables targetTables; + protected CnosDBExpressionGenerator gen; + protected CnosDBSelect select; + + public CnosDBTLPBase(CnosDBGlobalState state) { + super(state); + } + + public static CnosDBSubquery createSubquery(CnosDBGlobalState globalState, String name, CnosDBTables tables) { + List columns = new ArrayList<>(); + CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + CnosDBSelect select = new CnosDBSelect(); + select.setFromList(tables.getTables().stream().map(CnosDBFromTable::new).collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, CnosDBDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + return new CnosDBSubquery(select, name); + } + + @Override + public void check() throws Exception { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + List tables = targetTables.getTables(); + List joins = getJoinStatements(targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + protected List getJoinStatements(List columns, List tables) { + return CnosDBNoRECOracle.getJoinStatements(state, columns, tables); + } + + protected void generateSelectBase(List tables, List joins) { + List tableList = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); + gen = new CnosDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new CnosDBSelect(); + select.setFetchColumns(generateFetchColumns()); + select.setFromList(tableList); + select.setWhereClause(null); + select.setJoinClauses(joins); + } + + List generateFetchColumns() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return List.of(new CnosDBColumnValue(CnosDBColumn.createDummy("*"))); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = targetTables.getRandomColumnsWithOnlyOneField(); + + ArrayList columns = new ArrayList<>(); + targetColumns.forEach(column -> column.getTable().getColumns().stream() + .filter(field -> field instanceof CnosDBSchema.CnosDBFieldColumn).findFirst().ifPresent(columns::add)); + targetColumns.addAll(columns); + + targetColumns = targetColumns.stream().distinct().collect(Collectors.toList()); + + for (CnosDBColumn c : targetColumns) { + fetchColumns.add(new CnosDBColumnValue(c)); + } + return fetchColumns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java new file mode 100644 index 000000000..39b611a81 --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java @@ -0,0 +1,65 @@ +package sqlancer.cnosdb.oracle.tlp; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBComparatorHelper; +import sqlancer.cnosdb.CnosDBExpectedError; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBVisitor; +import sqlancer.cnosdb.ast.CnosDBExpression; + +public class CnosDBTLPHavingOracle extends CnosDBTLPBase { + + public CnosDBTLPHavingOracle(CnosDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + havingCheck(); + } + + protected void havingCheck() throws Exception { + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(CnosDBDataType.BOOLEAN)); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = CnosDBVisitor.asString(select); + List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, + CnosDBExpectedError.expectedErrors(), state); + + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + select.setHavingClause(predicate); + String firstQueryString = CnosDBVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = CnosDBVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = CnosDBVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, CnosDBExpectedError.expectedErrors()); + CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + protected CnosDBExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List generateFetchColumns() { + List expressions = gen.allowAggregates(true).generateExpressions(Randomly.smallNumber() + 1); + gen.allowAggregates(false); + return expressions; + } + +} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java new file mode 100644 index 000000000..3b3b6e4dd --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java @@ -0,0 +1,46 @@ +package sqlancer.cnosdb.oracle.tlp; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBComparatorHelper; +import sqlancer.cnosdb.CnosDBExpectedError; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBVisitor; + +public class CnosDBTLPWhereOracle extends CnosDBTLPBase { + + public CnosDBTLPWhereOracle(CnosDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + whereCheck(); + } + + protected void whereCheck() throws Exception { + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + String originalQueryString = CnosDBVisitor.asString(select); + List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, + CnosDBExpectedError.expectedErrors(), state); + + select.setOrderByExpressions(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = CnosDBVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = CnosDBVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = CnosDBVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, CnosDBExpectedError.expectedErrors()); + CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } +} diff --git a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java new file mode 100644 index 000000000..bd48c7b7e --- /dev/null +++ b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java @@ -0,0 +1,30 @@ +package sqlancer.cnosdb.query; + +import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.common.query.ExpectedErrors; + +public class CnosDBOtherQuery extends CnosDBQueryAdapter { + public CnosDBOtherQuery(String query, ExpectedErrors errors) { + super(query, errors); + } + + @Override + public boolean couldAffectSchema() { + return true; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + try { + globalState.getConnection().getClient().execute(query); + } catch (Exception e) { + if (this.errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + } + return true; + } +} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java new file mode 100644 index 000000000..2b2dda5e6 --- /dev/null +++ b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java @@ -0,0 +1,41 @@ +package sqlancer.cnosdb.query; + +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; + +public abstract class CnosDBQueryAdapter extends Query { + + String query; + ExpectedErrors errors; + + public CnosDBQueryAdapter(String query, ExpectedErrors errors) { + this.query = query; + this.errors = errors; + } + + @Override + public String getLogString() { + return query; + } + + @Override + public String getQueryString() { + return query; + } + + @Override + public String getUnterminatedQueryString() { + return null; + } + + @Override + public boolean couldAffectSchema() { + return false; + } + + @Override + public ExpectedErrors getExpectedErrors() { + return errors; + } +} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java b/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java new file mode 100644 index 000000000..dee38abf4 --- /dev/null +++ b/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java @@ -0,0 +1,6 @@ +package sqlancer.cnosdb.query; + +@FunctionalInterface +public interface CnosDBQueryProvider { + CnosDBOtherQuery getQuery(S globalState) throws Exception; +} diff --git a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java new file mode 100644 index 000000000..ca7f61c3a --- /dev/null +++ b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java @@ -0,0 +1,38 @@ +package sqlancer.cnosdb.query; + +import sqlancer.GlobalState; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; + +public class CnosDBSelectQuery extends CnosDBQueryAdapter { + CnosDBResultSet resultSet; + + public CnosDBSelectQuery(String query, ExpectedErrors errors) { + super(query, errors); + } + + @Override + public boolean couldAffectSchema() { + return false; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + globalState.getConnection().getClient().execute(query); + return false; + } + + @Override + public > SQLancerResultSet executeAndGet(G globalState, + String... fills) throws Exception { + resultSet = globalState.getConnection().getClient().executeQuery(query); + return null; + } + + public CnosDBResultSet getResultSet() { + return resultSet; + } +} From f2d3d823296ea8ee0cacf905cb2b8d965ed15ab6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 22:28:58 +0800 Subject: [PATCH 0735/1284] Add a FAQ section Address a question that frequently comes up (e.g., see https://github.com/sqlancer/sqlancer/issues/704 or https://github.com/sqlancer/sqlancer/issues/684). Also mention the locale issue (see https://github.com/sqlancer/sqlancer/issues/533). --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 283efcd29..780b11025 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,13 @@ We would appreciate it if you mention SQLancer when you report bugs found by it. We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. SQLancer's official Twitter handle is [@sqlancer_dbms](https://twitter.com/sqlancer_dbms). +# FAQ + +## I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors? + +In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) directory to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). + +Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html). # Additional Documentation From 44ead10dcfde8111a6d4ea86653e4d9362d77daa Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 22:32:55 +0800 Subject: [PATCH 0736/1284] Fix minor doc issues --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 780b11025..1fc7cdf06 100644 --- a/README.md +++ b/README.md @@ -93,9 +93,9 @@ We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_inv ## I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors? -In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) directory to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). +In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) file to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). -Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html). +Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html)). # Additional Documentation From 378f82ad76a1121a57695e6f84de66e23cc89fc7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 22:37:30 +0800 Subject: [PATCH 0737/1284] No longer mention that we will soon release the reducer --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1fc7cdf06..96e1985c6 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ SQLancer stores logs in the `target/logs` subdirectory. By default, the option ` ## Reducing a Bug -After finding a bug, it is useful to produce a minimal test case before reporting the bug, to save the DBMS developers' time and effort. For many test cases, [C-Reduce](https://embed.cs.utah.edu/creduce/) does a great job. In addition, we have been working on a SQL-specific reducer, which we plan to release soon. +After finding a bug, it is useful to produce a minimal test case before reporting the bug, to save the DBMS developers' time and effort. For many test cases, [C-Reduce](https://embed.cs.utah.edu/creduce/) does a great job. ## Found Bugs From 403ab6f6d4ff9e926e2e675d0e0f726739f550e2 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 22:46:01 +0800 Subject: [PATCH 0738/1284] Describe that a "test" database might be required For example, see https://github.com/sqlancer/sqlancer/issues/677 for this question. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 96e1985c6..047a30012 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,10 @@ In many cases, SQLancer does not support the latest version of a DBMS. You can c Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html)). +## When starting SQLancer, I get an error such as "database 'test' does not exist". How can I run SQLancer without this error? + +For some DBMSs, SQLancer expects that a database "test" exists, which it then uses as an initial database to connect to. If you have not yet created such a database, you can use a command such as `CREATE DATABASE test` to create this database (e.g., see the [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-createdatabase.html)). + # Additional Documentation * [Contributing to SQLancer](CONTRIBUTING.md) From a4a21c00129a276b7978599d574bdda9f5bb02f9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 23:05:27 +0800 Subject: [PATCH 0739/1284] Add additional suggestions for contributors --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b28a1be29..08749f141 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,6 +14,13 @@ If you do not find an option to import Maven projects, you might need to install The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see `DuckDBNoRECOracle` or `DuckDBQueryPartitioningWhereTester` for TLP). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). +Please consider the following suggestions when creating a PR to contribute a new DBMS: +* Ensure that `mvn verify -DskipTests=true` does not result in style violations. +* Add a [CI test](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) to ensure that future changes to SQLancer are unlikely to break the newly-supported DBMS. It is reasonable to do this in a follow-up PR—please indicate whether you plan to do so in the PR description. +* Add the DBMS' name to the [check_names.py](https://github.com/sqlancer/sqlancer/blob/master/src/check_names.py) script, which ensures adherence to a common prefix in the Java classes. +* Add the DBMS' name to the [README.md](https://github.com/sqlancer/sqlancer/blob/master/README.md#supported-dbms) file. +* It would be easier to review multiple smaller PRs, than one PR that contains the complete implementation. Consider contributing parts of your implementation as you work on their implementation. + ### Typed vs. Untyped Expression Generation Each DBMS implementation provides an expression generator used, for example, to generate expressions used in `WHERE` clauses. We found that DBMS can be roughly classified into "permissive" ones, which apply implicit type conversions when needed and "strict" ones, which provide only few implicit conversions and output an error when the type is unexpected. For example, consider the following test case: From 5f5ffb5c2717caa2f374cccc11102b9e2da9aa8e Mon Sep 17 00:00:00 2001 From: ZuoTiJia <49109132+ZuoTiJia@users.noreply.github.com> Date: Sun, 29 Jan 2023 11:06:52 +0800 Subject: [PATCH 0740/1284] Add CnosDB Info (#705) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 047a30012..346209908 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | | Databend | Working | Typed | | | QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | +| CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | # Using SQLancer From 5ec6334ab0622c37a3c43cc28187eb545eda3fe7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 29 Jan 2023 13:36:37 +0800 Subject: [PATCH 0741/1284] Ignore a known Citus bug (#706) See https://github.com/citusdata/citus/issues/6298. Partly addresses https://github.com/sqlancer/sqlancer/issues/698. --- src/sqlancer/citus/CitusBugs.java | 3 +++ src/sqlancer/citus/gen/CitusInsertGenerator.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/sqlancer/citus/CitusBugs.java b/src/sqlancer/citus/CitusBugs.java index bc81b9db6..a6f4910e6 100644 --- a/src/sqlancer/citus/CitusBugs.java +++ b/src/sqlancer/citus/CitusBugs.java @@ -30,6 +30,9 @@ public final class CitusBugs { // https://github.com/citusdata/citus/issues/4079 public static boolean bug4079 = true; + // https://github.com/citusdata/citus/issues/6298 + public static boolean bug6298 = true; + private CitusBugs() { } diff --git a/src/sqlancer/citus/gen/CitusInsertGenerator.java b/src/sqlancer/citus/gen/CitusInsertGenerator.java index 269d94bbe..78b5794e7 100644 --- a/src/sqlancer/citus/gen/CitusInsertGenerator.java +++ b/src/sqlancer/citus/gen/CitusInsertGenerator.java @@ -1,5 +1,6 @@ package sqlancer.citus.gen; +import sqlancer.citus.CitusBugs; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -14,6 +15,9 @@ public static SQLQueryAdapter insert(PostgresGlobalState globalState) { SQLQueryAdapter insertQuery = PostgresInsertGenerator.insert(globalState); ExpectedErrors errors = insertQuery.getExpectedErrors(); CitusCommon.addCitusErrors(errors); + if (CitusBugs.bug6298) { + errors.add("columnar_tuple_insert_speculative not implemented"); + } return insertQuery; } From 080eb773924440a8c4ba72167d50d0678b87fd0c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 29 Jan 2023 13:43:05 +0800 Subject: [PATCH 0742/1284] Ignore invalid datetime casts in TiDB (#708) Partly addresses https://github.com/sqlancer/sqlancer/issues/669. --- src/sqlancer/tidb/TiDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 0022908a9..c2db11757 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -13,6 +13,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("BIGINT UNSIGNED value is out of range"); errors.add("Data truncation: Truncated incorrect time value"); errors.add("Data truncation: Incorrect time value"); + errors.add("Data truncation: Incorrect datetime value"); errors.add("overflows double"); errors.add("overflows bigint"); errors.add("strconv.ParseFloat: parsing"); From 6d90c985950f4afbf12728d1f2cf99c00dc2a86d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 29 Jan 2023 14:14:04 +0800 Subject: [PATCH 0743/1284] Check the naming conventions of all current database systems (#709) --- src/check_names.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/check_names.py b/src/check_names.py index 72191ac7d..31d6853a4 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -18,12 +18,21 @@ def verify_prefix(prefix, files): print('The class name of ' + f + ' does not start with ' + prefix) exit(-1) +verify_prefix('ArangoDB', get_java_files("sqlancer/arangodb/")) +verify_prefix('Citus', get_java_files("sqlancer/citus/")) verify_prefix('ClickHouse', get_java_files("sqlancer/clickhouse/")) +verify_prefix('CnosDB', get_java_files("sqlancer/cnosdb/")) verify_prefix('CockroachDB', get_java_files("sqlancer/cockroachdb/")) +verify_prefix('Cosmos', get_java_files("sqlancer/cosmos/")) +verify_prefix('Databend', get_java_files("sqlancer/databend/")) verify_prefix('DuckDB', get_java_files("sqlancer/duckdb")) +verify_prefix('H2', get_java_files("sqlancer/h2")) +verify_prefix('HSQLDB', get_java_files("sqlancer/hsqldb")) verify_prefix('MariaDB', get_java_files("sqlancer/mariadb/")) verify_prefix('MySQL', get_java_files("sqlancer/mysql/")) +verify_prefix('OceanBase', get_java_files("sqlancer/oceanbase/")) verify_prefix('Postgres', get_java_files("sqlancer/postgres/")) +verify_prefix('QuestDB', get_java_files("sqlancer/questdb/")) verify_prefix('SQLite3', get_java_files("sqlancer/sqlite3/")) verify_prefix('TiDB', get_java_files("sqlancer/tidb/")) -verify_prefix('CnosDB', get_java_files("sqlancer/cnosdb/")) +verify_prefix('Y', get_java_files("sqlancer/yugabyte/")) # has both YCQL and YSQL prefixes From 222ae47a0e897968b403de98a516d02ed5bafae8 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Tue, 31 Jan 2023 12:19:46 +0800 Subject: [PATCH 0744/1284] fix: credential env for databend docker image (#713) --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1587aee78..329f108da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -146,7 +146,7 @@ jobs: run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test - + databend: name: DBMS Tests (Databend) runs-on: ubuntu-latest @@ -154,8 +154,8 @@ jobs: databend: image: datafuselabs/databend env: - DATABEND_DEFAULT_USER: sqlancer - DATABEND_DEFAULT_PASSWORD: sqlancer + QUERY_DEFAULT_USER: sqlancer + QUERY_DEFAULT_PASSWORD: sqlancer ports: - 8000:8000 - 3307:3307 From e148652d1533b8f122c29c3d4bbb41c77b1747c9 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Wed, 1 Feb 2023 15:00:58 +0800 Subject: [PATCH 0745/1284] fix(databend):get data type error (#715) * add(databend): databend bugs Signed-off-by: Yisong Han * fix(databend):get data type error Signed-off-by: Yisong Han * fix: credential env for databend docker image (#713) * fix(databend):reduce unimportant error messages Signed-off-by: Yisong Han --------- Signed-off-by: Yisong Han Co-authored-by: everpcpc Co-authored-by: Manuel Rigger --- src/sqlancer/databend/DatabendBugs.java | 1 + src/sqlancer/databend/DatabendErrors.java | 9 ++++++++- src/sqlancer/databend/DatabendSchema.java | 3 +++ .../databend/ast/DatabendBinaryArithmeticOperation.java | 2 +- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index 336443511..eec2c39d0 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -14,6 +14,7 @@ public final class DatabendBugs { public static boolean bug9235 = true; // https://github.com/datafuselabs/databend/issues/9235 public static boolean bug9236 = true; // https://github.com/datafuselabs/databend/issues/9236 public static boolean bug9264 = true; // https://github.com/datafuselabs/databend/issues/9264 + public static boolean bug9806 = true; // https://github.com/datafuselabs/databend/issues/9806 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 8f9d2b942..5e813dcbd 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -9,9 +9,13 @@ private DatabendErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("divided by zero"); errors.add("/ by zero"); errors.add("ORDER BY position"); errors.add("GROUP BY position"); + errors.add("no overload satisfies `not(Float64 NULL)`"); // TODO databend不允许出现not(float)，而a/b为float + errors.add("no overload satisfies `not(Float64)`"); + errors.add("number overflowed while evaluating function"); // 表达式数值溢出 if (DatabendBugs.bug9162) { errors.add("downcast column error"); } @@ -30,7 +34,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (DatabendBugs.bug9264) { errors.add("assertion failed: offset + length <= self.length"); } - // errors.add("validity's length must be equal"); // bug + if (DatabendBugs.bug9806) { + errors.add("segment pruning failure"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE @@ -45,6 +51,7 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Division by zero"); errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); + errors.add("number overflowed while evaluating function `"); // 不能在int16类型column上插入int64的数据 } public static void addGroupByErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 24d47ec46..2e8e3c2bf 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -339,6 +339,9 @@ private static List getTableColumns(SQLConnection con, String ta while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); + if (dataType.contains("NULL")) { + dataType = dataType.substring(0, dataType.indexOf(' ')); + } boolean isNullable = rs.getBoolean("is_nullable"); // boolean isPrimaryKey = rs.getString("pk").contains("true"); boolean isPrimaryKey = false; // 没找到主键元数据 diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java index f052b3ccb..391597941 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -34,7 +34,7 @@ public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { return applyOperation(left, right, (l, r) -> l * r); } }, - DIVISION("/") { + DIVISION("/") { // TODO databend不允许出现not(float)，而a/b为float @Override public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); From 34424b79d532ddf972413f828cc30f6b23c1803f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Feb 2023 14:47:58 +0800 Subject: [PATCH 0746/1284] Add a fix to address the recent TiDB data type issues (#720) Addresses https://github.com/sqlancer/sqlancer/issues/669. --- src/sqlancer/tidb/TiDBSchema.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 04bde5e33..609d1dbe7 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -238,6 +238,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { primitiveType = TiDBDataType.FLOATING; break; case "double": + case "double(8,6)": // workaround to address https://github.com/sqlancer/sqlancer/issues/669 size = 8; primitiveType = TiDBDataType.FLOATING; break; @@ -271,6 +272,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { break; case "date": case "datetime": + case "datetime(6)": // workaround to address https://github.com/sqlancer/sqlancer/issues/669 case "timestamp": case "time": case "year": From 0ba08b51de2ef94b698f5abce573269ad5719a50 Mon Sep 17 00:00:00 2001 From: Ming Cheng Date: Sun, 5 Feb 2023 21:09:13 -0500 Subject: [PATCH 0747/1284] Fix QuestDB execution bug (#721) --- src/sqlancer/questdb/QuestDBProvider.java | 2 +- src/sqlancer/questdb/QuestDBSchema.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/questdb/QuestDBProvider.java b/src/sqlancer/questdb/QuestDBProvider.java index d9932c8a3..79c1554be 100644 --- a/src/sqlancer/questdb/QuestDBProvider.java +++ b/src/sqlancer/questdb/QuestDBProvider.java @@ -106,7 +106,7 @@ public SQLConnection createDatabase(QuestDBGlobalState globalState) throws Excep } // TODO(anxing): maybe not hardcode here... String databaseName = "qdb"; - String tableName = "test"; + String tableName = "sqlancer_test"; String url = String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName); // use QuestDB default username & password for Postgres JDBC Properties properties = new Properties(); diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index 0b5563a31..61d27161a 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -177,7 +177,7 @@ public boolean isNullable() { public static class QuestDBTables extends AbstractTables { public static final Set RESERVED_TABLES = new HashSet<>( - Arrays.asList("sys.column_versions_purge_log", "telemetry_config", "telemetry")); + Arrays.asList("sys.column_versions_purge_log", "telemetry_config", "telemetry", "sys.telemetry_wal")); public QuestDBTables(List tables) { super(tables); From 504ab26125b7a1d67ed817ff65ffce536eac46ac Mon Sep 17 00:00:00 2001 From: Ming Cheng Date: Mon, 6 Feb 2023 00:50:33 -0500 Subject: [PATCH 0748/1284] Forbid multiple table selection from QuestDB (#722) * Forbid multiple table selection from QuestDB * Forbid multiple table selection from QuestDB * Format code * Remove unused field --- .../questdb/test/QuestDBQueryPartitioningBase.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index 45e3db9c4..5db302dbd 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -18,7 +18,6 @@ import sqlancer.questdb.QuestDBSchema; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBTable; -import sqlancer.questdb.QuestDBSchema.QuestDBTables; import sqlancer.questdb.ast.QuestDBExpression; import sqlancer.questdb.ast.QuestDBSelect; import sqlancer.questdb.gen.QuestDBExpressionGenerator; @@ -28,7 +27,7 @@ public class QuestDBQueryPartitioningBase implements TestOracle { QuestDBSchema s; - QuestDBTables targetTables; + QuestDBTable targetTable; // single table QuestDBExpressionGenerator gen; QuestDBSelect select; @@ -42,7 +41,7 @@ List> generateFetchColumns() { if (Randomly.getBoolean()) { columns.add(new ColumnReferenceNode<>(new QuestDBColumn("*", null, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + columns = Randomly.nonEmptySubset(targetTable.getColumns()).stream() .map(c -> new ColumnReferenceNode(c)) .collect(Collectors.toList()); } @@ -66,12 +65,15 @@ protected ExpressionGenerator> getGen() { @Override public void check() throws SQLException { s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new QuestDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + // Only return one table instead of multiple tables, which is regarded as illegal by QuestDB + // e.g. "SELECT * FROM t0, t1;" + targetTable = s.getRandomTable(); + gen = new QuestDBExpressionGenerator(state).setColumns(targetTable.getColumns()); initializeTernaryPredicateVariants(); select = new QuestDBSelect(); select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); + List tables = new ArrayList<>(); + tables.add(targetTable); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); // Ignore JOINs for now From 40a1d26adaa0cc76050aa9380d271565862e4f95 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Wed, 8 Feb 2023 12:36:51 +0800 Subject: [PATCH 0749/1284] Update QGP preprint link (#724) --- docs/PAPERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/PAPERS.md b/docs/PAPERS.md index 45f57e887..6510930e1 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -53,7 +53,7 @@ This paper describes TLP, a metamorphic testing approach that can detect various ## Testing Database Engines via Query Plan Guidance -This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. +This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. A preprint is available [here](http://jinshengba.me/assets/pdf/qpg_icse23.pdf). ``` @inproceedings{Ba2023QPG, From aa0c0eccba4eefa75bfd518f608c9222c692c11d Mon Sep 17 00:00:00 2001 From: Ming Cheng Date: Sat, 11 Feb 2023 21:44:40 -0500 Subject: [PATCH 0750/1284] Add SYMBOL type support for QuestDB (#727) * add symbol support for QuestDB * add symbol support * fix: symbol type is null --- src/sqlancer/questdb/QuestDBSchema.java | 14 +++++++++-- src/sqlancer/questdb/ast/QuestDBConstant.java | 24 +++++++++++++++++++ .../gen/QuestDBExpressionGenerator.java | 4 ++++ .../test/QuestDBQueryPartitioningBase.java | 2 +- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index 61d27161a..e490d0cda 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -27,8 +27,7 @@ public enum QuestDBDataType { BOOLEAN, // CHAR, /* STRING, */ - INT, FLOAT, - /* SYMBOL, */ + INT, FLOAT, SYMBOL, // DATE, TIMESTAMP, /* GEOHASH, */ NULL; @@ -70,6 +69,9 @@ public QuestDBCompositeDataType(QuestDBDataType dataType, int size) { case BOOLEAN: isNullable = false; break; + case SYMBOL: + isNullable = true; + break; default: isNullable = true; } @@ -106,6 +108,9 @@ public static QuestDBCompositeDataType getRandomWithoutNull() { // case TIMESTAMP: size = 0; break; + case SYMBOL: + size = 0; + break; default: throw new AssertionError(type); } @@ -142,6 +147,8 @@ public String toString() { } case BOOLEAN: return Randomly.fromOptions("BOOLEAN"); + case SYMBOL: + return Randomly.fromOptions("SYMBOL"); // case TIMESTAMP: // return Randomly.fromOptions("TIMESTAMP"); // case DATE: @@ -233,6 +240,9 @@ private static QuestDBCompositeDataType getColumnType(String typeString) { primitiveType = QuestDBDataType.INT; size = 2; break; + case "SYMBOL": + primitiveType = QuestDBDataType.SYMBOL; + break; case "NULL": primitiveType = QuestDBDataType.NULL; break; diff --git a/src/sqlancer/questdb/ast/QuestDBConstant.java b/src/sqlancer/questdb/ast/QuestDBConstant.java index a59ce1053..f7e5ab0f8 100644 --- a/src/sqlancer/questdb/ast/QuestDBConstant.java +++ b/src/sqlancer/questdb/ast/QuestDBConstant.java @@ -47,6 +47,26 @@ public boolean getValue() { } } + public static class QuestDBSymbolConstant extends QuestDBConstant { + private final String value; + + public QuestDBSymbolConstant(String value) { + this.value = value; + } + + @Override + public String toString() { + if (value.equals("")) { + return "NULL"; + } + return "'" + value + "'"; + } + + public String getValue() { + return value; + } + } + public static Node createIntConstant(long val) { return new QuestDBIntConstant(val); } @@ -86,4 +106,8 @@ public static Node createNullConstant() { public static Node createFloatConstant(double val) { return new QuestDBDoubleConstant(val); } + + public static Node createSymbolConstant(String val) { + return new QuestDBSymbolConstant(val); + } } diff --git a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java index 8f9ea443a..73f57c055 100644 --- a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java @@ -5,6 +5,7 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.Randomly.StringGenerationStrategy; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; @@ -54,6 +55,9 @@ public Node generateConstant() { return QuestDBConstant.createBooleanConstant(Randomly.getBoolean()); case FLOAT: return QuestDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); + case SYMBOL: + StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; + return QuestDBConstant.createSymbolConstant(strategy.getString(globalState.getRandomly())); // case CHAR: // case DATE: // case TIMESTAMP: diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index 5db302dbd..fa5283507 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -27,7 +27,7 @@ public class QuestDBQueryPartitioningBase implements TestOracle { QuestDBSchema s; - QuestDBTable targetTable; // single table + QuestDBTable targetTable; QuestDBExpressionGenerator gen; QuestDBSelect select; From 30948f34acc2354d6be18a70bdeeebff1e73fa48 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Feb 2023 18:18:54 +0800 Subject: [PATCH 0751/1284] Describe the expected errors concept --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 08749f141..58b6af199 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,10 @@ Please consider the following suggestions when creating a PR to contribute a ne * Add the DBMS' name to the [README.md](https://github.com/sqlancer/sqlancer/blob/master/README.md#supported-dbms) file. * It would be easier to review multiple smaller PRs, than one PR that contains the complete implementation. Consider contributing parts of your implementation as you work on their implementation. +## Expected Errors + +Most statements have an [ExpectedError](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/common/query/ExpectedErrors.java) object associated with them. This object essentially contains a list of errors, one of which the database system might return if it cannot successfully execute the statement. These errors are typically added through a trial-and-error process while considering various tradeoffs. For example, consider the [DuckDBInsertGenerator](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java#L38) class, whose expected errors are specified in [DuckDBErrors](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/DuckDBErrors.java#L90). When implementing such a generator, the list of expected errors might first be empty. When running the generator for the first time, you might receive an error such as "create unique index, table contains duplicate data", indicating that creating the index failed due to duplicate data. In principle, this error could be avoided by first checking whether the column contains any duplicate values. However, checking this would be expensive and error-prone (e.g., consider string similarity, which might depend on collations); thus, the obvious choice would be to add this string to the list of expected errors, and run the generator again to check for any other expected errors. In other cases, errors might be best addressed through improvements in the generators. For example, it is typically straightforward to generate syntactically-valid statements, which is why syntax errors should not be ignored. This approach is effective in uncovering internal errors; rather than ignoring them as an expected error, report them, and see [Unfixed Bugs](###unfixed-bugs) below. + ### Typed vs. Untyped Expression Generation Each DBMS implementation provides an expression generator used, for example, to generate expressions used in `WHERE` clauses. We found that DBMS can be roughly classified into "permissive" ones, which apply implicit type conversions when needed and "strict" ones, which provide only few implicit conversions and output an error when the type is unexpected. For example, consider the following test case: From 29c44513a2106f72b3610058368d3377135f8a8c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Feb 2023 18:30:08 +0800 Subject: [PATCH 0752/1284] Explain how to bail out while generating statements --- CONTRIBUTING.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58b6af199..f2217b1f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,9 +21,14 @@ Please consider the following suggestions when creating a PR to contribute a ne * Add the DBMS' name to the [README.md](https://github.com/sqlancer/sqlancer/blob/master/README.md#supported-dbms) file. * It would be easier to review multiple smaller PRs, than one PR that contains the complete implementation. Consider contributing parts of your implementation as you work on their implementation. -## Expected Errors +### Expected Errors + +Most statements have an [ExpectedError](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/common/query/ExpectedErrors.java) object associated with them. This object essentially contains a list of errors, one of which the database system might return if it cannot successfully execute the statement. These errors are typically added through a trial-and-error process while considering various tradeoffs. For example, consider the [DuckDBInsertGenerator](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java#L38) class, whose expected errors are specified in [DuckDBErrors](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/DuckDBErrors.java#L90). When implementing such a generator, the list of expected errors might first be empty. When running the generator for the first time, you might receive an error such as "create unique index, table contains duplicate data", indicating that creating the index failed due to duplicate data. In principle, this error could be avoided by first checking whether the column contains any duplicate values. However, checking this would be expensive and error-prone (e.g., consider string similarity, which might depend on collations); thus, the obvious choice would be to add this string to the list of expected errors, and run the generator again to check for any other expected errors. In other cases, errors might be best addressed through improvements in the generators. For example, it is typically straightforward to generate syntactically-valid statements, which is why syntax errors should not be ignored. This approach is effective in uncovering internal errors; rather than ignoring them as an expected error, report them, and see [Unfixed Bugs](#unfixed-bugs) below. + +### Bailing Out While Generating a Statement + +In some cases, it might be undesirable or even impossible to generate a specific statement type. For example, consider that SQLancer tries to execute a `DROP TABLE` statement (e.g., see [TiDBDropTableGenerator](https://github.com/sqlancer/sqlancer/blob/30948f34acc2354d6be18a70bdeeebff1e73fa48/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java)), but the database contains only a single table. Dropping the table would result in all subsequent attempts to insert data or query it to fail. Thus, in such a case, it might be more efficient to "bail out" by abandoning the current attempt to generate the statement. This can be achieved by throwing a `IgnoreMeException`. Unlike for other exceptions, SQLancer silently continues execution rather than reporting this exception to the user. -Most statements have an [ExpectedError](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/common/query/ExpectedErrors.java) object associated with them. This object essentially contains a list of errors, one of which the database system might return if it cannot successfully execute the statement. These errors are typically added through a trial-and-error process while considering various tradeoffs. For example, consider the [DuckDBInsertGenerator](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java#L38) class, whose expected errors are specified in [DuckDBErrors](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/DuckDBErrors.java#L90). When implementing such a generator, the list of expected errors might first be empty. When running the generator for the first time, you might receive an error such as "create unique index, table contains duplicate data", indicating that creating the index failed due to duplicate data. In principle, this error could be avoided by first checking whether the column contains any duplicate values. However, checking this would be expensive and error-prone (e.g., consider string similarity, which might depend on collations); thus, the obvious choice would be to add this string to the list of expected errors, and run the generator again to check for any other expected errors. In other cases, errors might be best addressed through improvements in the generators. For example, it is typically straightforward to generate syntactically-valid statements, which is why syntax errors should not be ignored. This approach is effective in uncovering internal errors; rather than ignoring them as an expected error, report them, and see [Unfixed Bugs](###unfixed-bugs) below. ### Typed vs. Untyped Expression Generation From 0ec43d5974ebabb2faba86b28c17ed48514e610e Mon Sep 17 00:00:00 2001 From: Ming Cheng Date: Thu, 16 Feb 2023 22:18:08 -0500 Subject: [PATCH 0753/1284] Fix add index bug for QuestDB (#729) * add symbol support for QuestDB * add symbol support * fix: symbol type is null * Fix add index bug for QuestDB * Fix add index bug for QuestDB * Apply code formatter * Apply mvn verify for code * Compare data type directly instead of converting to string * Use equality operator --- src/sqlancer/common/schema/AbstractTable.java | 11 +++++++++++ src/sqlancer/questdb/QuestDBSchema.java | 2 +- .../questdb/gen/QuestDBAlterIndexGenerator.java | 10 +++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index c40f1a197..bf14abc1a 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -2,9 +2,11 @@ import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.stream.Collectors; import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; public abstract class AbstractTable, I extends TableIndex, G extends GlobalState> @@ -60,6 +62,15 @@ public C getRandomColumn() { return Randomly.fromList(columns); } + public C getRandomColumnOrBailout(Predicate predicate) { + List relevantColumns = columns.stream().filter(predicate).collect(Collectors.toList()); + if (relevantColumns.isEmpty()) { + throw new IgnoreMeException(); + } + + return Randomly.fromList(relevantColumns); + } + public boolean hasIndexes() { return !indexes.isEmpty(); } diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index e490d0cda..8253ec82c 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -148,7 +148,7 @@ public String toString() { case BOOLEAN: return Randomly.fromOptions("BOOLEAN"); case SYMBOL: - return Randomly.fromOptions("SYMBOL"); + return "SYMBOL"; // case TIMESTAMP: // return Randomly.fromOptions("TIMESTAMP"); // case DATE: diff --git a/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java index 1e8242148..85103738f 100644 --- a/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java @@ -4,6 +4,8 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBDataType; import sqlancer.questdb.QuestDBSchema.QuestDBTable; public final class QuestDBAlterIndexGenerator { @@ -29,7 +31,13 @@ public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { sb.append(" "); sb.append("ALTER COLUMN "); - String columnName = table.getFreeColumnName(); + + // We should always choose column with SYMBOL type + QuestDBColumn columnWithSymbolType = table + .getRandomColumnOrBailout(c -> c.getType().getPrimitiveDataType() == QuestDBDataType.SYMBOL); + + String columnName = columnWithSymbolType.getName(); + sb.append(columnName); sb.append(" "); From b19a41e0dcf7cbd5fd84f66694743dc12323e2ef Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 2 Mar 2023 04:11:45 +0100 Subject: [PATCH 0754/1284] Add analyser flag (#738) --- src/sqlancer/clickhouse/ClickHouseOptions.java | 3 +++ src/sqlancer/clickhouse/ClickHouseProvider.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 0aafab84d..94297ad20 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -31,6 +31,9 @@ public class ClickHouseOptions implements DBMSSpecificOptions { TLPWhere { @Override diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 5c0c371c6..86f6fd392 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -143,7 +143,8 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ } con.close(); con = DriverManager.getConnection( - String.format("jdbc:clickhouse://%s:%d/%s?socket_timeout=300000", host, port, databaseName), + String.format("jdbc:clickhouse://%s:%d/%s?socket_timeout=300000%s", host, port, databaseName, + clickHouseOptions.enableAnalyzer ? "&allow_experimental_analyzer=1" : ""), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); return new SQLConnection(con); } From 5c5440f5aa4aad1be0c6a8c31dad19e4ce8273c8 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Sat, 4 Mar 2023 02:16:55 +0100 Subject: [PATCH 0755/1284] Add another error message for ambiguous column in ClickHouse (#741) * one more to skip * one more to skip --- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 97a5e300c..e824faef9 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -78,6 +78,8 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { // We may not count it as an issue, but it makes no sense to add more complex AST generation logic errors.add("MULTIPLE_EXPRESSIONS_FOR_ALIAS"); errors.add("AMBIGUOUS_IDENTIFIER"); // https://github.com/ClickHouse/ClickHouse/issues/45389 + errors.add("AMBIGUOUS_COLUMN_NAME"); // same https://github.com/ClickHouse/ClickHouse/issues/45389 + errors.add("Cannot parse number with multiple sign"); } } From 85bea1991954e0a39c0afbd485f8879a7b0a409a Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 6 Mar 2023 14:42:11 +0800 Subject: [PATCH 0756/1284] refactor: better code logic, try-with-resources with multiple resources (#750) * refactor: better code logic, try-with-resources with multiple resources * mvn formatter:format --- src/sqlancer/citus/CitusSchema.java | 56 +++++++++++++---------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index defda8804..560191c05 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -58,41 +58,35 @@ public Integer getColocationId() { public static CitusSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); - try { - List databaseTables = new ArrayList<>(); - try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery( - "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { - while (rs.next()) { - String tableName = rs.getString("table_name"); - /* citus_tables is a helper view, we don't need to test with it so we let's ignore it */ - if (tableName.equals("citus_tables")) { - continue; - } - String distributionColumnName = rs.getString("dist_col_name"); - Integer colocationId = rs.getInt("colocationid"); - if (rs.wasNull()) { - colocationId = null; - } - PostgresTable t = schema.getDatabaseTable(tableName); - PostgresColumn distributionColumn = null; - if (t == null) { - continue; - } - if (distributionColumnName != null && !distributionColumnName.equals("")) { - distributionColumn = t.getColumns().stream() - .filter(c -> c.getName().equals(distributionColumnName)) - .collect(Collectors.toList()).get(0); - } - CitusTable tCitus = new CitusTable(t, distributionColumn, colocationId); - databaseTables.add(tCitus); - } + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement(); ResultSet rs = s.executeQuery( + "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + /* citus_tables is a helper view, we don't need to test with it so we let's ignore it */ + if (tableName.equals("citus_tables")) { + continue; } + String distributionColumnName = rs.getString("dist_col_name"); + Integer colocationId = rs.getInt("colocationid"); + if (rs.wasNull()) { + colocationId = null; + } + PostgresTable t = schema.getDatabaseTable(tableName); + PostgresColumn distributionColumn = null; + if (t == null) { + continue; + } + if (distributionColumnName != null && !distributionColumnName.equals("")) { + distributionColumn = t.getColumns().stream().filter(c -> c.getName().equals(distributionColumnName)) + .collect(Collectors.toList()).get(0); + } + CitusTable tCitus = new CitusTable(t, distributionColumn, colocationId); + databaseTables.add(tCitus); } - return new CitusSchema(databaseTables, databaseName); } catch (SQLIntegrityConstraintViolationException e) { throw new AssertionError(e); } + return new CitusSchema(databaseTables, databaseName); } - } From 466400139b4124be28093041fa4b7d4dd8c99885 Mon Sep 17 00:00:00 2001 From: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> Date: Tue, 7 Mar 2023 11:20:59 +0530 Subject: [PATCH 0757/1284] Refactor several classes (#752) * Refactor Several classes Signed-off-by: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> * mvn formatter:format Signed-off-by: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> * Checkstyle Error fix Signed-off-by: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> --------- Signed-off-by: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> --- src/sqlancer/ProviderAdapter.java | 8 ++++---- src/sqlancer/Randomly.java | 15 +++++---------- src/sqlancer/SQLProviderAdapter.java | 2 +- src/sqlancer/StateToReproduce.java | 5 ++++- src/sqlancer/StatementExecutor.java | 2 +- src/sqlancer/common/DBMSCommon.java | 2 +- src/sqlancer/common/ast/BinaryNode.java | 2 +- src/sqlancer/common/ast/BinaryOperatorNode.java | 2 +- src/sqlancer/common/ast/FunctionNode.java | 2 +- src/sqlancer/common/ast/TernaryNode.java | 2 +- src/sqlancer/common/ast/UnaryNode.java | 2 +- src/sqlancer/common/ast/UnaryOperatorNode.java | 2 +- src/sqlancer/common/log/SQLLoggableFactory.java | 8 ++++---- src/sqlancer/common/oracle/NoRECBase.java | 2 +- .../common/oracle/PivotedQuerySynthesisBase.java | 2 +- src/sqlancer/common/schema/AbstractRowValue.java | 2 +- src/sqlancer/common/schema/AbstractSchema.java | 2 +- src/sqlancer/common/schema/AbstractTable.java | 4 ++-- 18 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 47b15ee39..ab5ca8e9a 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -26,7 +26,7 @@ public abstract class ProviderAdapter globalClass, Class optionClass) { + protected ProviderAdapter(Class globalClass, Class optionClass) { this.globalClass = globalClass; this.optionClass = optionClass; } @@ -85,7 +85,7 @@ protected TestOracle getTestOracle(G globalState) throws Exception { List> testOracleFactory = globalState.getDbmsSpecificOptions() .getTestOracleFactory(); boolean testOracleRequiresMoreThanZeroRows = testOracleFactory.stream() - .anyMatch(p -> p.requiresAllTablesToContainRows()); + .anyMatch(OracleFactory::requiresAllTablesToContainRows); boolean userRequiresMoreThanZeroRows = globalState.getOptions().testOnlyWithMoreThanZeroRows(); boolean checkZeroRows = testOracleRequiresMoreThanZeroRows || userRequiresMoreThanZeroRows; if (checkZeroRows && globalState.getSchema().containsTableWithZeroRows(globalState)) { @@ -98,7 +98,7 @@ protected TestOracle getTestOracle(G globalState) throws Exception { if (testOracleFactory.size() == 1) { return testOracleFactory.get(0).create(globalState); } else { - return new CompositeTestOracle(testOracleFactory.stream().map(o -> { + return new CompositeTestOracle<>(testOracleFactory.stream().map(o -> { try { return o.create(globalState); } catch (Exception e1) { @@ -170,7 +170,7 @@ private synchronized boolean mutateTables(G globalState) throws Exception { if (Randomly.getPercentage() < globalState.getOptions().getQPGProbability()) { selectedActionIndex = globalState.getRandomly().getInteger(0, weightedAverageReward.length); } else { - selectedActionIndex = DBMSCommon.getMaxIndexInDoubleArrary(weightedAverageReward); + selectedActionIndex = DBMSCommon.getMaxIndexInDoubleArray(weightedAverageReward); } int reward = 0; diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index fa317f1f0..ae34ba1b1 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -3,6 +3,7 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Random; import java.util.function.Supplier; @@ -133,17 +134,12 @@ public static List subset(List columns) { public static List subset(int nr, @SuppressWarnings("unchecked") T... values) { List list = new ArrayList<>(); - for (T val : values) { - list.add(val); - } + Collections.addAll(list, values); return extractNrRandomColumns(list, nr); } public static List subset(@SuppressWarnings("unchecked") T... values) { - List list = new ArrayList<>(); - for (T val : values) { - list.add(val); - } + List list = new ArrayList<>(Arrays.asList(values)); return subset(list); } @@ -350,7 +346,6 @@ public long getNonZeroInteger() { do { value = getInteger(); } while (value == 0); - assert value != 0; addToCache(value); return value; } @@ -425,7 +420,7 @@ public long getLong(long left, long right) { } public BigDecimal getRandomBigDecimal() { - return new BigDecimal(getThreadRandom().get().nextDouble()); + return BigDecimal.valueOf(getThreadRandom().get().nextDouble()); } public long getPositiveIntegerNotNull() { @@ -494,7 +489,7 @@ private static long getNextLong(long lower, long upper) { if (lower == upper) { return lower; } - return (long) (getThreadRandom().get().longs(lower, upper).findFirst().getAsLong()); + return getThreadRandom().get().longs(lower, upper).findFirst().getAsLong(); } private static int getNextInt(int lower, int upper) { diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java index 8de7523a3..efb4fab67 100644 --- a/src/sqlancer/SQLProviderAdapter.java +++ b/src/sqlancer/SQLProviderAdapter.java @@ -10,7 +10,7 @@ public abstract class SQLProviderAdapter>, O extends DBMSSpecificOptions>> extends ProviderAdapter { - public SQLProviderAdapter(Class globalClass, Class optionClass) { + protected SQLProviderAdapter(Class globalClass, Class optionClass) { super(globalClass, optionClass); } diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 7ac3e9ccf..33bfe3398 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -70,6 +70,9 @@ public List> getStatements() { return Collections.unmodifiableList(statements); } + /** + * @deprecated + */ @Deprecated public void commentStatements() { for (int i = 0; i < statements.size(); i++) { @@ -100,7 +103,7 @@ public class OracleRunReproductionState implements Closeable { private final List> statements = new ArrayList<>(); - public boolean success; + private boolean success; public OracleRunReproductionState() { StateToReproduce.this.localState = this; diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index 26265d136..4f8f48b8f 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -70,7 +70,7 @@ public void executeStatements() throws Exception { success = globalState.executeStatement(query); } while (nextAction.canBeRetried() && !success && nrTries++ < globalState.getOptions().getNrStatementRetryCount()); - } catch (IgnoreMeException e) { + } catch (IgnoreMeException ignored) { } if (query != null && query.couldAffectSchema()) { diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java index 122b34a83..f302d48ad 100644 --- a/src/sqlancer/common/DBMSCommon.java +++ b/src/sqlancer/common/DBMSCommon.java @@ -27,7 +27,7 @@ public static boolean matchesIndexName(String indexName) { return matcher.matches(); } - public static int getMaxIndexInDoubleArrary(double... doubleArray) { + public static int getMaxIndexInDoubleArray(double... doubleArray) { int maxIndex = 0; double maxValue = 0.0; for (int j = 0; j < doubleArray.length; j++) { diff --git a/src/sqlancer/common/ast/BinaryNode.java b/src/sqlancer/common/ast/BinaryNode.java index dec1854c4..90a56a078 100644 --- a/src/sqlancer/common/ast/BinaryNode.java +++ b/src/sqlancer/common/ast/BinaryNode.java @@ -7,7 +7,7 @@ public abstract class BinaryNode implements BinaryOperation { private final T left; private final T right; - public BinaryNode(T left, T right) { + protected BinaryNode(T left, T right) { this.left = left; this.right = right; } diff --git a/src/sqlancer/common/ast/BinaryOperatorNode.java b/src/sqlancer/common/ast/BinaryOperatorNode.java index 38d340a4b..586315059 100644 --- a/src/sqlancer/common/ast/BinaryOperatorNode.java +++ b/src/sqlancer/common/ast/BinaryOperatorNode.java @@ -10,7 +10,7 @@ public interface Operator { String getTextRepresentation(); } - public BinaryOperatorNode(T left, T right, O op) { + protected BinaryOperatorNode(T left, T right, O op) { super(left, right); this.op = op; } diff --git a/src/sqlancer/common/ast/FunctionNode.java b/src/sqlancer/common/ast/FunctionNode.java index a841d2956..ad795b998 100644 --- a/src/sqlancer/common/ast/FunctionNode.java +++ b/src/sqlancer/common/ast/FunctionNode.java @@ -7,7 +7,7 @@ public abstract class FunctionNode { protected F function; protected List args; - public FunctionNode(F function, List args) { + protected FunctionNode(F function, List args) { this.function = function; this.args = args; } diff --git a/src/sqlancer/common/ast/TernaryNode.java b/src/sqlancer/common/ast/TernaryNode.java index 146a46474..1a6605317 100644 --- a/src/sqlancer/common/ast/TernaryNode.java +++ b/src/sqlancer/common/ast/TernaryNode.java @@ -8,7 +8,7 @@ public abstract class TernaryNode implements BinaryOperation { private final T middle; private final T right; - public TernaryNode(T left, T middle, T right) { + protected TernaryNode(T left, T middle, T right) { this.left = left; this.middle = middle; this.right = right; diff --git a/src/sqlancer/common/ast/UnaryNode.java b/src/sqlancer/common/ast/UnaryNode.java index a6664c864..dea4220b9 100644 --- a/src/sqlancer/common/ast/UnaryNode.java +++ b/src/sqlancer/common/ast/UnaryNode.java @@ -6,7 +6,7 @@ public abstract class UnaryNode implements UnaryOperation { protected final T expr; - public UnaryNode(T expr) { + protected UnaryNode(T expr) { this.expr = expr; } diff --git a/src/sqlancer/common/ast/UnaryOperatorNode.java b/src/sqlancer/common/ast/UnaryOperatorNode.java index 979479c12..71556496c 100644 --- a/src/sqlancer/common/ast/UnaryOperatorNode.java +++ b/src/sqlancer/common/ast/UnaryOperatorNode.java @@ -6,7 +6,7 @@ public abstract class UnaryOperatorNode extends UnaryNode protected final O op; - public UnaryOperatorNode(T expr, O op) { + protected UnaryOperatorNode(T expr, O op) { super(expr); this.op = op; } diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java index 790d10709..7b7dfe4f1 100644 --- a/src/sqlancer/common/log/SQLLoggableFactory.java +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -37,10 +37,10 @@ public SQLQueryAdapter commentOutQuery(Query query) { @Override protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { StringBuilder sb = new StringBuilder(); - sb.append("-- Time: " + time + "\n"); - sb.append("-- Database: " + databaseName + "\n"); - sb.append("-- Database version: " + databaseVersion + "\n"); - sb.append("-- seed value: " + seedValue + "\n"); + sb.append("-- Time: ").append(time).append("\n"); + sb.append("-- Database: ").append(databaseName).append("\n"); + sb.append("-- Database version: ").append(databaseVersion).append("\n"); + sb.append("-- seed value: ").append(seedValue).append("\n"); return new LoggedString(sb.toString()); } diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java index c099b2592..2ac0dbb43 100644 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ b/src/sqlancer/common/oracle/NoRECBase.java @@ -16,7 +16,7 @@ public abstract class NoRECBase> implements TestO protected String optimizedQueryString; protected String unoptimizedQueryString; - public NoRECBase(S state) { + protected NoRECBase(S state) { this.state = state; this.con = state.getConnection(); this.logger = state.getLogger(); diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index 6c551d18e..cde8a2e4b 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -29,7 +29,7 @@ public abstract class PivotedQuerySynthesisBase, protected final S globalState; protected R pivotRow; - public PivotedQuerySynthesisBase(S globalState) { + protected PivotedQuerySynthesisBase(S globalState) { this.globalState = globalState; } diff --git a/src/sqlancer/common/schema/AbstractRowValue.java b/src/sqlancer/common/schema/AbstractRowValue.java index c7f933c9b..2a979514d 100644 --- a/src/sqlancer/common/schema/AbstractRowValue.java +++ b/src/sqlancer/common/schema/AbstractRowValue.java @@ -63,7 +63,7 @@ public String asStringGroupedByTables() { sb.append("\n"); } AbstractTable t = tableList.get(j); - sb.append("-- " + t.getName() + "\n"); + sb.append("-- ").append(t.getName()).append("\n"); List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) .collect(Collectors.toList()); for (int i = 0; i < columnsForTable.size(); i++) { diff --git a/src/sqlancer/common/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java index 84a854f72..1c66363e7 100644 --- a/src/sqlancer/common/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -45,7 +45,7 @@ public A getRandomTable(Predicate predicate) { } public A getRandomTableOrBailout(Function f) { - List relevantTables = databaseTables.stream().filter(t -> f.apply(t)).collect(Collectors.toList()); + List relevantTables = databaseTables.stream().filter(f::apply).collect(Collectors.toList()); if (relevantTables.isEmpty()) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index bf14abc1a..4a57fa846 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -19,7 +19,7 @@ public abstract class AbstractTable, I exten private final boolean isView; protected long rowCount = NO_ROW_COUNT_AVAILABLE; - public AbstractTable(String name, List columns, List indexes, boolean isView) { + protected AbstractTable(String name, List columns, List indexes, boolean isView) { this.name = name; this.indexes = indexes; this.isView = isView; @@ -41,7 +41,7 @@ public String toString() { sb.append(getName()); sb.append("\n"); for (C c : columns) { - sb.append("\t" + c + "\n"); + sb.append("\t").append(c).append("\n"); } return sb.toString(); } From 3d5197701fcc7a927d2541922649ca14817d86e9 Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: Thu, 9 Mar 2023 11:14:14 +0500 Subject: [PATCH 0758/1284] Add regex error strings on encoding and infinity conversion to generalize skip of these errors. (#731) --- src/sqlancer/postgres/gen/PostgresCommon.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index a3e04d046..99bb7fa66 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; @@ -53,7 +54,7 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("invalid hexadecimal digit"); errors.add("invalid hexadecimal data: odd number of digits"); errors.add("zero raised to a negative power is undefined"); - errors.add("cannot convert infinity to numeric"); + errors.addRegex(Pattern.compile("cannot convert infinity to \\w+")); errors.add("division by zero"); errors.add("invalid input syntax for type money"); errors.add("invalid input syntax for type"); @@ -101,7 +102,20 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("requested character not valid for encoding"); // chr errors.add("requested length too large"); // repeat errors.add("invalid memory alloc request size"); // repeat - errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + + /* + * PostgreSQL support only a few conversion variants to ASCII: LATIN1, LATIN2, LATIN9 and WINDOWS1250. So, it is + * better to skip this error at all. + */ + errors.addRegex(Pattern.compile("encoding conversion from \\w+ to ASCII not supported")); + + /* + * In accordance with PostgreSQL code, commit 0ab1a2e, conversions to or from SQL_ASCII is meaningless. So + * disable errors on such an attempt. + */ + errors.addRegex(Pattern.compile("encoding conversion from SQL_ASCII to \\w+ not supported")); + errors.addRegex(Pattern.compile("encoding conversion from \\w+ to SQL_ASCII not supported")); + errors.add("negative substring length not allowed"); // substr errors.add("invalid mask length"); // set_masklen } From 2aea15a440fed7b0448cd7c6613be47a0e0455f4 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 9 Mar 2023 15:03:15 +0800 Subject: [PATCH 0759/1284] feat: add missing python naming convention test (#755) * feat: add missing python naming convention test * fix: fix working directory in GitHub Action --------- Co-authored-by: Manuel Rigger --- .github/workflows/main.yml | 21 +++++++--- src/check_names.py | 86 ++++++++++++++++++++++---------------- 2 files changed, 66 insertions(+), 41 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 329f108da..7c2ccf637 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,6 +25,12 @@ jobs: run: mvn -B verify -DskipTests=true - name: Misc Tests run: mvn -B '-Dtest=!sqlancer.dbms.**' test + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Naming Convention Tests + run: python src/check_names.py citus: name: DBMS Tests (Citus) @@ -72,7 +78,6 @@ jobs: - name: Run Tests run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test - clickhouse: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest @@ -101,6 +106,7 @@ jobs: docker rm clickhouse-server + cockroachdb: name: DBMS Tests (CockroachDB) runs-on: ubuntu-latest @@ -123,7 +129,7 @@ jobs: run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - + cockroachdb-qpg: name: QPG Tests (CockroachDB) runs-on: ubuntu-latest @@ -173,6 +179,7 @@ jobs: run: | DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test + duckdb: name: DBMS Tests (DuckDB) runs-on: ubuntu-latest @@ -260,6 +267,7 @@ jobs: MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP + postgres: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest @@ -298,10 +306,11 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests - run: | + run: | mvn -Dtest=TestSQLitePQS test mvn -Dtest=TestSQLite3 test + sqlite-qpg: name: QPG Tests (SQLite) runs-on: ubuntu-latest @@ -317,9 +326,10 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests for QPG - run: | + run: | mvn -Dtest=TestSQLiteQPG test + tidb: name: DBMS Tests (TiDB) runs-on: ubuntu-latest @@ -365,7 +375,7 @@ jobs: run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test - + yugabyte: name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest @@ -421,7 +431,6 @@ jobs: - name: Shortly run DuckDB run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - java15: name: Java 15 EA Compatibility (DuckDB) runs-on: ubuntu-latest diff --git a/src/check_names.py b/src/check_names.py index 31d6853a4..9208cc00e 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -1,38 +1,54 @@ import os +import sys +from typing import List -def get_java_files(directory): - java_files = [] - for root, dirs, files in os.walk(directory): - for f in files: - if f.endswith('.java'): - java_files.append(f) - return java_files - -def verify_prefix(prefix, files): - if len(files) == 0: - print(prefix + ' directory does not contain any files!') - exit(-1) - for f in files: - if not f.startswith(prefix): - print('The class name of ' + f + ' does not start with ' + prefix) - exit(-1) - -verify_prefix('ArangoDB', get_java_files("sqlancer/arangodb/")) -verify_prefix('Citus', get_java_files("sqlancer/citus/")) -verify_prefix('ClickHouse', get_java_files("sqlancer/clickhouse/")) -verify_prefix('CnosDB', get_java_files("sqlancer/cnosdb/")) -verify_prefix('CockroachDB', get_java_files("sqlancer/cockroachdb/")) -verify_prefix('Cosmos', get_java_files("sqlancer/cosmos/")) -verify_prefix('Databend', get_java_files("sqlancer/databend/")) -verify_prefix('DuckDB', get_java_files("sqlancer/duckdb")) -verify_prefix('H2', get_java_files("sqlancer/h2")) -verify_prefix('HSQLDB', get_java_files("sqlancer/hsqldb")) -verify_prefix('MariaDB', get_java_files("sqlancer/mariadb/")) -verify_prefix('MySQL', get_java_files("sqlancer/mysql/")) -verify_prefix('OceanBase', get_java_files("sqlancer/oceanbase/")) -verify_prefix('Postgres', get_java_files("sqlancer/postgres/")) -verify_prefix('QuestDB', get_java_files("sqlancer/questdb/")) -verify_prefix('SQLite3', get_java_files("sqlancer/sqlite3/")) -verify_prefix('TiDB', get_java_files("sqlancer/tidb/")) -verify_prefix('Y', get_java_files("sqlancer/yugabyte/")) # has both YCQL and YSQL prefixes +def get_java_files(directory_path: str) -> List[str]: + java_files: List[str] = [] + for root, dirs, files in os.walk(directory_path): + for f in files: + if f.endswith('.java'): + java_files.append(f) + return java_files + + +def verify_one_db(prefix: str, files: List[str]): + print('checking database, name: {0}, files: {1}'.format(prefix, files)) + if len(files) == 0: + print(prefix + ' directory does not contain any files!', file=sys.stderr) + exit(-1) + for f in files: + if not f.startswith(prefix): + print('The class name of ' + f + ' does not start with ' + prefix, file=sys.stderr) + exit(-1) + print('checking database pass: ', prefix) + + +def verify_all_dbs(name_to_files: dict[str:List[str]]): + for db_name, files in name_to_files.items(): + verify_one_db(db_name, files) + + +if __name__ == '__main__': + cwd = os.getcwd() + print("Current working directory: {0}".format(cwd)) + name_to_files: dict[str:List[str]] = dict() + name_to_files["ArangoDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "arangodb")) + name_to_files["Citus"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "citus")) + name_to_files["ClickHouse"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "clickhouse")) + name_to_files["CnosDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cnosdb")) + name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) + name_to_files["Cosmos"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cosmos")) + name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) + name_to_files["DuckDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "duckdb")) + name_to_files["H2"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "h2")) + name_to_files["HSQLDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "hsqldb")) + name_to_files["MariaDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mariadb")) + name_to_files["MySQL"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mysql")) + name_to_files["OceanBase"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "oceanbase")) + name_to_files["Postgres"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "postgres")) + name_to_files["QuestDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "questdb")) + name_to_files["SQLite3"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "sqlite3")) + name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) + name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes + verify_all_dbs(name_to_files) From 7e77b2a75b8dccf6a7676014cb547637176c5202 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Fri, 10 Mar 2023 19:23:03 +0800 Subject: [PATCH 0760/1284] feat: add support for TimescaleDB (#748) * feat: add TimeScaleDB, initial basic structure of bugs, global state and schema * feat: add TimeScaleDB, check_names.py * feat: add Common, Error, Options, Provider. modify Schema * style: run mvn formatter * fix: error: The method getTimeScaleDBErrors() from the TimeScaleDBProvider is never used locally * fix: TestUsageNamingConvention.testNonEmptyDescription:23 TimeScaleDB should be in lowercase! * fix: change access right * refactor: rename TimeScaleDB to TimescaleDB * style: run mvn formatter * fix: remove expected errors timesaledb --- src/sqlancer/timescaledb/TimescaleDBBugs.java | 7 +++ .../timescaledb/TimescaleDBGlobalState.java | 12 +++++ .../timescaledb/TimescaleDBOptions.java | 27 ++++++++++ .../timescaledb/TimescaleDBProvider.java | 30 +++++++++++ .../timescaledb/TimescaleDBSchema.java | 52 +++++++++++++++++++ .../timescaledb/gen/TimescaleDBCommon.java | 13 +++++ 6 files changed, 141 insertions(+) create mode 100644 src/sqlancer/timescaledb/TimescaleDBBugs.java create mode 100644 src/sqlancer/timescaledb/TimescaleDBGlobalState.java create mode 100644 src/sqlancer/timescaledb/TimescaleDBOptions.java create mode 100644 src/sqlancer/timescaledb/TimescaleDBProvider.java create mode 100644 src/sqlancer/timescaledb/TimescaleDBSchema.java create mode 100644 src/sqlancer/timescaledb/gen/TimescaleDBCommon.java diff --git a/src/sqlancer/timescaledb/TimescaleDBBugs.java b/src/sqlancer/timescaledb/TimescaleDBBugs.java new file mode 100644 index 000000000..079dfee74 --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBBugs.java @@ -0,0 +1,7 @@ +package sqlancer.timescaledb; + +public final class TimescaleDBBugs { + private TimescaleDBBugs() { + + } +} diff --git a/src/sqlancer/timescaledb/TimescaleDBGlobalState.java b/src/sqlancer/timescaledb/TimescaleDBGlobalState.java new file mode 100644 index 000000000..519001b90 --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBGlobalState.java @@ -0,0 +1,12 @@ +package sqlancer.timescaledb; + +import java.sql.SQLException; + +import sqlancer.postgres.PostgresGlobalState; + +public class TimescaleDBGlobalState extends PostgresGlobalState { + @Override + public TimescaleDBSchema readSchema() throws SQLException { + return TimescaleDBSchema.fromConnection(getConnection(), getDatabaseName()); + } +} diff --git a/src/sqlancer/timescaledb/TimescaleDBOptions.java b/src/sqlancer/timescaledb/TimescaleDBOptions.java new file mode 100644 index 000000000..03b479662 --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBOptions.java @@ -0,0 +1,27 @@ +package sqlancer.timescaledb; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; + +public class TimescaleDBOptions extends PostgresOptions { + @Parameter(names = "--timescaledboracle", description = "Specifies which test oracle should be used for TimeScaleDB extension to PostgreSQL") + public List timescaleDBOracle = Arrays.asList(TimescaleDBOracleFactory.PQS); + + public enum TimescaleDBOracleFactory implements OracleFactory { + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + }, + } +} diff --git a/src/sqlancer/timescaledb/TimescaleDBProvider.java b/src/sqlancer/timescaledb/TimescaleDBProvider.java new file mode 100644 index 000000000..756669128 --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBProvider.java @@ -0,0 +1,30 @@ +package sqlancer.timescaledb; + +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.PostgresProvider; +import sqlancer.timescaledb.gen.TimescaleDBCommon; + +@AutoService(DatabaseProvider.class) +public class TimescaleDBProvider extends PostgresProvider { + @SuppressWarnings("unchecked") + public TimescaleDBProvider() { + super((Class) (Object) TimescaleDBGlobalState.class, + (Class) (Object) TimescaleDBOptions.class); + } + + @Override + public String getDBMSName() { + return "timescaledb"; + } + + public static ExpectedErrors getTimescaleDBErrors() { + ExpectedErrors errors = new ExpectedErrors(); + TimescaleDBCommon.addTimescaleDBErrors(errors); + return errors; + } +} diff --git a/src/sqlancer/timescaledb/TimescaleDBSchema.java b/src/sqlancer/timescaledb/TimescaleDBSchema.java new file mode 100644 index 000000000..0c09d579c --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBSchema.java @@ -0,0 +1,52 @@ +package sqlancer.timescaledb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.SQLConnection; +import sqlancer.postgres.PostgresSchema; + +public class TimescaleDBSchema extends PostgresSchema { + + public TimescaleDBSchema(List databaseTables, String databaseName) { + super(new ArrayList<>(databaseTables), databaseName); + } + + public static class TimescaleDBTable extends PostgresTable { + public TimescaleDBTable(String tableName, List columns, List indexes, + TableType tableType, List statistics, boolean isView, boolean isInsertable) { + super(tableName, columns, indexes, tableType, statistics, isView, isInsertable); + } + + public TimescaleDBTable(PostgresTable table) { + super(table.getName(), table.getColumns(), table.getIndexes(), table.getTableType(), table.getStatistics(), + table.isView(), table.isInsertable()); + } + } + + public static TimescaleDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement(); + ResultSet rs = s.executeQuery("SELECT table_name FROM information_schema.tables")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + + PostgresTable t = schema.getDatabaseTable(tableName); + if (t == null) { + continue; + } + + TimescaleDBTable table = new TimescaleDBTable(t); + databaseTables.add(table); + } + } catch (SQLIntegrityConstraintViolationException e) { + throw new AssertionError(e); + } + return new TimescaleDBSchema(databaseTables, databaseName); + } +} diff --git a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java new file mode 100644 index 000000000..1a13b9cb4 --- /dev/null +++ b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java @@ -0,0 +1,13 @@ +package sqlancer.timescaledb.gen; + +import sqlancer.common.query.ExpectedErrors; + +public final class TimescaleDBCommon { + private TimescaleDBCommon() { + + } + + public static void addTimescaleDBErrors(ExpectedErrors errors) { + + } +} From 354b95369b618f74a2e3b920ef7f75d963fcebd4 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 19 Mar 2023 18:54:42 +0800 Subject: [PATCH 0761/1284] Disable QPG tests in general Action (#766) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c2ccf637..3abbac71a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**' test + run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - name: Setup Python uses: actions/setup-python@v4 with: From 07db8b1be8447d10dc0a5f04fdb7e8d77e2890d4 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 20 Mar 2023 13:09:30 +0800 Subject: [PATCH 0762/1284] fix: ahere to max number of queries in QPG (#767) --- src/sqlancer/ProviderAdapter.java | 2 +- src/sqlancer/common/query/SQLQueryAdapter.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index ab5ca8e9a..1972557d6 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -125,7 +125,7 @@ public void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws E while (executedQueryCount < globalState.getOptions().getNrQueries()) { int numOfNoNewQueryPlans = 0; TestOracle oracle = getTestOracle(globalState); - while (true) { + while (executedQueryCount < globalState.getOptions().getNrQueries()) { try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { assert localState != null; try { diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index ab9124230..22eef268a 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -92,6 +92,8 @@ public String getUnterminatedQueryString() { Main.nrUnsuccessfulActions.addAndGet(1); checkException(e); return false; + } finally { + s.close(); } } From bc0fa8e7a0634dc637f1fd94a694618a9578c115 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Apr 2023 08:25:42 +0200 Subject: [PATCH 0763/1284] Add Materialize support (#773) * Copy postgres -> materialize as basis * Comment out all Postgres actions which don't work in Materialize * Rename Postgres to Materialize * Use table name in index to make sure we don't try recreating same index names * Fix formatting * Remove commented out original Pg code * Revert index name change * Add Materialize to CI * Prevent creating many indexes on Materialize * More reasonable defaults * Materialize: Fix QPG * Add Materialize to README * Fix issue running Main() method twice --- .github/workflows/main.yml | 50 ++ README.md | 3 +- configs/spotbugs-exclude.xml | 5 + src/check_names.py | 1 + .../MaterializeCompoundDataType.java | 46 ++ .../MaterializeExpectedValueVisitor.java | 162 ++++++ .../materialize/MaterializeGlobalState.java | 292 ++++++++++ .../materialize/MaterializeOptions.java | 93 ++++ .../materialize/MaterializeProvider.java | 331 +++++++++++ .../materialize/MaterializeSchema.java | 330 +++++++++++ .../MaterializeToStringVisitor.java | 335 ++++++++++++ .../materialize/MaterializeVisitor.java | 127 +++++ .../materialize/ast/MaterializeAggregate.java | 58 ++ .../materialize/ast/MaterializeAlias.java | 35 ++ .../ast/MaterializeBetweenOperation.java | 66 +++ .../MaterializeBinaryArithmeticOperation.java | 103 ++++ .../ast/MaterializeBinaryBitOperation.java | 46 ++ .../MaterializeBinaryComparisonOperation.java | 121 ++++ .../MaterializeBinaryLogicalOperation.java | 89 +++ .../ast/MaterializeBinaryRangeOperation.java | 74 +++ .../ast/MaterializeCastOperation.java | 45 ++ .../ast/MaterializeColumnValue.java | 34 ++ .../ast/MaterializeConcatOperation.java | 37 ++ .../materialize/ast/MaterializeConstant.java | 516 +++++++++++++++++ .../ast/MaterializeExpression.java | 14 + .../materialize/ast/MaterializeFunction.java | 256 +++++++++ .../MaterializeFunctionWithUnknownResult.java | 148 +++++ .../ast/MaterializeInOperation.java | 66 +++ .../materialize/ast/MaterializeJoin.java | 50 ++ .../ast/MaterializeLikeOperation.java | 38 ++ .../ast/MaterializeOrderByTerm.java | 42 ++ .../MaterializePOSIXRegularExpression.java | 65 +++ .../ast/MaterializePostfixOperation.java | 151 +++++ .../ast/MaterializePostfixText.java | 37 ++ .../ast/MaterializePrefixOperation.java | 119 ++++ .../materialize/ast/MaterializeSelect.java | 135 +++++ .../materialize/ast/MaterializeSimilarTo.java | 40 ++ .../materialize/gen/MaterializeCommon.java | 351 ++++++++++++ .../gen/MaterializeDeleteGenerator.java | 38 ++ .../gen/MaterializeDropIndexGenerator.java | 53 ++ .../gen/MaterializeExpressionGenerator.java | 517 ++++++++++++++++++ .../gen/MaterializeIndexGenerator.java | 80 +++ .../gen/MaterializeInsertGenerator.java | 106 ++++ .../gen/MaterializeRandomQueryGenerator.java | 64 +++ .../gen/MaterializeTableGenerator.java | 127 +++++ .../gen/MaterializeUpdateGenerator.java | 78 +++ .../gen/MaterializeViewGenerator.java | 73 +++ .../oracle/MaterializeNoRECOracle.java | 176 ++++++ ...aterializePivotedQuerySynthesisOracle.java | 151 +++++ .../tlp/MaterializeTLPAggregateOracle.java | 187 +++++++ .../oracle/tlp/MaterializeTLPBase.java | 123 +++++ .../tlp/MaterializeTLPHavingOracle.java | 83 +++ .../oracle/tlp/MaterializeTLPWhereOracle.java | 52 ++ test/sqlancer/dbms/TestMaterialize.java | 24 + test/sqlancer/dbms/TestMaterializePQS.java | 25 + .../qpg/materialize/TestMaterializeQPG.java | 23 + .../materialize/TestMaterializeQueryPlan.java | 49 ++ 57 files changed, 6539 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/materialize/MaterializeCompoundDataType.java create mode 100644 src/sqlancer/materialize/MaterializeExpectedValueVisitor.java create mode 100644 src/sqlancer/materialize/MaterializeGlobalState.java create mode 100644 src/sqlancer/materialize/MaterializeOptions.java create mode 100644 src/sqlancer/materialize/MaterializeProvider.java create mode 100644 src/sqlancer/materialize/MaterializeSchema.java create mode 100644 src/sqlancer/materialize/MaterializeToStringVisitor.java create mode 100644 src/sqlancer/materialize/MaterializeVisitor.java create mode 100644 src/sqlancer/materialize/ast/MaterializeAggregate.java create mode 100644 src/sqlancer/materialize/ast/MaterializeAlias.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBetweenOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeCastOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeColumnValue.java create mode 100644 src/sqlancer/materialize/ast/MaterializeConcatOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeConstant.java create mode 100644 src/sqlancer/materialize/ast/MaterializeExpression.java create mode 100644 src/sqlancer/materialize/ast/MaterializeFunction.java create mode 100644 src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java create mode 100644 src/sqlancer/materialize/ast/MaterializeInOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeJoin.java create mode 100644 src/sqlancer/materialize/ast/MaterializeLikeOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeOrderByTerm.java create mode 100644 src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java create mode 100644 src/sqlancer/materialize/ast/MaterializePostfixOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializePostfixText.java create mode 100644 src/sqlancer/materialize/ast/MaterializePrefixOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeSelect.java create mode 100644 src/sqlancer/materialize/ast/MaterializeSimilarTo.java create mode 100644 src/sqlancer/materialize/gen/MaterializeCommon.java create mode 100644 src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeIndexGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeInsertGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeTableGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeViewGenerator.java create mode 100644 src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java create mode 100644 src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java create mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java create mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java create mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java create mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java create mode 100644 test/sqlancer/dbms/TestMaterialize.java create mode 100644 test/sqlancer/dbms/TestMaterializePQS.java create mode 100644 test/sqlancer/qpg/materialize/TestMaterializeQPG.java create mode 100644 test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3abbac71a..1892285b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -239,6 +239,56 @@ jobs: - name: Run Tests run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test + materialize: + name: DBMS Tests (Materialize) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Materialize + run: | + docker pull materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest + sleep 5 + # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 + psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterialize + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializePQS + + materialize-qpg: + name: QPG Tests (Materialize) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Materialize + run: | + docker pull materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest + sleep 5 + # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 + psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQPG + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQueryPlan + mysql: name: DBMS Tests (MySQL) runs-on: ubuntu-18.04 diff --git a/README.md b/README.md index 346209908..59f69cce2 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ If you launch SQLancer without parameters, available options and commands are di | Approach | Description | |----------|-------------| | Random Generation | Random generation is the default test case generation approach in SQLancer. First, random tables are generated. Then queries are randomly generated based on the schemas of the tables. | -| Query Plan Guidance (QPG) | QPG is a test case generation method guided by query plan coverage. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. This approach is enabled by option `--qpg-enable` and now supports TLP and NoREC oracles for SQLite, CockroachDB, and TiDB. | +| Query Plan Guidance (QPG) | QPG is a test case generation method guided by query plan coverage. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. This approach is enabled by option `--qpg-enable` and now supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. | Please find the `.bib` entries [here](docs/PAPERS.md). @@ -69,6 +69,7 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | Databend | Working | Typed | | | QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | | CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | +| Materialize |Working | Typed | | # Using SQLancer diff --git a/configs/spotbugs-exclude.xml b/configs/spotbugs-exclude.xml index 5cc1baf73..7fa4de560 100644 --- a/configs/spotbugs-exclude.xml +++ b/configs/spotbugs-exclude.xml @@ -7,6 +7,11 @@ + + + + + diff --git a/src/check_names.py b/src/check_names.py index 9208cc00e..6dd4f1a72 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -44,6 +44,7 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["H2"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "h2")) name_to_files["HSQLDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "hsqldb")) name_to_files["MariaDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mariadb")) + name_to_files["Materialize"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "materialize")) name_to_files["MySQL"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mysql")) name_to_files["OceanBase"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "oceanbase")) name_to_files["Postgres"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "postgres")) diff --git a/src/sqlancer/materialize/MaterializeCompoundDataType.java b/src/sqlancer/materialize/MaterializeCompoundDataType.java new file mode 100644 index 000000000..a4bddf7ca --- /dev/null +++ b/src/sqlancer/materialize/MaterializeCompoundDataType.java @@ -0,0 +1,46 @@ +package sqlancer.materialize; + +import java.util.Optional; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public final class MaterializeCompoundDataType { + + private final MaterializeDataType dataType; + private final MaterializeCompoundDataType elemType; + private final Integer size; + + private MaterializeCompoundDataType(MaterializeDataType dataType, MaterializeCompoundDataType elemType, + Integer size) { + this.dataType = dataType; + this.elemType = elemType; + this.size = size; + } + + public MaterializeDataType getDataType() { + return dataType; + } + + public MaterializeCompoundDataType getElemType() { + if (elemType == null) { + throw new AssertionError(); + } + return elemType; + } + + public Optional getSize() { + if (size == null) { + return Optional.empty(); + } else { + return Optional.of(size); + } + } + + public static MaterializeCompoundDataType create(MaterializeDataType type, int size) { + return new MaterializeCompoundDataType(type, null, size); + } + + public static MaterializeCompoundDataType create(MaterializeDataType type) { + return new MaterializeCompoundDataType(type, null, null); + } +} diff --git a/src/sqlancer/materialize/MaterializeExpectedValueVisitor.java b/src/sqlancer/materialize/MaterializeExpectedValueVisitor.java new file mode 100644 index 000000000..caa801268 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeExpectedValueVisitor.java @@ -0,0 +1,162 @@ +package sqlancer.materialize; + +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSimilarTo; + +public final class MaterializeExpectedValueVisitor implements MaterializeVisitor { + + private final StringBuilder sb = new StringBuilder(); + private static final int NR_TABS = 0; + + private void print(MaterializeExpression expr) { + MaterializeToStringVisitor v = new MaterializeToStringVisitor(); + v.visit(expr); + for (int i = 0; i < NR_TABS; i++) { + sb.append("\t"); + } + sb.append(v.get()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + @Override + public void visit(MaterializeConstant constant) { + print(constant); + } + + @Override + public void visit(MaterializePostfixOperation op) { + print(op); + visit(op.getExpression()); + } + + public String get() { + return sb.toString(); + } + + @Override + public void visit(MaterializeColumnValue c) { + print(c); + } + + @Override + public void visit(MaterializePrefixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(MaterializeSelect op) { + visit(op.getWhereClause()); + } + + @Override + public void visit(MaterializeOrderByTerm op) { + + } + + @Override + public void visit(MaterializeFunction f) { + print(f); + for (int i = 0; i < f.getArguments().length; i++) { + visit(f.getArguments()[i]); + } + } + + @Override + public void visit(MaterializeCastOperation cast) { + print(cast); + visit(cast.getExpression()); + } + + @Override + public void visit(MaterializeBetweenOperation op) { + print(op); + visit(op.getExpr()); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(MaterializeInOperation op) { + print(op); + visit(op.getExpr()); + for (MaterializeExpression right : op.getListElements()) { + visit(right); + } + } + + @Override + public void visit(MaterializePostfixText op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(MaterializeAggregate op) { + print(op); + for (MaterializeExpression expr : op.getArgs()) { + visit(expr); + } + } + + @Override + public void visit(MaterializeSimilarTo op) { + print(op); + visit(op.getString()); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + } + + @Override + public void visit(MaterializePOSIXRegularExpression op) { + print(op); + visit(op.getString()); + visit(op.getRegex()); + } + + @Override + public void visit(MaterializeFromTable from) { + print(from); + } + + @Override + public void visit(MaterializeSubquery subquery) { + print(subquery); + } + + @Override + public void visit(MaterializeBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(MaterializeLikeOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + +} diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java new file mode 100644 index 000000000..77cbe5d14 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -0,0 +1,292 @@ +package sqlancer.materialize; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; + +public class MaterializeGlobalState extends SQLGlobalState { + + public static final char IMMUTABLE = 'i'; + public static final char STABLE = 's'; + public static final char VOLATILE = 'v'; + + private List operators = Collections.emptyList(); + private List collates = Collections.emptyList(); + private List opClasses = Collections.emptyList(); + private List tableAccessMethods = Collections.emptyList(); + // store and allow filtering by function volatility classifications + private final Map functionsAndTypes = new HashMap<>(); + private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + + @Override + public void setConnection(SQLConnection con) { + super.setConnection(con); + try { + this.opClasses = getOpclasses(); + this.operators = getOperators(getConnection()); + this.collates = getCollnames(getConnection()); + this.tableAccessMethods = getTableAccessMethods(getConnection()); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + private List getCollnames(SQLConnection con) throws SQLException { + List collNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("SELECT collname FROM pg_collation WHERE collname LIKE '%utf8' or collname = 'C';")) { + while (rs.next()) { + collNames.add(rs.getString(1)); + } + } + } + return collNames; + } + + private List getOpclasses() throws SQLException { + List opClasses = new ArrayList<>(); + // select opcname FROM pg_opclass; + // ERROR: unknown catalog item 'pg_opclass' + opClasses.add("array_ops"); + opClasses.add("array_ops"); + opClasses.add("bit_ops"); + opClasses.add("bool_ops"); + opClasses.add("bpchar_ops"); + opClasses.add("bpchar_ops"); + opClasses.add("bytea_ops"); + opClasses.add("char_ops"); + opClasses.add("char_ops"); + opClasses.add("cidr_ops"); + opClasses.add("cidr_ops"); + opClasses.add("date_ops"); + opClasses.add("date_ops"); + opClasses.add("float4_ops"); + opClasses.add("float4_ops"); + opClasses.add("float8_ops"); + opClasses.add("float8_ops"); + opClasses.add("inet_ops"); + opClasses.add("inet_ops"); + opClasses.add("inet_ops"); + opClasses.add("inet_ops"); + opClasses.add("int2_ops"); + opClasses.add("int2_ops"); + opClasses.add("int4_ops"); + opClasses.add("int4_ops"); + opClasses.add("int8_ops"); + opClasses.add("int8_ops"); + opClasses.add("interval_ops"); + opClasses.add("interval_ops"); + opClasses.add("macaddr_ops"); + opClasses.add("macaddr_ops"); + opClasses.add("macaddr8_ops"); + opClasses.add("macaddr8_ops"); + opClasses.add("name_ops"); + opClasses.add("name_ops"); + opClasses.add("numeric_ops"); + opClasses.add("numeric_ops"); + opClasses.add("oid_ops"); + opClasses.add("oid_ops"); + opClasses.add("oidvector_ops"); + opClasses.add("oidvector_ops"); + opClasses.add("record_ops"); + opClasses.add("record_image_ops"); + opClasses.add("text_ops"); + opClasses.add("text_ops"); + opClasses.add("time_ops"); + opClasses.add("time_ops"); + opClasses.add("timestamptz_ops"); + opClasses.add("timestamptz_ops"); + opClasses.add("timetz_ops"); + opClasses.add("timetz_ops"); + opClasses.add("varbit_ops"); + opClasses.add("varchar_ops"); + opClasses.add("varchar_ops"); + opClasses.add("timestamp_ops"); + opClasses.add("timestamp_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("money_ops"); + opClasses.add("bool_ops"); + opClasses.add("bytea_ops"); + opClasses.add("tid_ops"); + opClasses.add("xid_ops"); + opClasses.add("cid_ops"); + opClasses.add("tid_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("aclitem_ops"); + opClasses.add("box_ops"); + opClasses.add("point_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("money_ops"); + opClasses.add("bool_ops"); + opClasses.add("bytea_ops"); + opClasses.add("tid_ops"); + opClasses.add("xid_ops"); + opClasses.add("cid_ops"); + opClasses.add("tid_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("aclitem_ops"); + opClasses.add("box_ops"); + opClasses.add("point_ops"); + opClasses.add("poly_ops"); + opClasses.add("circle_ops"); + opClasses.add("array_ops"); + opClasses.add("uuid_ops"); + opClasses.add("uuid_ops"); + opClasses.add("pg_lsn_ops"); + opClasses.add("pg_lsn_ops"); + opClasses.add("enum_ops"); + opClasses.add("enum_ops"); + opClasses.add("tsvector_ops"); + opClasses.add("tsvector_ops"); + opClasses.add("tsvector_ops"); + opClasses.add("tsquery_ops"); + opClasses.add("tsquery_ops"); + opClasses.add("range_ops"); + opClasses.add("range_ops"); + opClasses.add("range_ops"); + opClasses.add("range_ops"); + opClasses.add("box_ops"); + opClasses.add("quad_point_ops"); + opClasses.add("kd_point_ops"); + opClasses.add("text_ops"); + opClasses.add("poly_ops"); + opClasses.add("jsonb_ops"); + opClasses.add("jsonb_ops"); + opClasses.add("jsonb_ops"); + opClasses.add("jsonb_path_ops"); + opClasses.add("bytea_minmax_ops"); + opClasses.add("char_minmax_ops"); + opClasses.add("name_minmax_ops"); + opClasses.add("int8_minmax_ops"); + opClasses.add("int2_minmax_ops"); + opClasses.add("int4_minmax_ops"); + opClasses.add("text_minmax_ops"); + opClasses.add("oid_minmax_ops"); + opClasses.add("tid_minmax_ops"); + opClasses.add("float4_minmax_ops"); + opClasses.add("float8_minmax_ops"); + opClasses.add("macaddr_minmax_ops"); + opClasses.add("macaddr8_minmax_ops"); + opClasses.add("inet_minmax_ops"); + opClasses.add("inet_inclusion_ops"); + opClasses.add("bpchar_minmax_ops"); + opClasses.add("time_minmax_ops"); + opClasses.add("date_minmax_ops"); + opClasses.add("timestamp_minmax_ops"); + opClasses.add("timestamptz_minmax_ops"); + opClasses.add("interval_minmax_ops"); + opClasses.add("timetz_minmax_ops"); + opClasses.add("bit_minmax_ops"); + opClasses.add("varbit_minmax_ops"); + opClasses.add("numeric_minmax_ops"); + opClasses.add("uuid_minmax_ops"); + opClasses.add("range_inclusion_ops"); + opClasses.add("pg_lsn_minmax_ops"); + opClasses.add("box_inclusion_ops"); + return opClasses; + } + + private List getOperators(SQLConnection con) throws SQLException { + List operators = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { + while (rs.next()) { + operators.add(rs.getString(1)); + } + } + } + return operators; + } + + private List getTableAccessMethods(SQLConnection con) throws SQLException { + List tableAccessMethods = new ArrayList<>(); + try (Statement s = con.createStatement()) { + /* + * pg_am includes both index and table access methods so we need to filter with amtype = 't' + */ + try (ResultSet rs = s.executeQuery("SELECT amname FROM pg_am WHERE amtype = 't';")) { + while (rs.next()) { + tableAccessMethods.add(rs.getString(1)); + } + } + } + return tableAccessMethods; + } + + public List getOperators() { + return operators; + } + + public String getRandomOperator() { + return Randomly.fromList(operators); + } + + public List getCollates() { + return collates; + } + + public String getRandomCollate() { + return Randomly.fromList(collates); + } + + public List getOpClasses() { + return opClasses; + } + + public String getRandomOpclass() { + return Randomly.fromList(opClasses); + } + + public List getTableAccessMethods() { + return tableAccessMethods; + } + + public String getRandomTableAccessMethod() { + return Randomly.fromList(tableAccessMethods); + } + + @Override + public MaterializeSchema readSchema() throws SQLException { + return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + } + + public void addFunctionAndType(String functionName, Character functionType) { + this.functionsAndTypes.put(functionName, functionType); + } + + public Map getFunctionsAndTypes() { + return this.functionsAndTypes; + } + + public void setAllowedFunctionTypes(List types) { + this.allowedFunctionTypes = types; + } + + public void setDefaultAllowedFunctionTypes() { + this.allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + } + + public List getAllowedFunctionTypes() { + return this.allowedFunctionTypes; + } + +} diff --git a/src/sqlancer/materialize/MaterializeOptions.java b/src/sqlancer/materialize/MaterializeOptions.java new file mode 100644 index 000000000..23191e3a9 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeOptions.java @@ -0,0 +1,93 @@ +package sqlancer.materialize; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.materialize.MaterializeOptions.MaterializeOracleFactory; +import sqlancer.materialize.oracle.MaterializeNoRECOracle; +import sqlancer.materialize.oracle.MaterializePivotedQuerySynthesisOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPAggregateOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPHavingOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPWhereOracle; + +@Parameters(separators = "=", commandDescription = "Materialize (default port: " + MaterializeOptions.DEFAULT_PORT + + ", default host: " + MaterializeOptions.DEFAULT_HOST + ", default user: " + MaterializeOptions.DEFAULT_USER + + ")") +public class MaterializeOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final String DEFAULT_USER = "materialize"; + public static final int DEFAULT_PORT = 6875; + + @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) + public boolean allowBulkInsert; + + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for Materialize") + public List oracle = Arrays.asList(MaterializeOracleFactory.QUERY_PARTITIONING); + + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) + public boolean testCollations = true; + + @Parameter(names = "--set-max-tables-mvs", description = "Specifies whether to set the maximum number of tables and materialized views intiially", arity = 1) + public boolean setMaxTablesMVs; + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the Materialize server", arity = 1) + public String connectionURL = String.format("postgresql://%s@%s:%d/test", MaterializeOptions.DEFAULT_USER, + MaterializeOptions.DEFAULT_HOST, MaterializeOptions.DEFAULT_PORT); + + @Parameter(names = "--extensions", description = "Specifies a comma-separated list of extension names to be created in each test database", arity = 1) + public String extensions = ""; + + public enum MaterializeOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializeNoRECOracle(globalState); + } + }, + PQS { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + HAVING { + + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializeTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); + oracles.add(new MaterializeTLPWhereOracle(globalState)); + oracles.add(new MaterializeTLPHavingOracle(globalState)); + oracles.add(new MaterializeTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + + } + + @Override + public List getTestOracleFactory() { + return oracle; + } + +} diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java new file mode 100644 index 000000000..dc802d2aa --- /dev/null +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -0,0 +1,331 @@ +package sqlancer.materialize; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.materialize.MaterializeOptions.MaterializeOracleFactory; +import sqlancer.materialize.gen.MaterializeDeleteGenerator; +import sqlancer.materialize.gen.MaterializeDropIndexGenerator; +import sqlancer.materialize.gen.MaterializeIndexGenerator; +import sqlancer.materialize.gen.MaterializeInsertGenerator; +import sqlancer.materialize.gen.MaterializeTableGenerator; +import sqlancer.materialize.gen.MaterializeUpdateGenerator; +import sqlancer.materialize.gen.MaterializeViewGenerator; + +// EXISTS +// IN +@AutoService(DatabaseProvider.class) +public class MaterializeProvider extends SQLProviderAdapter { + + /** + * Generate only data types and expressions that are understood by PQS. + */ + public static boolean generateOnlyKnown; + + protected String entryURL; + protected String username; + protected String password; + protected String entryPath; + protected String host; + protected int port; + protected String testURL; + protected String databaseName; + protected String createDatabaseCommand; + protected String extensionsList; + + public MaterializeProvider() { + super(MaterializeGlobalState.class, MaterializeOptions.class); + } + + protected MaterializeProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + public enum Action implements AbstractAction { + DELETE(MaterializeDeleteGenerator::create), // + DROP_INDEX(MaterializeDropIndexGenerator::create), // + INSERT(MaterializeInsertGenerator::insert), // + UPDATE(MaterializeUpdateGenerator::create), // + CREATE_INDEX(MaterializeIndexGenerator::generate), // + CREATE_VIEW(MaterializeViewGenerator::create); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(MaterializeGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + protected static int mapActions(MaterializeGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + switch (a) { + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 3); + break; + case DROP_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case DELETE: + nrPerformed = r.getInteger(0, 5); + break; + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + + } + + @Override + public void generateDatabase(MaterializeGlobalState globalState) throws Exception { + readFunctions(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + prepareTables(globalState); + + extensionsList = globalState.getDbmsSpecificOptions().extensions; + if (!extensionsList.isEmpty()) { + String[] extensionNames = extensionsList.split(","); + + /* + * To avoid of a test interference with an extension objects, create them in a separate schema. Of course, + * they must be truly relocatable. + */ + globalState.executeStatement(new SQLQueryAdapter("CREATE SCHEMA extensions;", true)); + for (int i = 0; i < extensionNames.length; i++) { + globalState.executeStatement(new SQLQueryAdapter( + "CREATE EXTENSION " + extensionNames[i] + " WITH SCHEMA extensions;", true)); + } + } + } + + @Override + public SQLConnection createDatabase(MaterializeGlobalState globalState) throws SQLException { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == MaterializeOracleFactory.PQS)) { + generateOnlyKnown = true; + } + + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + entryPath = "/test"; + entryURL = globalState.getDbmsSpecificOptions().connectionURL; + // trim URL to exclude "jdbc:" + if (entryURL.startsWith("jdbc:")) { + entryURL = entryURL.substring(5); + } + String entryDatabaseName = entryPath.substring(1); + databaseName = globalState.getDatabaseName(); + + try { + URI uri = new URI(entryURL); + String userInfoURI = uri.getUserInfo(); + String pathURI = uri.getPath(); + if (userInfoURI != null) { + // username and password specified in URL take precedence + if (userInfoURI.contains(":")) { + String[] userInfo = userInfoURI.split(":", 2); + username = userInfo[0]; + password = userInfo[1]; + } else { + username = userInfoURI; + password = null; + } + int userInfoIndex = entryURL.indexOf(userInfoURI); + String preUserInfo = entryURL.substring(0, userInfoIndex); + String postUserInfo = entryURL.substring(userInfoIndex + userInfoURI.length() + 1); + entryURL = preUserInfo + postUserInfo; + } + if (pathURI != null) { + entryPath = pathURI; + } + if (host == null) { + host = uri.getHost(); + } + if (port == MainOptions.NO_SET_PORT) { + port = uri.getPort(); + } + entryURL = String.format("%s://%s:%d/%s", uri.getScheme(), host, port, entryDatabaseName); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + createDatabaseCommand = getCreateDatabaseCommand(globalState); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + if (globalState.getDbmsSpecificOptions().setMaxTablesMVs) { + Connection conMzSystem = DriverManager.getConnection("jdbc:postgresql://localhost:6877/materialize", + "mz_system", "materialize"); + try (Statement s = conMzSystem.createStatement()) { + s.execute("ALTER SYSTEM SET max_tables TO 1000"); + } + try (Statement s = conMzSystem.createStatement()) { + s.execute("ALTER SYSTEM SET max_materialized_views TO 1000"); + } + conMzSystem.close(); + } + int databaseIndex = entryURL.indexOf(entryDatabaseName); + String preDatabaseName = entryURL.substring(0, databaseIndex); + String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); + testURL = preDatabaseName + databaseName + postDatabaseName; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + + con = DriverManager.getConnection("jdbc:" + testURL, username, password); + return new SQLConnection(con); + } + + protected void readFunctions(MaterializeGlobalState globalState) throws SQLException { + // ERROR: column "provolatile" does not exist + SQLQueryAdapter query = new SQLQueryAdapter("SELECT proname, 1 FROM pg_proc;"); + SQLancerResultSet rs = query.executeAndGet(globalState); + while (rs.next()) { + String functionName = rs.getString(1); + Character functionType = rs.getString(2).charAt(0); + globalState.addFunctionAndType(functionName, functionType); + } + } + + protected void createTables(MaterializeGlobalState globalState, int numTables) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = MaterializeTableGenerator.generate(tableName, globalState.getSchema(), + generateOnlyKnown, globalState); + globalState.executeStatement(createTable); + } catch (IgnoreMeException e) { + + } + } + } + + protected void prepareTables(MaterializeGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + MaterializeProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 5000;\n")); + } + + private String getCreateDatabaseCommand(MaterializeGlobalState state) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE DATABASE " + databaseName + " "); + if (Randomly.getBoolean() && ((MaterializeOptions) state.getDbmsSpecificOptions()).testCollations) { + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } + } + } + return sb.toString(); + } + + @Override + public String getDBMSName() { + return "materialize"; + } + + @Override + public String getQueryPlan(String selectStr, MaterializeGlobalState globalState) throws Exception { + String queryPlan = ""; + String explainQuery = "EXPLAIN OPTIMIZED PLAN FOR " + selectStr; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery); + boolean afterProjection = false; // Remove the concrete expression after each Projection operator + SQLancerResultSet rs = q.executeAndGet(globalState); + if (rs != null) { + while (rs.next()) { + String line; + BufferedReader bufReader = new BufferedReader(new StringReader(rs.getString(1))); + while ((line = bufReader.readLine()) != null) { + String targetQueryPlan = line.trim() + ";"; // Unify format + if (targetQueryPlan.startsWith("Explained Query:")) { + continue; + } + if (afterProjection) { + afterProjection = false; + continue; + } + if (targetQueryPlan.startsWith("Project")) { + afterProjection = true; + } + // Remove all concrete expressions by keywords + if (targetQueryPlan.contains(">") || targetQueryPlan.contains("<") || targetQueryPlan.contains("=") + || targetQueryPlan.contains("*") || targetQueryPlan.contains("+") + || targetQueryPlan.contains("'")) { + continue; + } + queryPlan += targetQueryPlan; + } + } + } + + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, MaterializeGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } +} diff --git a/src/sqlancer/materialize/MaterializeSchema.java b/src/sqlancer/materialize/MaterializeSchema.java new file mode 100644 index 000000000..1762c5cb9 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeSchema.java @@ -0,0 +1,330 @@ +package sqlancer.materialize; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.postgresql.util.PSQLException; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTable.TableType; +import sqlancer.materialize.ast.MaterializeConstant; + +public class MaterializeSchema extends AbstractSchema { + + private final String databaseName; + private final List indexNames; + + public List getIndexNames() { + return indexNames; + } + + public enum MaterializeDataType { + INT, BOOLEAN, TEXT, DECIMAL, FLOAT, REAL, BIT; + + public static MaterializeDataType getRandomType() { + List dataTypes = new ArrayList<>(Arrays.asList(values())); + if (MaterializeProvider.generateOnlyKnown) { + dataTypes.remove(MaterializeDataType.DECIMAL); + dataTypes.remove(MaterializeDataType.FLOAT); + dataTypes.remove(MaterializeDataType.REAL); + dataTypes.remove(MaterializeDataType.BIT); + } + return Randomly.fromList(dataTypes); + } + } + + public static class MaterializeColumn extends AbstractTableColumn { + + public MaterializeColumn(String name, MaterializeDataType columnType) { + super(name, null, columnType); + } + + public static MaterializeColumn createDummy(String name) { + return new MaterializeColumn(name, MaterializeDataType.INT); + } + + } + + public static class MaterializeTables extends AbstractTables { + + public MaterializeTables(List tables) { + super(tables); + } + + public MaterializeRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String randomRow = String.format("SELECT %s FROM %s LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet randomRowValues = s.executeQuery(randomRow); + if (!randomRowValues.next()) { + throw new AssertionError("could not find random row! " + randomRow + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + MaterializeColumn column = getColumns().get(i); + int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + MaterializeConstant constant; + if (randomRowValues.getString(columnIndex) == null) { + constant = MaterializeConstant.createNullConstant(); + } else { + switch (column.getType()) { + case INT: + constant = MaterializeConstant.createIntConstant(randomRowValues.getLong(columnIndex)); + break; + case BOOLEAN: + constant = MaterializeConstant + .createBooleanConstant(randomRowValues.getBoolean(columnIndex)); + break; + case TEXT: + constant = MaterializeConstant.createTextConstant(randomRowValues.getString(columnIndex)); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !randomRowValues.next(); + return new MaterializeRowValue(this, values); + } catch (PSQLException e) { + throw new IgnoreMeException(); + } + + } + + } + + public static MaterializeDataType getColumnType(String typeString) { + switch (typeString) { + case "smallint": + case "integer": + case "bigint": + return MaterializeDataType.INT; + case "boolean": + return MaterializeDataType.BOOLEAN; + case "text": + case "character": + case "character varying": + case "name": + case "regclass": + return MaterializeDataType.TEXT; + case "numeric": + return MaterializeDataType.DECIMAL; + case "double precision": + return MaterializeDataType.FLOAT; + case "real": + return MaterializeDataType.REAL; + case "bit": + return MaterializeDataType.BIT; + default: + throw new AssertionError(typeString); + } + } + + public static class MaterializeRowValue + extends AbstractRowValue { + + protected MaterializeRowValue(MaterializeTables tables, Map values) { + super(tables, values); + } + + } + + public static class MaterializeTable + extends AbstractRelationalTable { + + public enum TableType { + STANDARD, TEMPORARY + } + + private final TableType tableType; + private final List statistics; + private final boolean isInsertable; + + public MaterializeTable(String tableName, List columns, List indexes, + TableType tableType, List statistics, boolean isView, + boolean isInsertable) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + } + + public List getStatistics() { + return statistics; + } + + public TableType getTableType() { + return tableType; + } + + public boolean isInsertable() { + return isInsertable; + } + + } + + public static final class MaterializeStatisticsObject { + private final String name; + + public MaterializeStatisticsObject(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + public static final class MaterializeIndex extends TableIndex { + + private MaterializeIndex(String indexName) { + super(indexName); + } + + public static MaterializeIndex create(String indexName) { + return new MaterializeIndex(indexName); + } + + @Override + public String getIndexName() { + if (super.getIndexName().contentEquals("PRIMARY")) { + return "`PRIMARY`"; + } else { + return super.getIndexName(); + } + } + + } + + public static MaterializeSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + try { + List databaseTables = new ArrayList<>(); + List indexNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + // ERROR: column "is_insertable_into" does not exist + try (ResultSet rs = s.executeQuery( + "SELECT table_name, table_schema, table_type FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + String tableTypeSchema = rs.getString("table_schema"); + boolean isInsertable = true; + String type = rs.getString("table_type"); + boolean isView = type.equals("VIEW") || type.equals("MATERIALIZED VIEW"); + if (isView) { + isInsertable = false; + } + MaterializeTable.TableType tableType = getTableType(tableTypeSchema); + List databaseColumns = getTableColumns(con, tableName); + List indexes = getIndexes(con, tableName); + List statistics = getStatistics(con); + MaterializeTable t = new MaterializeTable(tableName, databaseColumns, indexes, tableType, + statistics, isView, isInsertable); + for (MaterializeColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT mz_indexes.name, mz_databases.name FROM mz_indexes JOIN mz_relations ON mz_indexes.on_id = mz_relations.id JOIN mz_schemas ON mz_relations.schema_id = mz_schemas.id JOIN mz_databases ON mz_schemas.database_id = mz_databases.id WHERE mz_databases.name = '%s';", + databaseName))) { + while (rs.next()) { + String name = rs.getString(1); + indexNames.add(name); + } + } + } + return new MaterializeSchema(databaseTables, databaseName, indexNames); + } catch (SQLIntegrityConstraintViolationException e) { + throw new AssertionError(e); + } + } + + protected static List getStatistics(SQLConnection con) throws SQLException { + return new ArrayList<>(); + } + + protected static MaterializeTable.TableType getTableType(String tableTypeStr) throws AssertionError { + MaterializeTable.TableType tableType; + if (tableTypeStr.contentEquals("public")) { + tableType = TableType.STANDARD; + } else if (tableTypeStr.startsWith("pg_temp")) { + tableType = TableType.TEMPORARY; + } else { + throw new AssertionError(tableTypeStr); + } + return tableType; + } + + protected static List getIndexes(SQLConnection con, String tableName) throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String + // org.postgresql.util.PSQLException: ERROR: unknown catalog item 'pg_indexes' + .format("SELECT c.relname as indexname FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid WHERE c.relkind IN ('i','I','') AND n.nspname <> 'pg_catalog' AND n.nspname !~ '^pg_toast' AND n.nspname <> 'information_schema' AND c2.relname = '%s' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY indexname;", + tableName))) { + while (rs.next()) { + String indexName = rs.getString("indexname"); + if (DBMSCommon.matchesIndexName(indexName)) { + indexes.add(MaterializeIndex.create(indexName)); + } + } + } + } + return indexes; + } + + protected static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("select column_name, data_type from INFORMATION_SCHEMA.COLUMNS where table_name = '" + + tableName + "' ORDER BY column_name")) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + MaterializeColumn c = new MaterializeColumn(columnName, getColumnType(dataType)); + columns.add(c); + } + } + } + return columns; + } + + public MaterializeSchema(List databaseTables, String databaseName, List indexNames) { + super(databaseTables); + this.databaseName = databaseName; + this.indexNames = indexNames; + } + + public MaterializeTables getRandomTableNonEmptyTables() { + return new MaterializeTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public String getDatabaseName() { + return databaseName; + } + +} diff --git a/src/sqlancer/materialize/MaterializeToStringVisitor.java b/src/sqlancer/materialize/MaterializeToStringVisitor.java new file mode 100644 index 000000000..6a3f06445 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeToStringVisitor.java @@ -0,0 +1,335 @@ +package sqlancer.materialize; + +import java.util.Optional; + +import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSimilarTo; + +public final class MaterializeToStringVisitor extends ToStringVisitor + implements MaterializeVisitor { + + @Override + public void visitSpecific(MaterializeExpression expr) { + MaterializeVisitor.super.visit(expr); + } + + @Override + public void visit(MaterializeConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(MaterializePostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" "); + sb.append(op.getOperatorTextRepresentation()); + } + + @Override + public void visit(MaterializeColumnValue c) { + sb.append(c.getColumn().getFullQualifiedName()); + } + + @Override + public void visit(MaterializePrefixOperation op) { + sb.append(op.getTextRepresentation()); + sb.append(" ("); + visit(op.getExpression()); + sb.append(")"); + } + + @Override + public void visit(MaterializeFromTable from) { + sb.append(from.getTable().getName()); + } + + @Override + public void visit(MaterializeSubquery subquery) { + sb.append("("); + visit(subquery.getSelect()); + sb.append(") AS "); + sb.append(subquery.getName()); + } + + @Override + public void visit(MaterializeSelect s) { + sb.append("SELECT "); + switch (s.getSelectOption()) { + case DISTINCT: + sb.append("DISTINCT "); + if (s.getDistinctOnClause() != null) { + sb.append("ON ("); + visit(s.getDistinctOnClause()); + sb.append(") "); + } + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + visit(s.getFetchColumns()); + sb.append(" FROM "); + visit(s.getFromList()); + + for (MaterializeJoin j : s.getJoinClauses()) { + sb.append(" "); + switch (j.getType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } + sb.append("JOIN"); + break; + case LEFT: + sb.append("LEFT OUTER JOIN"); + break; + case RIGHT: + sb.append("RIGHT OUTER JOIN"); + break; + case FULL: + sb.append("FULL OUTER JOIN"); + break; + case CROSS: + sb.append("CROSS JOIN"); + break; + default: + throw new AssertionError(j.getType()); + } + sb.append(" "); + visit(j.getTableReference()); + if (j.getType() != MaterializeJoinType.CROSS) { + sb.append(" ON "); + visit(j.getOnClause()); + } + } + + if (s.getWhereClause() != null) { + sb.append(" WHERE "); + visit(s.getWhereClause()); + } + if (s.getGroupByExpressions().size() > 0) { + sb.append(" GROUP BY "); + visit(s.getGroupByExpressions()); + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + + } + if (!s.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(s.getOrderByExpressions()); + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(MaterializeOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder()); + } + + @Override + public void visit(MaterializeFunction f) { + sb.append(f.getFunctionName()); + sb.append("("); + int i = 0; + for (MaterializeExpression arg : f.getArguments()) { + if (i++ != 0) { + sb.append(", "); + } + visit(arg); + } + sb.append(")"); + } + + @Override + public void visit(MaterializeCastOperation cast) { + if (cast.getCompoundType().getDataType() == MaterializeDataType.BOOLEAN) { + sb.append("("); + MaterializeExpression expr = cast.getExpression(); + visit(expr); + if (expr.getExpressionType() == MaterializeDataType.TEXT) { + sb.append(" != '')"); + } else if (expr.getExpressionType() == MaterializeDataType.BOOLEAN) { + sb.append(" != FALSE)"); + } else { + sb.append(" != 0)"); + } + } else if (Randomly.getBoolean()) { + if (cast.getCompoundType().getDataType() == MaterializeDataType.REAL + || cast.getCompoundType().getDataType() == MaterializeDataType.FLOAT) { + sb.append("CAST(CAST("); + visit(cast.getExpression()); + sb.append(" AS INT) AS "); + appendType(cast); + sb.append(")"); + } else { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + appendType(cast); + sb.append(")"); + } + } else { + if (cast.getCompoundType().getDataType() == MaterializeDataType.REAL + || cast.getCompoundType().getDataType() == MaterializeDataType.FLOAT) { + sb.append("("); + visit(cast.getExpression()); + sb.append(")::INT::"); + appendType(cast); + } else { + sb.append("("); + visit(cast.getExpression()); + sb.append(")::"); + appendType(cast); + } + } + } + + private void appendType(MaterializeCastOperation cast) { + MaterializeCompoundDataType compoundType = cast.getCompoundType(); + switch (compoundType.getDataType()) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: // TODO support also other int types + sb.append("INT"); + break; + case TEXT: + // TODO: append TEXT, CHAR + sb.append(Randomly.fromOptions("VARCHAR")); + break; + case REAL: + sb.append("FLOAT"); + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + sb.append("REAL"); + break; + case BIT: + sb.append("INT"); + break; + default: + throw new AssertionError(cast.getType()); + } + Optional size = compoundType.getSize(); + if (size.isPresent()) { + sb.append("("); + sb.append(size.get()); + sb.append(")"); + } + } + + @Override + public void visit(MaterializeBetweenOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(") BETWEEN "); + sb.append("("); + visit(op.getLeft()); + sb.append(") AND ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(MaterializeInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN ("); + visit(op.getListElements()); + sb.append(")"); + } + + @Override + public void visit(MaterializePostfixText op) { + visit(op.getExpr()); + sb.append(op.getText()); + } + + @Override + public void visit(MaterializeAggregate op) { + sb.append(op.getFunction()); + sb.append("("); + visit(op.getArgs()); + sb.append(")"); + } + + @Override + public void visit(MaterializeSimilarTo op) { + sb.append("("); + visit(op.getString()); + sb.append(" SIMILAR TO "); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + sb.append(")"); + } + + @Override + public void visit(MaterializePOSIXRegularExpression op) { + visit(op.getString()); + sb.append(op.getOp().getStringRepresentation()); + visit(op.getRegex()); + } + + @Override + public void visit(MaterializeBinaryLogicalOperation op) { + super.visit((BinaryOperation) op); + } + + @Override + public void visit(MaterializeLikeOperation op) { + super.visit((BinaryOperation) op); + } + +} diff --git a/src/sqlancer/materialize/MaterializeVisitor.java b/src/sqlancer/materialize/MaterializeVisitor.java new file mode 100644 index 000000000..40ed4e3f2 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeVisitor.java @@ -0,0 +1,127 @@ +package sqlancer.materialize; + +import java.util.List; + +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSimilarTo; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public interface MaterializeVisitor { + + void visit(MaterializeConstant constant); + + void visit(MaterializePostfixOperation op); + + void visit(MaterializeColumnValue c); + + void visit(MaterializePrefixOperation op); + + void visit(MaterializeSelect op); + + void visit(MaterializeOrderByTerm op); + + void visit(MaterializeFunction f); + + void visit(MaterializeCastOperation cast); + + void visit(MaterializeBetweenOperation op); + + void visit(MaterializeInOperation op); + + void visit(MaterializePostfixText op); + + void visit(MaterializeAggregate op); + + void visit(MaterializeSimilarTo op); + + void visit(MaterializePOSIXRegularExpression op); + + void visit(MaterializeFromTable from); + + void visit(MaterializeSubquery subquery); + + void visit(MaterializeBinaryLogicalOperation op); + + void visit(MaterializeLikeOperation op); + + default void visit(MaterializeExpression expression) { + if (expression instanceof MaterializeConstant) { + visit((MaterializeConstant) expression); + } else if (expression instanceof MaterializePostfixOperation) { + visit((MaterializePostfixOperation) expression); + } else if (expression instanceof MaterializeColumnValue) { + visit((MaterializeColumnValue) expression); + } else if (expression instanceof MaterializePrefixOperation) { + visit((MaterializePrefixOperation) expression); + } else if (expression instanceof MaterializeSelect) { + visit((MaterializeSelect) expression); + } else if (expression instanceof MaterializeOrderByTerm) { + visit((MaterializeOrderByTerm) expression); + } else if (expression instanceof MaterializeFunction) { + visit((MaterializeFunction) expression); + } else if (expression instanceof MaterializeCastOperation) { + visit((MaterializeCastOperation) expression); + } else if (expression instanceof MaterializeBetweenOperation) { + visit((MaterializeBetweenOperation) expression); + } else if (expression instanceof MaterializeInOperation) { + visit((MaterializeInOperation) expression); + } else if (expression instanceof MaterializeAggregate) { + visit((MaterializeAggregate) expression); + } else if (expression instanceof MaterializePostfixText) { + visit((MaterializePostfixText) expression); + } else if (expression instanceof MaterializeSimilarTo) { + visit((MaterializeSimilarTo) expression); + } else if (expression instanceof MaterializePOSIXRegularExpression) { + visit((MaterializePOSIXRegularExpression) expression); + } else if (expression instanceof MaterializeFromTable) { + visit((MaterializeFromTable) expression); + } else if (expression instanceof MaterializeSubquery) { + visit((MaterializeSubquery) expression); + } else if (expression instanceof MaterializeLikeOperation) { + visit((MaterializeLikeOperation) expression); + } else { + throw new AssertionError(expression); + } + } + + static String asString(MaterializeExpression expr) { + MaterializeToStringVisitor visitor = new MaterializeToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + static String asExpectedValues(MaterializeExpression expr) { + MaterializeExpectedValueVisitor v = new MaterializeExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + + static String getExpressionAsString(MaterializeGlobalState globalState, MaterializeDataType type, + List columns) { + MaterializeExpression expression = MaterializeExpressionGenerator.generateExpression(globalState, columns, + type); + MaterializeToStringVisitor visitor = new MaterializeToStringVisitor(); + visitor.visit(expression); + return visitor.get(); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeAggregate.java b/src/sqlancer/materialize/ast/MaterializeAggregate.java new file mode 100644 index 000000000..e9dc638f3 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeAggregate.java @@ -0,0 +1,58 @@ +package sqlancer.materialize.ast; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; + +/** + * @see Built-in Aggregate Functions + */ +public class MaterializeAggregate extends FunctionNode + implements MaterializeExpression { + + public enum MaterializeAggregateFunction { + AVG(MaterializeDataType.INT, MaterializeDataType.FLOAT, MaterializeDataType.REAL, MaterializeDataType.DECIMAL), + BIT_AND(MaterializeDataType.INT), BIT_OR(MaterializeDataType.INT), BOOL_AND(MaterializeDataType.BOOLEAN), + BOOL_OR(MaterializeDataType.BOOLEAN), COUNT(MaterializeDataType.INT), MAX, MIN, + SUM(MaterializeDataType.INT, MaterializeDataType.FLOAT, MaterializeDataType.REAL, MaterializeDataType.DECIMAL); + + private MaterializeDataType[] supportedReturnTypes; + + MaterializeAggregateFunction(MaterializeDataType... supportedReturnTypes) { + this.supportedReturnTypes = supportedReturnTypes.clone(); + } + + public List getTypes(MaterializeDataType returnType) { + return Arrays.asList(returnType); + } + + public boolean supportsReturnType(MaterializeDataType returnType) { + return Arrays.asList(supportedReturnTypes).stream().anyMatch(t -> t == returnType) + || supportedReturnTypes.length == 0; + } + + public static List getAggregates(MaterializeDataType type) { + return Arrays.asList(values()).stream().filter(p -> p.supportsReturnType(type)) + .collect(Collectors.toList()); + } + + public MaterializeDataType getRandomReturnType() { + if (supportedReturnTypes.length == 0) { + return Randomly.fromOptions(MaterializeDataType.getRandomType()); + } else { + return Randomly.fromOptions(supportedReturnTypes); + } + } + + } + + public MaterializeAggregate(List args, MaterializeAggregateFunction func) { + super(func, args); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeAlias.java b/src/sqlancer/materialize/ast/MaterializeAlias.java new file mode 100644 index 000000000..ec5445125 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeAlias.java @@ -0,0 +1,35 @@ +package sqlancer.materialize.ast; + +import sqlancer.common.visitor.UnaryOperation; + +public class MaterializeAlias implements UnaryOperation, MaterializeExpression { + + private final MaterializeExpression expr; + private final String alias; + + public MaterializeAlias(MaterializeExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + @Override + public MaterializeExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return " as " + alias; + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBetweenOperation.java b/src/sqlancer/materialize/ast/MaterializeBetweenOperation.java new file mode 100644 index 000000000..dcb7b7cdb --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBetweenOperation.java @@ -0,0 +1,66 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryComparisonOperation.MaterializeBinaryComparisonOperator; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation.BinaryLogicalOperator; + +public final class MaterializeBetweenOperation implements MaterializeExpression { + + private final MaterializeExpression expr; + private final MaterializeExpression left; + private final MaterializeExpression right; + private final boolean isSymmetric; + + public MaterializeBetweenOperation(MaterializeExpression expr, MaterializeExpression left, + MaterializeExpression right, boolean symmetric) { + this.expr = expr; + this.left = left; + this.right = right; + isSymmetric = symmetric; + } + + public MaterializeExpression getExpr() { + return expr; + } + + public MaterializeExpression getLeft() { + return left; + } + + public MaterializeExpression getRight() { + return right; + } + + public boolean isSymmetric() { + return isSymmetric; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeBinaryComparisonOperation leftComparison = new MaterializeBinaryComparisonOperation(left, expr, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryComparisonOperation rightComparison = new MaterializeBinaryComparisonOperation(expr, right, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryLogicalOperation andOperation = new MaterializeBinaryLogicalOperation(leftComparison, + rightComparison, MaterializeBinaryLogicalOperation.BinaryLogicalOperator.AND); + if (isSymmetric) { + MaterializeBinaryComparisonOperation leftComparison2 = new MaterializeBinaryComparisonOperation(right, expr, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryComparisonOperation rightComparison2 = new MaterializeBinaryComparisonOperation(expr, left, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryLogicalOperation andOperation2 = new MaterializeBinaryLogicalOperation(leftComparison2, + rightComparison2, MaterializeBinaryLogicalOperation.BinaryLogicalOperator.AND); + MaterializeBinaryLogicalOperation orOp = new MaterializeBinaryLogicalOperation(andOperation, andOperation2, + BinaryLogicalOperator.OR); + return orOp.getExpectedValue(); + } else { + return andOperation.getExpectedValue(); + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java new file mode 100644 index 000000000..0ba48924f --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java @@ -0,0 +1,103 @@ +package sqlancer.materialize.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryArithmeticOperation.MaterializeBinaryOperator; + +public class MaterializeBinaryArithmeticOperation + extends BinaryOperatorNode implements MaterializeExpression { + + public enum MaterializeBinaryOperator implements Operator { + + ADDITION("+") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> l + r); + } + + }, + SUBTRACTION("-") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + + } + + }, + MODULO("%") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + + } + }; + + private String textRepresentation; + + private static MaterializeConstant applyBitOperation(MaterializeConstant left, MaterializeConstant right, + BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + long leftVal = left.cast(MaterializeDataType.INT).asInt(); + long rightVal = right.cast(MaterializeDataType.INT).asInt(); + long value = op.apply(leftVal, rightVal); + return MaterializeConstant.createIntConstant(value); + } + } + + MaterializeBinaryOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public abstract MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right); + + public static MaterializeBinaryOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public MaterializeBinaryArithmeticOperation(MaterializeExpression left, MaterializeExpression right, + MaterializeBinaryOperator op) { + super(left, right, op); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpected = getLeft().getExpectedValue(); + MaterializeConstant rightExpected = getRight().getExpectedValue(); + if (leftExpected == null || rightExpected == null) { + return null; + } + return getOp().apply(leftExpected, rightExpected); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java new file mode 100644 index 000000000..8f143da76 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java @@ -0,0 +1,46 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryBitOperation.MaterializeBinaryBitOperator; + +public class MaterializeBinaryBitOperation extends + BinaryOperatorNode implements MaterializeExpression { + + public enum MaterializeBinaryBitOperator implements Operator { + BITWISE_AND("&"), // + BITWISE_OR("|"), // + BITWISE_XOR("#"), // + BITWISE_SHIFT_LEFT("<<"), // + BITWISE_SHIFT_RIGHT(">>"); + + private String text; + + MaterializeBinaryBitOperator(String text) { + this.text = text; + } + + public static MaterializeBinaryBitOperator getRandom() { + return Randomly.fromOptions(MaterializeBinaryBitOperator.values()); + } + + @Override + public String getTextRepresentation() { + return text; + } + + } + + public MaterializeBinaryBitOperation(MaterializeBinaryBitOperator op, MaterializeExpression left, + MaterializeExpression right) { + super(left, right, op); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java new file mode 100644 index 000000000..25121935a --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java @@ -0,0 +1,121 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryComparisonOperation.MaterializeBinaryComparisonOperator; + +public class MaterializeBinaryComparisonOperation + extends BinaryOperatorNode + implements MaterializeExpression { + + public enum MaterializeBinaryComparisonOperator implements Operator { + EQUALS("=") { + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + return leftVal.isEquals(rightVal); + } + }, + NOT_EQUALS("!=") { + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant isEquals = leftVal.isEquals(rightVal); + if (isEquals.isBoolean()) { + return MaterializeConstant.createBooleanConstant(!isEquals.asBoolean()); + } + return isEquals; + } + }, + LESS("<") { + + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + return leftVal.isLessThan(rightVal); + } + }, + LESS_EQUALS("<=") { + + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant lessThan = leftVal.isLessThan(rightVal); + if (lessThan.isBoolean() && !lessThan.asBoolean()) { + return leftVal.isEquals(rightVal); + } else { + return lessThan; + } + } + }, + GREATER(">") { + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return MaterializeConstant.createFalse(); + } else { + MaterializeConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return MaterializeConstant.createNullConstant(); + } + return MaterializePrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + }, + GREATER_EQUALS(">=") { + + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return MaterializeConstant.createTrue(); + } else { + MaterializeConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return MaterializeConstant.createNullConstant(); + } + return MaterializePrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + + }; + + private final String textRepresentation; + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + MaterializeBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public abstract MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal); + + public static MaterializeBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(MaterializeBinaryComparisonOperator.values()); + } + + } + + public MaterializeBinaryComparisonOperation(MaterializeExpression left, MaterializeExpression right, + MaterializeBinaryComparisonOperator op) { + super(left, right, op); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpectedValue = getLeft().getExpectedValue(); + MaterializeConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java new file mode 100644 index 000000000..8b8e457e2 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java @@ -0,0 +1,89 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation.BinaryLogicalOperator; + +public class MaterializeBinaryLogicalOperation extends BinaryOperatorNode + implements MaterializeExpression { + + public enum BinaryLogicalOperator implements Operator { + AND { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + MaterializeConstant leftBool = left.cast(MaterializeDataType.BOOLEAN); + MaterializeConstant rightBool = right.cast(MaterializeDataType.BOOLEAN); + if (leftBool.isNull()) { + if (rightBool.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + if (rightBool.asBoolean()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant.createFalse(); + } + } + } else if (!leftBool.asBoolean()) { + return MaterializeConstant.createFalse(); + } + assert leftBool.asBoolean(); + if (rightBool.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant.createBooleanConstant(rightBool.isBoolean() && rightBool.asBoolean()); + } + } + }, + OR { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + MaterializeConstant leftBool = left.cast(MaterializeDataType.BOOLEAN); + MaterializeConstant rightBool = right.cast(MaterializeDataType.BOOLEAN); + if (leftBool.isBoolean() && leftBool.asBoolean()) { + return MaterializeConstant.createTrue(); + } + if (rightBool.isBoolean() && rightBool.asBoolean()) { + return MaterializeConstant.createTrue(); + } + if (leftBool.isNull() || rightBool.isNull()) { + return MaterializeConstant.createNullConstant(); + } + return MaterializeConstant.createFalse(); + } + }; + + public abstract MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right); + + public static BinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + + public MaterializeBinaryLogicalOperation(MaterializeExpression left, MaterializeExpression right, + BinaryLogicalOperator op) { + super(left, right, op); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpectedValue = getLeft().getExpectedValue(); + MaterializeConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java new file mode 100644 index 000000000..cf1ef66ef --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java @@ -0,0 +1,74 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeBinaryRangeOperation extends BinaryNode + implements MaterializeExpression { + + private final String op; + + public enum MaterializeBinaryRangeOperator implements Operator { + UNION("+"), INTERSECTION("*"), DIFFERENCE("-"); + + private final String textRepresentation; + + MaterializeBinaryRangeOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public static MaterializeBinaryRangeOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum MaterializeBinaryRangeComparisonOperator { + CONTAINS_RANGE_OR_ELEMENT("@>"), RANGE_OR_ELEMENT_IS_CONTAINED("<@"), OVERLAP("&&"), STRICT_LEFT_OF("<<"), + STRICT_RIGHT_OF(">>"); + + private final String textRepresentation; + + MaterializeBinaryRangeComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public String getTextRepresentation() { + return textRepresentation; + } + + public static MaterializeBinaryRangeComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public MaterializeBinaryRangeOperation(MaterializeBinaryRangeComparisonOperator op, MaterializeExpression left, + MaterializeExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + public MaterializeBinaryRangeOperation(MaterializeBinaryRangeOperator op, MaterializeExpression left, + MaterializeExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public String getOperatorRepresentation() { + return op; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeCastOperation.java b/src/sqlancer/materialize/ast/MaterializeCastOperation.java new file mode 100644 index 000000000..ff7e20180 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeCastOperation.java @@ -0,0 +1,45 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeCastOperation implements MaterializeExpression { + + private final MaterializeExpression expression; + private final MaterializeCompoundDataType type; + + public MaterializeCastOperation(MaterializeExpression expression, MaterializeCompoundDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + @Override + public MaterializeDataType getExpressionType() { + return type.getDataType(); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant expectedValue = expression.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type.getDataType()); + } + + public MaterializeExpression getExpression() { + return expression; + } + + public MaterializeDataType getType() { + return type.getDataType(); + } + + public MaterializeCompoundDataType getCompoundType() { + return type; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeColumnValue.java b/src/sqlancer/materialize/ast/MaterializeColumnValue.java new file mode 100644 index 000000000..054f34ffd --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeColumnValue.java @@ -0,0 +1,34 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeColumnValue implements MaterializeExpression { + + private final MaterializeColumn c; + private final MaterializeConstant expectedValue; + + public MaterializeColumnValue(MaterializeColumn c, MaterializeConstant expectedValue) { + this.c = c; + this.expectedValue = expectedValue; + } + + @Override + public MaterializeDataType getExpressionType() { + return c.getType(); + } + + @Override + public MaterializeConstant getExpectedValue() { + return expectedValue; + } + + public static MaterializeColumnValue create(MaterializeColumn c, MaterializeConstant expected) { + return new MaterializeColumnValue(c, expected); + } + + public MaterializeColumn getColumn() { + return c; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeConcatOperation.java b/src/sqlancer/materialize/ast/MaterializeConcatOperation.java new file mode 100644 index 000000000..c1963c871 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeConcatOperation.java @@ -0,0 +1,37 @@ +package sqlancer.materialize.ast; + +import sqlancer.common.ast.BinaryNode; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeConcatOperation extends BinaryNode implements MaterializeExpression { + + public MaterializeConcatOperation(MaterializeExpression left, MaterializeExpression right) { + super(left, right); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.TEXT; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpectedValue = getLeft().getExpectedValue(); + MaterializeConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + if (leftExpectedValue.isNull() || rightExpectedValue.isNull()) { + return MaterializeConstant.createNullConstant(); + } + String leftStr = leftExpectedValue.cast(MaterializeDataType.TEXT).getUnquotedTextRepresentation(); + String rightStr = rightExpectedValue.cast(MaterializeDataType.TEXT).getUnquotedTextRepresentation(); + return MaterializeConstant.createTextConstant(leftStr + rightStr); + } + + @Override + public String getOperatorRepresentation() { + return "||"; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeConstant.java b/src/sqlancer/materialize/ast/MaterializeConstant.java new file mode 100644 index 000000000..dbf668806 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeConstant.java @@ -0,0 +1,516 @@ +package sqlancer.materialize.ast; + +import java.math.BigDecimal; + +import sqlancer.IgnoreMeException; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public abstract class MaterializeConstant implements MaterializeExpression { + + public abstract String getTextRepresentation(); + + public abstract String getUnquotedTextRepresentation(); + + public static class BooleanConstant extends MaterializeConstant { + + private final boolean value; + + public BooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return value ? "TRUE" : "FALSE"; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return MaterializeConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else if (rightVal.isString()) { + return MaterializeConstant + .createBooleanConstant(value == rightVal.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isString()) { + return isLessThan(rightVal.cast(MaterializeDataType.BOOLEAN)); + } else { + assert rightVal.isBoolean(); + return MaterializeConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + switch (type) { + case BOOLEAN: + return this; + case INT: + return MaterializeConstant.createIntConstant(value ? 1 : 0); + case TEXT: + return MaterializeConstant.createTextConstant(value ? "true" : "false"); + default: + return null; + } + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + } + + public static class MaterializeNullConstant extends MaterializeConstant { + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + return MaterializeConstant.createNullConstant(); + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + return MaterializeConstant.createNullConstant(); + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + return MaterializeConstant.createNullConstant(); + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + } + + public static class StringConstant extends MaterializeConstant { + + private final String value; + + public StringConstant(String value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'", value.replace("'", "''")); + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(MaterializeDataType.INT).isEquals(rightVal.cast(MaterializeDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(MaterializeDataType.BOOLEAN).isEquals(rightVal.cast(MaterializeDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(MaterializeDataType.INT).isLessThan(rightVal.cast(MaterializeDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(MaterializeDataType.BOOLEAN).isLessThan(rightVal.cast(MaterializeDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + if (type == MaterializeDataType.TEXT) { + return this; + } + String s = value.trim(); + switch (type) { + case BOOLEAN: + try { + return MaterializeConstant.createBooleanConstant(Long.parseLong(s) != 0); + } catch (NumberFormatException e) { + } + switch (s.toUpperCase()) { + case "T": + case "TR": + case "TRU": + case "TRUE": + case "1": + case "YES": + case "YE": + case "Y": + case "ON": + return MaterializeConstant.createTrue(); + case "F": + case "FA": + case "FAL": + case "FALS": + case "FALSE": + case "N": + case "NO": + case "OF": + case "OFF": + default: + return MaterializeConstant.createFalse(); + } + case INT: + try { + return MaterializeConstant.createIntConstant(Long.parseLong(s)); + } catch (NumberFormatException e) { + return MaterializeConstant.createIntConstant(-1); + } + case TEXT: + return this; + default: + return null; + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.TEXT; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public String asString() { + return value; + } + + @Override + public String getUnquotedTextRepresentation() { + return value; + } + + } + + public static class IntConstant extends MaterializeConstant { + + private final long val; + + public IntConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + + @Override + public long asInt() { + return val; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(MaterializeDataType.BOOLEAN).isEquals(rightVal); + } else if (rightVal.isInt()) { + return MaterializeConstant.createBooleanConstant(val == rightVal.asInt()); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(val == rightVal.cast(MaterializeDataType.INT).asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return MaterializeConstant.createBooleanConstant(val < rightVal.asInt()); + } else if (rightVal.isBoolean()) { + throw new AssertionError(rightVal); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(val < rightVal.cast(MaterializeDataType.INT).asInt()); + } else { + throw new IgnoreMeException(); + } + + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + switch (type) { + case BOOLEAN: + return MaterializeConstant.createBooleanConstant(val != 0); + case INT: + return this; + case TEXT: + return MaterializeConstant.createTextConstant(String.valueOf(val)); + default: + return null; + } + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + } + + public static MaterializeConstant createNullConstant() { + return new MaterializeNullConstant(); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isString() { + return false; + } + + public static MaterializeConstant createIntConstant(long val) { + return new IntConstant(val); + } + + public static MaterializeConstant createBooleanConstant(boolean val) { + return new BooleanConstant(val); + } + + @Override + public MaterializeConstant getExpectedValue() { + return this; + } + + public boolean isNull() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public static MaterializeConstant createFalse() { + return createBooleanConstant(false); + } + + public static MaterializeConstant createTrue() { + return createBooleanConstant(true); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isBoolean() { + return false; + } + + public abstract MaterializeConstant isEquals(MaterializeConstant rightVal); + + public boolean isInt() { + return false; + } + + protected abstract MaterializeConstant isLessThan(MaterializeConstant rightVal); + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract MaterializeConstant cast(MaterializeDataType type); + + public static MaterializeConstant createTextConstant(String string) { + return new StringConstant(string); + } + + public abstract static class MaterializeConstantBase extends MaterializeConstant { + + @Override + public String getUnquotedTextRepresentation() { + return null; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + return null; + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + return null; + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + return null; + } + } + + public static class DecimalConstant extends MaterializeConstantBase { + + private final BigDecimal val; + + public DecimalConstant(BigDecimal val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.DECIMAL; + } + + } + + public static class FloatConstant extends MaterializeConstantBase { + + private final float val; + + public FloatConstant(float val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.FLOAT; + } + + } + + public static class DoubleConstant extends MaterializeConstantBase { + + private final double val; + + public DoubleConstant(double val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.FLOAT; + } + + } + + public static class BitConstant extends MaterializeConstantBase { + + private final long val; + + public BitConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.format("%d", val); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BIT; + } + + } + + public static MaterializeConstant createDecimalConstant(BigDecimal bigDecimal) { + return new DecimalConstant(bigDecimal); + } + + public static MaterializeConstant createFloatConstant(float val) { + return new FloatConstant(val); + } + + public static MaterializeConstant createDoubleConstant(double val) { + return new DoubleConstant(val); + } + + public static MaterializeExpression createBitConstant(long integer) { + return new BitConstant(integer); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeExpression.java b/src/sqlancer/materialize/ast/MaterializeExpression.java new file mode 100644 index 000000000..8a42ad3f6 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeExpression.java @@ -0,0 +1,14 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public interface MaterializeExpression { + + default MaterializeDataType getExpressionType() { + return null; + } + + default MaterializeConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializeFunction.java b/src/sqlancer/materialize/ast/MaterializeFunction.java new file mode 100644 index 000000000..24fdce866 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeFunction.java @@ -0,0 +1,256 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeFunction implements MaterializeExpression { + + private final String func; + private final MaterializeExpression[] args; + private final MaterializeDataType returnType; + private MaterializeFunctionWithResult functionWithKnownResult; + + public MaterializeFunction(MaterializeFunctionWithResult func, MaterializeDataType returnType, + MaterializeExpression... args) { + functionWithKnownResult = func; + this.func = func.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public MaterializeFunction(MaterializeFunctionWithUnknownResult f, MaterializeDataType returnType, + MaterializeExpression... args) { + this.func = f.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public String getFunctionName() { + return func; + } + + public MaterializeExpression[] getArguments() { + return args.clone(); + } + + public enum MaterializeFunctionWithResult { + ABS(1, "abs") { + + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant + .createIntConstant(Math.abs(evaluatedArgs[0].cast(MaterializeDataType.INT).asInt())); + } + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { returnType }; + } + + }, + LOWER(1, "lower") { + + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return MaterializeConstant.createTextConstant(text.toLowerCase()); + } + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.TEXT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { MaterializeDataType.TEXT }; + } + + }, + LENGTH(1, "length") { + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } + String text = evaluatedArgs[0].asString(); + return MaterializeConstant.createIntConstant(text.length()); + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { MaterializeDataType.TEXT }; + } + }, + UPPER(1, "upper") { + + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return MaterializeConstant.createTextConstant(text.toUpperCase()); + } + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.TEXT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { MaterializeDataType.TEXT }; + } + + }, + NUM_NONNULLS(1, "num_nonnulls") { + @Override + public MaterializeConstant apply(MaterializeConstant[] args, MaterializeExpression... origArgs) { + int nr = 0; + for (MaterializeConstant c : args) { + if (!c.isNull()) { + nr++; + } + } + return MaterializeConstant.createIntConstant(nr); + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }, + NUM_NULLS(1, "num_nulls") { + @Override + public MaterializeConstant apply(MaterializeConstant[] args, MaterializeExpression... origArgs) { + int nr = 0; + for (MaterializeConstant c : args) { + if (c.isNull()) { + nr++; + } + } + return MaterializeConstant.createIntConstant(nr); + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }; + + private String functionName; + final int nrArgs; + private final boolean variadic; + + public MaterializeDataType[] getRandomTypes(int nr) { + MaterializeDataType[] types = new MaterializeDataType[nr]; + for (int i = 0; i < types.length; i++) { + types[i] = MaterializeDataType.getRandomType(); + } + return types; + } + + MaterializeFunctionWithResult(int nrArgs, String functionName) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = false; + } + + /** + * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number + * of arguments is returned. + * + * @return the number of arguments + */ + public int getNrArgs() { + return nrArgs; + } + + public abstract MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args); + + @Override + public String toString() { + return functionName; + } + + public boolean isVariadic() { + return variadic; + } + + public String getName() { + return functionName; + } + + public abstract boolean supportsReturnType(MaterializeDataType type); + + public abstract MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, + int nrArguments); + + public boolean checkArguments(MaterializeExpression... constants) { + return true; + } + + } + + @Override + public MaterializeConstant getExpectedValue() { + if (functionWithKnownResult == null) { + return null; + } + MaterializeConstant[] constants = new MaterializeConstant[args.length]; + for (int i = 0; i < constants.length; i++) { + constants[i] = args[i].getExpectedValue(); + if (constants[i] == null) { + return null; + } + } + return functionWithKnownResult.apply(constants, args); + } + + @Override + public MaterializeDataType getExpressionType() { + return returnType; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java b/src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java new file mode 100644 index 000000000..fb34c2247 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java @@ -0,0 +1,148 @@ +package sqlancer.materialize.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public enum MaterializeFunctionWithUnknownResult { + + CURRENT_DATABASE("current_database", MaterializeDataType.TEXT), // name + CURRENT_SCHEMA("current_schema", MaterializeDataType.TEXT), // name + PG_BACKEND_PID("pg_backend_pid", MaterializeDataType.INT), + PG_CURRENT_LOGFILE("pg_current_logfile", MaterializeDataType.TEXT), + PG_IS_OTHER_TEMP_SCHEMA("pg_is_other_temp_schema", MaterializeDataType.BOOLEAN), + PG_JIT_AVAILABLE("pg_jit_available", MaterializeDataType.BOOLEAN), + PG_NOTIFICATION_QUEUE_USAGE("pg_notification_queue_usage", MaterializeDataType.REAL), + PG_TRIGGER_DEPTH("pg_trigger_depth", MaterializeDataType.INT), VERSION("version", MaterializeDataType.TEXT), + + // + TO_CHAR("to_char", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT) { + @Override + public MaterializeExpression[] getArguments(MaterializeDataType returnType, MaterializeExpressionGenerator gen, + int depth) { + MaterializeExpression[] args = super.getArguments(returnType, gen, depth); + args[0] = gen.generateExpression(MaterializeDataType.getRandomType()); + return args; + } + }, + + // String functions + ASCII("ascii", MaterializeDataType.INT, MaterializeDataType.TEXT), + BTRIM("btrim", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + CHR("chr", MaterializeDataType.TEXT, MaterializeDataType.INT), + CONVERT_FROM("convert_from", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT) { + @Override + public MaterializeExpression[] getArguments(MaterializeDataType returnType, MaterializeExpressionGenerator gen, + int depth) { + MaterializeExpression[] args = super.getArguments(returnType, gen, depth); + args[1] = MaterializeConstant.createTextConstant(Randomly.fromOptions("UTF8", "LATIN1")); + return args; + } + }, + INITCAP("initcap", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + LEFT("left", MaterializeDataType.TEXT, MaterializeDataType.INT, MaterializeDataType.TEXT), + LOWER("lower", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + MD5("md5", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + UPPER("upper", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + QUOTE_LITERAL("quote_literal", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + QUOTE_IDENT("quote_ident", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + REGEX_REPLACE("regex_replace", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + REPLACE("replace", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + REVERSE("reverse", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + RIGHT("right", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.INT), + RPAD("rpad", MaterializeDataType.TEXT, MaterializeDataType.INT, MaterializeDataType.TEXT), + RTRIM("rtrim", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + SPLIT_PART("split_part", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.INT), + STRPOS("strpos", MaterializeDataType.INT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + SUBSTR("substr", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.INT, + MaterializeDataType.INT), + TO_ASCII("to_ascii", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + TO_HEX("to_hex", MaterializeDataType.INT, MaterializeDataType.TEXT), + TRANSLATE("translate", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT, + MaterializeDataType.TEXT), + // mathematical functions + ABS("abs", MaterializeDataType.REAL, MaterializeDataType.REAL), + CBRT("cbrt", MaterializeDataType.REAL, MaterializeDataType.REAL), CEILING("ceiling", MaterializeDataType.REAL), // + DEGREES("degrees", MaterializeDataType.REAL), EXP("exp", MaterializeDataType.REAL), + LN("ln", MaterializeDataType.REAL), LOG("log", MaterializeDataType.REAL), + LOG2("log", MaterializeDataType.REAL, MaterializeDataType.REAL), PI("pi", MaterializeDataType.REAL), + POWER("power", MaterializeDataType.REAL, MaterializeDataType.REAL), + TRUNC("trunc", MaterializeDataType.REAL, MaterializeDataType.INT), + TRUNC2("trunc", MaterializeDataType.REAL, MaterializeDataType.INT, MaterializeDataType.REAL), + FLOOR("floor", MaterializeDataType.REAL), + + // trigonometric functions - complete + ACOS("acos", MaterializeDataType.REAL), // + ACOSD("acosd", MaterializeDataType.REAL), // + ASIN("asin", MaterializeDataType.REAL), // + ASIND("asind", MaterializeDataType.REAL), // + ATAN("atan", MaterializeDataType.REAL), // + ATAND("atand", MaterializeDataType.REAL), // + ATAN2("atan2", MaterializeDataType.REAL, MaterializeDataType.REAL), // + ATAN2D("atan2d", MaterializeDataType.REAL, MaterializeDataType.REAL), // + COS("cos", MaterializeDataType.REAL), // + COSD("cosd", MaterializeDataType.REAL), // + COT("cot", MaterializeDataType.REAL), // + COTD("cotd", MaterializeDataType.REAL), // + SIN("sin", MaterializeDataType.REAL), // + SIND("sind", MaterializeDataType.REAL), // + TAN("tan", MaterializeDataType.REAL), // + TAND("tand", MaterializeDataType.REAL), // + + // hyperbolic functions - complete + SINH("sinh", MaterializeDataType.REAL), // + COSH("cosh", MaterializeDataType.REAL), // + TANH("tanh", MaterializeDataType.REAL), // + ASINH("asinh", MaterializeDataType.REAL), // + ACOSH("acosh", MaterializeDataType.REAL), // + ATANH("atanh", MaterializeDataType.REAL), // + + GET_BIT("get_bit", MaterializeDataType.INT, MaterializeDataType.TEXT, MaterializeDataType.INT), + GET_BYTE("get_byte", MaterializeDataType.INT, MaterializeDataType.TEXT, MaterializeDataType.INT), + + GET_COLUMN_SIZE("get_column_size", MaterializeDataType.INT, MaterializeDataType.TEXT); + + private String functionName; + private MaterializeDataType returnType; + private MaterializeDataType[] argTypes; + + MaterializeFunctionWithUnknownResult(String functionName, MaterializeDataType returnType, + MaterializeDataType... indexType) { + this.functionName = functionName; + this.returnType = returnType; + this.argTypes = indexType.clone(); + + } + + public boolean isCompatibleWithReturnType(MaterializeDataType t) { + return t == returnType; + } + + public MaterializeExpression[] getArguments(MaterializeDataType returnType, MaterializeExpressionGenerator gen, + int depth) { + MaterializeExpression[] args = new MaterializeExpression[argTypes.length]; + for (int i = 0; i < args.length; i++) { + args[i] = gen.generateExpression(depth, argTypes[i]); + } + return args; + + } + + public String getName() { + return functionName; + } + + public static List getSupportedFunctions(MaterializeDataType type) { + List functions = new ArrayList<>(); + for (MaterializeFunctionWithUnknownResult func : values()) { + if (func.isCompatibleWithReturnType(type)) { + functions.add(func); + } + } + return functions; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeInOperation.java b/src/sqlancer/materialize/ast/MaterializeInOperation.java new file mode 100644 index 000000000..270a973dc --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeInOperation.java @@ -0,0 +1,66 @@ +package sqlancer.materialize.ast; + +import java.util.List; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeInOperation implements MaterializeExpression { + + private final MaterializeExpression expr; + private final List listElements; + private final boolean isTrue; + + public MaterializeInOperation(MaterializeExpression expr, List listElements, + boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public MaterializeExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftValue = expr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return MaterializeConstant.createNullConstant(); + } + boolean isNull = false; + for (MaterializeExpression expr : getListElements()) { + MaterializeConstant rightExpectedValue = expr.getExpectedValue(); + if (rightExpectedValue == null) { + return null; + } + if (rightExpectedValue.isNull()) { + isNull = true; + } else if (rightExpectedValue.isEquals(this.expr.getExpectedValue()).isBoolean() + && rightExpectedValue.isEquals(this.expr.getExpectedValue()).asBoolean()) { + return MaterializeConstant.createBooleanConstant(isTrue); + } + } + + if (isNull) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant.createBooleanConstant(!isTrue); + } + } + + public boolean isTrue() { + return isTrue; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializeJoin.java b/src/sqlancer/materialize/ast/MaterializeJoin.java new file mode 100644 index 000000000..967cdf420 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeJoin.java @@ -0,0 +1,50 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeJoin implements MaterializeExpression { + + public enum MaterializeJoinType { + INNER, LEFT, RIGHT, FULL, CROSS; + + public static MaterializeJoinType getRandom() { + return Randomly.fromOptions(values()); + } + + } + + private final MaterializeExpression tableReference; + private final MaterializeExpression onClause; + private final MaterializeJoinType type; + + public MaterializeJoin(MaterializeExpression tableReference, MaterializeExpression onClause, + MaterializeJoinType type) { + this.tableReference = tableReference; + this.onClause = onClause; + this.type = type; + } + + public MaterializeExpression getTableReference() { + return tableReference; + } + + public MaterializeExpression getOnClause() { + return onClause; + } + + public MaterializeJoinType getType() { + return type; + } + + @Override + public MaterializeDataType getExpressionType() { + throw new AssertionError(); + } + + @Override + public MaterializeConstant getExpectedValue() { + throw new AssertionError(); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeLikeOperation.java b/src/sqlancer/materialize/ast/MaterializeLikeOperation.java new file mode 100644 index 000000000..b14209993 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeLikeOperation.java @@ -0,0 +1,38 @@ +package sqlancer.materialize.ast; + +import sqlancer.LikeImplementationHelper; +import sqlancer.common.ast.BinaryNode; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeLikeOperation extends BinaryNode implements MaterializeExpression { + + public MaterializeLikeOperation(MaterializeExpression left, MaterializeExpression right) { + super(left, right); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftVal = getLeft().getExpectedValue(); + MaterializeConstant rightVal = getRight().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } + if (leftVal.isNull() || rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + boolean val = LikeImplementationHelper.match(leftVal.asString(), rightVal.asString(), 0, 0, true); + return MaterializeConstant.createBooleanConstant(val); + } + } + + @Override + public String getOperatorRepresentation() { + return "LIKE"; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeOrderByTerm.java b/src/sqlancer/materialize/ast/MaterializeOrderByTerm.java new file mode 100644 index 000000000..d4704c87f --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeOrderByTerm.java @@ -0,0 +1,42 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeOrderByTerm implements MaterializeExpression { + + private final MaterializeOrder order; + private final MaterializeExpression expr; + + public enum MaterializeOrder { + ASC, DESC; + + public static MaterializeOrder getRandomOrder() { + return Randomly.fromOptions(MaterializeOrder.values()); + } + } + + public MaterializeOrderByTerm(MaterializeExpression expr, MaterializeOrder order) { + this.expr = expr; + this.order = order; + } + + public MaterializeOrder getOrder() { + return order; + } + + public MaterializeExpression getExpr() { + return expr; + } + + @Override + public MaterializeConstant getExpectedValue() { + throw new AssertionError(this); + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java b/src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java new file mode 100644 index 000000000..127bafe79 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java @@ -0,0 +1,65 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePOSIXRegularExpression implements MaterializeExpression { + + private MaterializeExpression string; + private MaterializeExpression regex; + private POSIXRegex op; + + public enum POSIXRegex implements Operator { + MATCH_CASE_SENSITIVE("~"), MATCH_CASE_INSENSITIVE("~*"), NOT_MATCH_CASE_SENSITIVE("!~"), + NOT_MATCH_CASE_INSENSITIVE("!~*"); + + private String repr; + + POSIXRegex(String repr) { + this.repr = repr; + } + + public String getStringRepresentation() { + return repr; + } + + public static POSIXRegex getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + + public MaterializePOSIXRegularExpression(MaterializeExpression string, MaterializeExpression regex, POSIXRegex op) { + this.string = string; + this.regex = regex; + this.op = op; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + return null; + } + + public MaterializeExpression getRegex() { + return regex; + } + + public MaterializeExpression getString() { + return string; + } + + public POSIXRegex getOp() { + return op; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializePostfixOperation.java b/src/sqlancer/materialize/ast/MaterializePostfixOperation.java new file mode 100644 index 000000000..7bee2e38c --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePostfixOperation.java @@ -0,0 +1,151 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePostfixOperation implements MaterializeExpression { + + private final MaterializeExpression expr; + private final PostfixOperator op; + private final String operatorTextRepresentation; + + public enum PostfixOperator implements Operator { + IS_NULL("IS NULL", "ISNULL") { + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return MaterializeDataType.values(); + } + + }, + IS_UNKNOWN("IS UNKNOWN") { + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + }, + + IS_NOT_NULL("IS NOT NULL") { + + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return MaterializeDataType.values(); + } + + }, + IS_NOT_UNKNOWN("IS NOT UNKNOWN") { + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + }, + IS_TRUE("IS TRUE") { + + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + return MaterializeConstant.createFalse(); + } else { + return MaterializeConstant + .createBooleanConstant(expectedValue.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + + }, + IS_FALSE("IS FALSE") { + + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + return MaterializeConstant.createFalse(); + } else { + return MaterializeConstant + .createBooleanConstant(!expectedValue.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + + }; + + private String[] textRepresentations; + + PostfixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public abstract MaterializeConstant apply(MaterializeConstant expectedValue); + + public abstract MaterializeDataType[] getInputDataTypes(); + + public static PostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + + public MaterializePostfixOperation(MaterializeExpression expr, PostfixOperator op) { + this.expr = expr; + this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); + this.op = op; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.apply(expectedValue); + } + + public String getOperatorTextRepresentation() { + return operatorTextRepresentation; + } + + public static MaterializeExpression create(MaterializeExpression expr, PostfixOperator op) { + return new MaterializePostfixOperation(expr, op); + } + + public MaterializeExpression getExpression() { + return expr; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializePostfixText.java b/src/sqlancer/materialize/ast/MaterializePostfixText.java new file mode 100644 index 000000000..a64b03c6d --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePostfixText.java @@ -0,0 +1,37 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePostfixText implements MaterializeExpression { + + private final MaterializeExpression expr; + private final String text; + private final MaterializeConstant expectedValue; + private final MaterializeDataType type; + + public MaterializePostfixText(MaterializeExpression expr, String text, MaterializeConstant expectedValue, + MaterializeDataType type) { + this.expr = expr; + this.text = text; + this.expectedValue = expectedValue; + this.type = type; + } + + public MaterializeExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + @Override + public MaterializeConstant getExpectedValue() { + return expectedValue; + } + + @Override + public MaterializeDataType getExpressionType() { + return type; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializePrefixOperation.java b/src/sqlancer/materialize/ast/MaterializePrefixOperation.java new file mode 100644 index 000000000..456c65b45 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePrefixOperation.java @@ -0,0 +1,119 @@ +package sqlancer.materialize.ast; + +import sqlancer.IgnoreMeException; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePrefixOperation implements MaterializeExpression { + + public enum PrefixOperator implements Operator { + NOT("NOT", MaterializeDataType.BOOLEAN) { + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + protected MaterializeConstant getExpectedValue(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant + .createBooleanConstant(!expectedValue.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } + } + }, + UNARY_PLUS("+", MaterializeDataType.INT) { + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + + @Override + protected MaterializeConstant getExpectedValue(MaterializeConstant expectedValue) { + // TODO: actual converts to double precision + return expectedValue; + } + + }, + UNARY_MINUS("-", MaterializeDataType.INT) { + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + + @Override + protected MaterializeConstant getExpectedValue(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + // TODO + throw new IgnoreMeException(); + } + if (expectedValue.isInt() && expectedValue.asInt() == Long.MIN_VALUE) { + throw new IgnoreMeException(); + } + try { + return MaterializeConstant.createIntConstant(-expectedValue.asInt()); + } catch (UnsupportedOperationException e) { + return null; + } + } + + }; + + private String textRepresentation; + private MaterializeDataType[] dataTypes; + + PrefixOperator(String textRepresentation, MaterializeDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract MaterializeDataType getExpressionType(); + + protected abstract MaterializeConstant getExpectedValue(MaterializeConstant expectedValue); + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + + private final MaterializeExpression expr; + private final PrefixOperator op; + + public MaterializePrefixOperation(MaterializeExpression expr, PrefixOperator op) { + this.expr = expr; + this.op = op; + } + + @Override + public MaterializeDataType getExpressionType() { + return op.getExpressionType(); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.getExpectedValue(expectedValue); + } + + public MaterializeDataType[] getInputDataTypes() { + return op.dataTypes; + } + + public String getTextRepresentation() { + return op.textRepresentation; + } + + public MaterializeExpression getExpression() { + return expr; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeSelect.java b/src/sqlancer/materialize/ast/MaterializeSelect.java new file mode 100644 index 000000000..cef761428 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeSelect.java @@ -0,0 +1,135 @@ +package sqlancer.materialize.ast; + +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; + +public class MaterializeSelect extends SelectBase implements MaterializeExpression { + + private SelectType selectOption = SelectType.ALL; + private List joinClauses = Collections.emptyList(); + private MaterializeExpression distinctOnClause; + private ForClause forClause; + + public enum ForClause { + UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); + + private final String textRepresentation; + + ForClause(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public String getTextRepresentation() { + return textRepresentation; + } + + public static ForClause getRandom() { + return Randomly.fromOptions(values()); + } + } + + public static class MaterializeFromTable implements MaterializeExpression { + private final MaterializeTable t; + private final boolean only; + + public MaterializeFromTable(MaterializeTable t, boolean only) { + this.t = t; + this.only = only; + } + + public MaterializeTable getTable() { + return t; + } + + public boolean isOnly() { + return only; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + } + + public static class MaterializeSubquery implements MaterializeExpression { + private final MaterializeSelect s; + private final String name; + + public MaterializeSubquery(MaterializeSelect s, String name) { + this.s = s; + this.name = name; + } + + public MaterializeSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + } + + public enum SelectType { + DISTINCT, ALL; + + public static SelectType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public void setSelectType(SelectType fromOptions) { + this.setSelectOption(fromOptions); + } + + public void setDistinctOnClause(MaterializeExpression distinctOnClause) { + if (selectOption != SelectType.DISTINCT) { + throw new IllegalArgumentException(); + } + this.distinctOnClause = distinctOnClause; + } + + public SelectType getSelectOption() { + return selectOption; + } + + public void setSelectOption(SelectType fromOptions) { + this.selectOption = fromOptions; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + + public void setJoinClauses(List joinStatements) { + this.joinClauses = joinStatements; + + } + + public List getJoinClauses() { + return joinClauses; + } + + public MaterializeExpression getDistinctOnClause() { + return distinctOnClause; + } + + public void setForClause(ForClause forClause) { + this.forClause = forClause; + } + + public ForClause getForClause() { + return forClause; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeSimilarTo.java b/src/sqlancer/materialize/ast/MaterializeSimilarTo.java new file mode 100644 index 000000000..dd675050d --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeSimilarTo.java @@ -0,0 +1,40 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeSimilarTo implements MaterializeExpression { + + private final MaterializeExpression string; + private final MaterializeExpression similarTo; + private final MaterializeExpression escapeCharacter; + + public MaterializeSimilarTo(MaterializeExpression string, MaterializeExpression similarTo, + MaterializeExpression escapeCharacter) { + this.string = string; + this.similarTo = similarTo; + this.escapeCharacter = escapeCharacter; + } + + public MaterializeExpression getString() { + return string; + } + + public MaterializeExpression getSimilarTo() { + return similarTo; + } + + public MaterializeExpression getEscapeCharacter() { + return escapeCharacter; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + return null; + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeCommon.java b/src/sqlancer/materialize/gen/MaterializeCommon.java new file mode 100644 index 000000000..0c4847bed --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeCommon.java @@ -0,0 +1,351 @@ +package sqlancer.materialize.gen; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeProvider; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public final class MaterializeCommon { + + private MaterializeCommon() { + } + + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); + errors.add("but it cannot be referenced from this part of the query"); + errors.add("missing FROM-clause entry for table"); + + errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in GROUP BY"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); + errors.add("results exceed max size of"); + + errors.add("does not exist"); + errors.add("aggregate functions are not allowed in"); + errors.add("is only defined for finite arguments"); + } + + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.add("is not commutative"); // exclude + errors.add("operator requires run-time type coercion"); // exclude + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.add("You might need to add explicit type casts"); + errors.add("invalid regular expression"); + errors.add("could not determine which collation to use"); + errors.add("invalid regular expression"); + errors.add("operator does not exist"); + errors.add("quantifier operand invalid"); + errors.add("collation mismatch"); + errors.add("collations are not supported"); + errors.add("operator is not unique"); + errors.add("is not a valid binary digit"); + errors.add("invalid hexadecimal digit"); + errors.add("invalid hexadecimal data: odd number of digits"); + errors.add("zero raised to a negative power is undefined"); + errors.add("cannot convert infinity to numeric"); + errors.add("division by zero"); + errors.add("invalid input syntax for type money"); + errors.add("invalid input syntax for type"); + errors.add("cannot cast type"); + errors.add("value overflows numeric format"); + errors.add("numeric field overflow"); + errors.add("LIKE pattern must not end with escape character"); + errors.add("is of type boolean but expression is of type text"); + errors.add("a negative number raised to a non-integer power yields a complex result"); + errors.add("could not determine polymorphic type because input has type unknown"); + errors.add("character number must be positive"); + errors.add("unterminated escape sequence"); + errors.add("cannot be matched"); + errors.add("clause must have type"); // "not" in having doesn't work + errors.add("argument must have type"); // "not" in having doesn't work + errors.add("CAST does not support casting from"); + errors.add("aggregate functions are not allowed in"); + errors.add("only defined for finite arguments"); + errors.add("unable to parse column reference in GROUP BY clause"); // TODO + addToCharFunctionErrors(errors); + addBitStringOperationErrors(errors); + addFunctionErrors(errors); + addCommonRangeExpressionErrors(errors); + addCommonRegexExpressionErrors(errors); + } + + private static void addToCharFunctionErrors(ExpectedErrors errors) { + errors.add("multiple decimal points"); + errors.add("and decimal point together"); + errors.add("multiple decimal points"); + errors.add("cannot use \"S\" twice"); + errors.add("must be ahead of \"PR\""); + errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); + errors.add("cannot use \"S\" and \"SG\" together"); + errors.add("cannot use \"S\" and \"MI\" together"); + errors.add("cannot use \"S\" and \"PL\" together"); + errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); + errors.add("is not a number"); + } + + private static void addBitStringOperationErrors(ExpectedErrors errors) { + errors.add("cannot XOR bit strings of different sizes"); + errors.add("cannot AND bit strings of different sizes"); + errors.add("cannot OR bit strings of different sizes"); + errors.add("must be type boolean, not type text"); + } + + private static void addFunctionErrors(ExpectedErrors errors) { + errors.add("out of valid range"); // get_bit/get_byte + errors.add("cannot take logarithm of a negative number"); + errors.add("cannot take logarithm of zero"); + errors.add("requested character too large for encoding"); // chr + errors.add("null character not permitted"); // chr + errors.add("requested character not valid for encoding"); // chr + errors.add("requested length too large"); // repeat + errors.add("invalid memory alloc request size"); // repeat + errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + } + + private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + errors.add("is not a valid hexadecimal digit"); + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.add("range lower bound must be less than or equal to range upper bound"); + errors.add("result of range difference would not be contiguous"); + errors.add("out of range"); + errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.add("value too long for type character"); + errors.add("not found in view targetlist"); + errors.add("CAST does not support casting from"); + } + + public static boolean appendDataType(MaterializeDataType type, StringBuilder sb, boolean allowSerial, + boolean generateOnlyKnown, List opClasses) throws AssertionError { + boolean serial = false; + switch (type) { + case BOOLEAN: + sb.append("boolean"); + break; + case INT: + sb.append(Randomly.fromOptions("smallint", "integer", "bigint")); + break; + case TEXT: + if (Randomly.getBoolean()) { + sb.append("TEXT"); + } else { + if (MaterializeProvider.generateOnlyKnown || Randomly.getBoolean()) { + sb.append("VAR"); + } + sb.append("CHAR"); + sb.append("("); + sb.append(ThreadLocalRandom.current().nextInt(1, 500)); + sb.append(")"); + } + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + sb.append("REAL"); + break; + case REAL: + sb.append("FLOAT"); + break; + case BIT: + sb.append("INT"); + break; + default: + throw new AssertionError(type); + } + return serial; + } + + public enum TableConstraints { + CHECK, PRIMARY_KEY, FOREIGN_KEY, EXCLUDE + } + + public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder sb, MaterializeTable table, + MaterializeGlobalState globalState, ExpectedErrors errors) { + // TODO constraint name + List tableConstraints = Randomly.nonEmptySubset(TableConstraints.values()); + if (excludePrimaryKey) { + tableConstraints.remove(TableConstraints.PRIMARY_KEY); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + tableConstraints.remove(TableConstraints.FOREIGN_KEY); + } + for (TableConstraints t : tableConstraints) { + sb.append(", "); + // TODO add index parameters + addTableConstraint(sb, table, globalState, t, errors); + } + } + + public static void addTableConstraint(StringBuilder sb, MaterializeTable table, MaterializeGlobalState globalState, + ExpectedErrors errors) { + addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); + } + + private static void addTableConstraint(StringBuilder sb, MaterializeTable table, MaterializeGlobalState globalState, + TableConstraints t, ExpectedErrors errors) { + List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); + List otherColumns; + MaterializeCommon.addCommonExpressionErrors(errors); + switch (t) { + case CHECK: + sb.append("CHECK("); + sb.append(MaterializeVisitor.getExpressionAsString(globalState, MaterializeDataType.BOOLEAN, + table.getColumns())); + sb.append(")"); + errors.add("constraint must be added to child tables too"); + errors.add("missing FROM-clause entry for table"); + break; + case PRIMARY_KEY: + sb.append("PRIMARY KEY("); + sb.append(randomNonEmptyColumnSubset.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + break; + case FOREIGN_KEY: + sb.append("FOREIGN KEY ("); + sb.append(randomNonEmptyColumnSubset.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(") REFERENCES "); + MaterializeTable randomOtherTable = globalState.getSchema().getRandomTable(tab -> !tab.isView()); + sb.append(randomOtherTable.getName()); + if (randomOtherTable.getColumns().size() < randomNonEmptyColumnSubset.size()) { + throw new IgnoreMeException(); + } + otherColumns = randomOtherTable.getRandomNonEmptyColumnSubset(randomNonEmptyColumnSubset.size()); + sb.append("("); + sb.append(otherColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("MATCH FULL", "MATCH SIMPLE")); + } + if (Randomly.getBoolean()) { + sb.append(" ON DELETE "); + errors.add("ERROR: invalid ON DELETE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" ON UPDATE "); + errors.add("invalid ON UPDATE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" "); + if (Randomly.getBoolean()) { + sb.append("DEFERRABLE"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("INITIALLY DEFERRED", "INITIALLY IMMEDIATE")); + } + } else { + sb.append("NOT DEFERRABLE"); + } + } + break; + case EXCLUDE: + sb.append("EXCLUDE "); + sb.append("("); + // TODO [USING index_method ] + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + appendExcludeElement(sb, globalState, table.getColumns()); + sb.append(" WITH "); + appendOperator(sb, globalState.getOperators()); + } + sb.append(")"); + errors.add("is not valid"); + errors.add("no operator matches"); + errors.add("operator does not exist"); + errors.add("unknown has no default operator class"); + errors.add("exclusion constraints are not supported on partitioned tables"); + errors.add("The exclusion operator must be related to the index operator class for the constraint"); + errors.add("could not create exclusion constraint"); + // TODO: index parameters + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append("("); + sb.append(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, + table.getColumns(), MaterializeDataType.BOOLEAN))); + sb.append(")"); + } + break; + default: + throw new AssertionError(t); + } + } + + private static void appendOperator(StringBuilder sb, List operators) { + sb.append(Randomly.fromList(operators)); + } + + // complete + private static void appendExcludeElement(StringBuilder sb, MaterializeGlobalState globalState, + List columns) { + if (Randomly.getBoolean()) { + // append column name + sb.append(Randomly.fromList(columns).getName()); + } else { + // append expression + sb.append("("); + sb.append(MaterializeVisitor + .asString(MaterializeExpressionGenerator.generateExpression(globalState, columns))); + sb.append(")"); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromList(globalState.getOpClasses())); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBoolean()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + + private static void deleteOrUpdateAction(StringBuilder sb) { + sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("unable to parse column reference in GROUP BY clause"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in"); + } + + public static String getFreeIndexName(MaterializeSchema s) { + List indexNames = s.getIndexNames(); + String candidateName; + do { + candidateName = DBMSCommon.createIndexName((int) Randomly.getNotCachedInteger(0, 100)); + } while (indexNames.contains(candidateName)); + return candidateName; + } +} diff --git a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java new file mode 100644 index 000000000..ded3e53f4 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java @@ -0,0 +1,38 @@ +package sqlancer.materialize.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public final class MaterializeDeleteGenerator { + + private MaterializeDeleteGenerator() { + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + MaterializeTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("violates foreign key constraint"); + errors.add("violates not-null constraint"); + errors.add("could not determine which collation to use for string comparison"); + StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append(" "); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, + table.getColumns(), MaterializeDataType.BOOLEAN))); + } + MaterializeCommon.addCommonExpressionErrors(errors); + errors.add("out of range"); + errors.add("does not support casting"); + errors.add("invalid input syntax for"); + errors.add("division by zero"); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java b/src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java new file mode 100644 index 000000000..4acbd329a --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.materialize.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeIndex; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; + +public final class MaterializeDropIndexGenerator { + + private MaterializeDropIndexGenerator() { + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + MaterializeTable randomTable = globalState.getSchema().getRandomTable(); + List indexes = randomTable.getIndexes(); + StringBuilder sb = new StringBuilder(); + sb.append("DROP INDEX "); + if (Randomly.getBoolean() || indexes.isEmpty()) { + sb.append("IF EXISTS "); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + if (indexes.isEmpty() || Randomly.getBoolean()) { + sb.append(DBMSCommon.createIndexName(Randomly.smallNumber())); + } else { + sb.append(Randomly.fromList(indexes).getIndexName()); + } + } + } else { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(Randomly.fromList(indexes).getIndexName()); + } + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot drop desired object(s) because other objects depend on them", + "cannot drop index", "does not exist"), + true); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java new file mode 100644 index 000000000..1396052a2 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -0,0 +1,517 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeProvider; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeRowValue; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryArithmeticOperation; +import sqlancer.materialize.ast.MaterializeBinaryArithmeticOperation.MaterializeBinaryOperator; +import sqlancer.materialize.ast.MaterializeBinaryBitOperation; +import sqlancer.materialize.ast.MaterializeBinaryBitOperation.MaterializeBinaryBitOperator; +import sqlancer.materialize.ast.MaterializeBinaryComparisonOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation.BinaryLogicalOperator; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConcatOperation; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeFunction.MaterializeFunctionWithResult; +import sqlancer.materialize.ast.MaterializeFunctionWithUnknownResult; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializeOrderByTerm.MaterializeOrder; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression.POSIXRegex; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializePrefixOperation.PrefixOperator; + +public class MaterializeExpressionGenerator implements ExpressionGenerator { + + private final int maxDepth; + + private final Randomly r; + + private List columns; + + private MaterializeRowValue rw; + + private boolean expectedResult; + + private MaterializeGlobalState globalState; + + private boolean allowAggregateFunctions; + + private final Map functionsAndTypes; + + private final List allowedFunctionTypes; + + public MaterializeExpressionGenerator(MaterializeGlobalState globalState) { + this.r = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + this.globalState = globalState; + this.functionsAndTypes = globalState.getFunctionsAndTypes(); + this.allowedFunctionTypes = globalState.getAllowedFunctionTypes(); + } + + public MaterializeExpressionGenerator setColumns(List columns) { + this.columns = columns; + return this; + } + + public MaterializeExpressionGenerator setRowValue(MaterializeRowValue rw) { + this.rw = rw; + return this; + } + + public MaterializeExpression generateExpression(int depth) { + return generateExpression(depth, MaterializeDataType.getRandomType()); + } + + public List generateOrderBy() { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new MaterializeOrderByTerm(MaterializeColumnValue.create(Randomly.fromList(columns), null), + MaterializeOrder.getRandomOrder())); + } + return orderBys; + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, LIKE, BETWEEN, IN_OPERATION, + POSIX_REGEX; + } + + private MaterializeExpression generateFunctionWithUnknownResult(int depth, MaterializeDataType type) { + List supportedFunctions = MaterializeFunctionWithUnknownResult + .getSupportedFunctions(type); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + supportedFunctions = supportedFunctions.stream() + .filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (supportedFunctions.isEmpty()) { + throw new IgnoreMeException(); + } + MaterializeFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); + return new MaterializeFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); + } + + private MaterializeExpression generateFunctionWithKnownResult(int depth, MaterializeDataType type) { + List functions = Stream + .of(MaterializeFunction.MaterializeFunctionWithResult.values()).filter(f -> f.supportsReturnType(type)) + .collect(Collectors.toList()); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (functions.isEmpty()) { + throw new IgnoreMeException(); + } + MaterializeFunctionWithResult randomFunction = Randomly.fromList(functions); + int nrArgs = randomFunction.getNrArgs(); + if (randomFunction.isVariadic()) { + nrArgs += Randomly.smallNumber(); + } + MaterializeDataType[] argTypes = randomFunction.getInputTypesForReturnType(type, nrArgs); + MaterializeExpression[] args = new MaterializeExpression[nrArgs]; + do { + for (int i = 0; i < args.length; i++) { + args[i] = generateExpression(depth + 1, argTypes[i]); + } + } while (!randomFunction.checkArguments(args)); + return new MaterializeFunction(randomFunction, type, args); + } + + private MaterializeExpression generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (MaterializeProvider.generateOnlyKnown) { + validOptions.remove(BooleanExpression.POSIX_REGEX); + } + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + PostfixOperator random = PostfixOperator.getRandom(); + return MaterializePostfixOperation + .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); + case IN_OPERATION: + return inOperation(depth + 1); + case NOT: + return new MaterializePrefixOperation(generateExpression(depth + 1, MaterializeDataType.BOOLEAN), + PrefixOperator.NOT); + case BINARY_LOGICAL_OPERATOR: + MaterializeExpression first = generateExpression(depth + 1, MaterializeDataType.BOOLEAN); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + first = new MaterializeBinaryLogicalOperation(first, + generateExpression(depth + 1, MaterializeDataType.BOOLEAN), BinaryLogicalOperator.getRandom()); + } + return first; + case BINARY_COMPARISON: + MaterializeDataType dataType = getMeaningfulType(); + return generateComparison(depth, dataType); + case FUNCTION: + return generateFunction(depth + 1, MaterializeDataType.BOOLEAN); + case LIKE: + return new MaterializeLikeOperation(generateExpression(depth + 1, MaterializeDataType.TEXT), + generateExpression(depth + 1, MaterializeDataType.TEXT)); + case BETWEEN: + MaterializeDataType type = getMeaningfulType(); + return new MaterializeBetweenOperation(generateExpression(depth + 1, type), + generateExpression(depth + 1, type), generateExpression(depth + 1, type), Randomly.getBoolean()); + case POSIX_REGEX: + assert !expectedResult; + return new MaterializePOSIXRegularExpression(generateExpression(depth + 1, MaterializeDataType.TEXT), + generateExpression(depth + 1, MaterializeDataType.TEXT), POSIXRegex.getRandom()); + default: + throw new AssertionError(); + } + } + + private MaterializeDataType getMeaningfulType() { + // make it more likely that the expression does not only consist of constant + // expressions + if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { + return MaterializeDataType.getRandomType(); + } else { + return Randomly.fromList(columns).getType(); + } + } + + private MaterializeExpression generateFunction(int depth, MaterializeDataType type) { + if (MaterializeProvider.generateOnlyKnown || Randomly.getBoolean()) { + return generateFunctionWithKnownResult(depth, type); + } else { + return generateFunctionWithUnknownResult(depth, type); + } + } + + private MaterializeExpression generateComparison(int depth, MaterializeDataType dataType) { + MaterializeExpression leftExpr = generateExpression(depth + 1, dataType); + MaterializeExpression rightExpr = generateExpression(depth + 1, dataType); + return getComparison(leftExpr, rightExpr); + } + + private MaterializeExpression getComparison(MaterializeExpression leftExpr, MaterializeExpression rightExpr) { + return new MaterializeBinaryComparisonOperation(leftExpr, rightExpr, + MaterializeBinaryComparisonOperation.MaterializeBinaryComparisonOperator.getRandom()); + } + + private MaterializeExpression inOperation(int depth) { + MaterializeDataType type = MaterializeDataType.getRandomType(); + MaterializeExpression leftExpr = generateExpression(depth + 1, type); + List rightExpr = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + rightExpr.add(generateExpression(depth + 1, type)); + } + return new MaterializeInOperation(leftExpr, rightExpr, Randomly.getBoolean()); + } + + public static MaterializeExpression generateExpression(MaterializeGlobalState globalState, + MaterializeDataType type) { + return new MaterializeExpressionGenerator(globalState).generateExpression(0, type); + } + + public MaterializeExpression generateExpression(int depth, MaterializeDataType originalType) { + MaterializeDataType dataType = originalType; + if (dataType == MaterializeDataType.REAL && Randomly.getBoolean()) { + dataType = Randomly.fromOptions(MaterializeDataType.INT, MaterializeDataType.FLOAT); + } + if (dataType == MaterializeDataType.FLOAT && Randomly.getBoolean()) { + dataType = MaterializeDataType.INT; + } + if (!filterColumns(dataType).isEmpty() && Randomly.getBoolean()) { + return createColumnOfType(dataType); + } + return generateExpressionInternal(depth, dataType); + } + + private MaterializeExpression generateExpressionInternal(int depth, MaterializeDataType dataType) + throws AssertionError { + if (allowAggregateFunctions && Randomly.getBoolean()) { + allowAggregateFunctions = false; // aggregate function calls cannot be nested + return getAggregate(dataType); + } + if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { + // generic expression + if (Randomly.getBoolean() || depth > maxDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(r, dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(r, dataType); + } else { + return createColumnOfType(dataType); + } + } + } else { + if (Randomly.getBoolean()) { + return new MaterializeCastOperation(generateExpression(depth + 1), getCompoundDataType(dataType)); + } else { + return generateFunctionWithUnknownResult(depth, dataType); + } + } + } else { + switch (dataType) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case TEXT: + return generateTextExpression(depth); + case DECIMAL: + case REAL: + case FLOAT: + return generateConstant(r, dataType); + case BIT: + return generateBitExpression(depth); + default: + throw new AssertionError(dataType); + } + } + } + + private static MaterializeCompoundDataType getCompoundDataType(MaterializeDataType type) { + switch (type) { + case BOOLEAN: + case DECIMAL: // TODO + case FLOAT: + case INT: + case REAL: + case BIT: + return MaterializeCompoundDataType.create(type); + case TEXT: // TODO + if (Randomly.getBoolean() || MaterializeProvider.generateOnlyKnown /* + * The PQS implementation does not check + * for size specifications + */) { + return MaterializeCompoundDataType.create(type); + } else { + return MaterializeCompoundDataType.create(type, (int) Randomly.getNotCachedInteger(1, 1000)); + } + default: + throw new AssertionError(type); + } + + } + + private enum TextExpression { + CAST, FUNCTION, CONCAT + } + + private MaterializeExpression generateTextExpression(int depth) { + TextExpression option; + List validOptions = new ArrayList<>(Arrays.asList(TextExpression.values())); + option = Randomly.fromList(validOptions); + + switch (option) { + case CAST: + return new MaterializeCastOperation(generateExpression(depth + 1), + getCompoundDataType(MaterializeDataType.TEXT)); + case FUNCTION: + return generateFunction(depth + 1, MaterializeDataType.TEXT); + case CONCAT: + return generateConcat(depth); + default: + throw new AssertionError(); + } + } + + private MaterializeExpression generateConcat(int depth) { + MaterializeExpression left = generateExpression(depth + 1, MaterializeDataType.TEXT); + MaterializeExpression right = generateExpression(depth + 1); + return new MaterializeConcatOperation(left, right); + } + + private enum BitExpression { + BINARY_OPERATION + }; + + private MaterializeExpression generateBitExpression(int depth) { + BitExpression option; + option = Randomly.fromOptions(BitExpression.values()); + switch (option) { + case BINARY_OPERATION: + return new MaterializeBinaryBitOperation(MaterializeBinaryBitOperator.getRandom(), + generateExpression(depth + 1, MaterializeDataType.BIT), + generateExpression(depth + 1, MaterializeDataType.BIT)); + default: + throw new AssertionError(); + } + } + + private enum IntExpression { + UNARY_OPERATION, FUNCTION, /* CAST, */BINARY_ARITHMETIC_EXPRESSION + } + + private MaterializeExpression generateIntExpression(int depth) { + IntExpression option; + option = Randomly.fromOptions(IntExpression.values()); + switch (option) { + case UNARY_OPERATION: + MaterializeExpression intExpression = generateExpression(depth + 1, MaterializeDataType.INT); + return new MaterializePrefixOperation(intExpression, + Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, MaterializeDataType.INT); + case BINARY_ARITHMETIC_EXPRESSION: + return new MaterializeBinaryArithmeticOperation(generateExpression(depth + 1, MaterializeDataType.INT), + generateExpression(depth + 1, MaterializeDataType.INT), MaterializeBinaryOperator.getRandom()); + default: + throw new AssertionError(); + } + } + + private MaterializeExpression createColumnOfType(MaterializeDataType type) { + List columns = filterColumns(type); + MaterializeColumn fromList = Randomly.fromList(columns); + MaterializeConstant value = rw == null ? null : rw.getValues().get(fromList); + return MaterializeColumnValue.create(fromList, value); + } + + final List filterColumns(MaterializeDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + public MaterializeExpression generateExpressionWithExpectedResult(MaterializeDataType type) { + this.expectedResult = true; + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns) + .setRowValue(rw); + MaterializeExpression expr; + do { + expr = gen.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + public static MaterializeExpression generateConstant(Randomly r, MaterializeDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return MaterializeConstant.createNullConstant(); + } + switch (type) { + case INT: + if (Randomly.getBooleanWithSmallProbability()) { + return MaterializeConstant.createTextConstant(String.valueOf(r.getInteger())); + } else { + return MaterializeConstant.createIntConstant(r.getInteger()); + } + case BOOLEAN: + if (Randomly.getBooleanWithSmallProbability() && !MaterializeProvider.generateOnlyKnown) { + return MaterializeConstant + .createTextConstant(Randomly.fromOptions("TR", "TRUE", "FA", "FALSE", "0", "1", "ON", "off")); + } else { + return MaterializeConstant.createBooleanConstant(Randomly.getBoolean()); + } + case TEXT: + return MaterializeConstant.createTextConstant(r.getString()); + case DECIMAL: + return MaterializeConstant.createDecimalConstant(r.getRandomBigDecimal()); + case FLOAT: + return MaterializeConstant.createFloatConstant((float) r.getDouble()); + case REAL: + return MaterializeConstant.createDoubleConstant(r.getDouble()); + case BIT: + return MaterializeConstant.createBitConstant(r.getInteger()); + default: + throw new AssertionError(type); + } + } + + public static MaterializeExpression generateExpression(MaterializeGlobalState globalState, + List columns, MaterializeDataType type) { + return new MaterializeExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); + } + + public static MaterializeExpression generateExpression(MaterializeGlobalState globalState, + List columns) { + return new MaterializeExpressionGenerator(globalState).setColumns(columns).generateExpression(0); + + } + + public List generateExpressions(int nr) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(0)); + } + return expressions; + } + + public MaterializeExpression generateExpression(MaterializeDataType dataType) { + return generateExpression(0, dataType); + } + + public MaterializeExpressionGenerator setGlobalState(MaterializeGlobalState globalState) { + this.globalState = globalState; + return this; + } + + public MaterializeExpression generateHavingClause() { + this.allowAggregateFunctions = true; + MaterializeExpression expression = generateExpression(MaterializeDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + + public MaterializeExpression generateAggregate() { + return getAggregate(MaterializeDataType.getRandomType()); + } + + private MaterializeExpression getAggregate(MaterializeDataType dataType) { + List aggregates = MaterializeAggregateFunction.getAggregates(dataType); + MaterializeAggregateFunction agg = Randomly.fromList(aggregates); + return generateArgsForAggregate(dataType, agg); + } + + public MaterializeAggregate generateArgsForAggregate(MaterializeDataType dataType, + MaterializeAggregateFunction agg) { + List types = agg.getTypes(dataType); + List args = new ArrayList<>(); + for (MaterializeDataType argType : types) { + args.add(generateExpression(argType)); + } + return new MaterializeAggregate(args, agg); + } + + public MaterializeExpressionGenerator allowAggregates(boolean value) { + allowAggregateFunctions = value; + return this; + } + + @Override + public MaterializeExpression generatePredicate() { + return generateExpression(MaterializeDataType.BOOLEAN); + } + + @Override + public MaterializeExpression negatePredicate(MaterializeExpression predicate) { + return new MaterializePrefixOperation(predicate, MaterializePrefixOperation.PrefixOperator.NOT); + } + + @Override + public MaterializeExpression isNull(MaterializeExpression expr) { + return new MaterializePostfixOperation(expr, PostfixOperator.IS_NULL); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java new file mode 100644 index 000000000..9d7a91b50 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java @@ -0,0 +1,80 @@ +package sqlancer.materialize.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; + +public final class MaterializeIndexGenerator { + + private MaterializeIndexGenerator() { + } + + public enum IndexType { + BTREE, HASH, GIST, GIN + } + + public static SQLQueryAdapter generate(MaterializeGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE"); + sb.append(" INDEX "); + MaterializeTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized + // views + sb.append(MaterializeCommon.getFreeIndexName(globalState.getSchema())); + sb.append(" ON "); + sb.append(randomTable.getName()); + IndexType method; + method = IndexType.BTREE; + + sb.append("("); + if (method == IndexType.HASH) { + sb.append(randomTable.getRandomColumn().getName()); + } else { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(randomTable.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + } + + sb.append(")"); + errors.add("already contains data"); // CONCURRENT INDEX failed + errors.add("You might need to add explicit type casts"); + errors.add(" collations are not supported"); + errors.add("because it has pending trigger events"); + errors.add("could not determine which collation to use for index expression"); + errors.add("could not determine which collation to use for string comparison"); + errors.add("is duplicated"); + errors.add("access method \"gin\" does not support unique indexes"); + errors.add("access method \"hash\" does not support unique indexes"); + errors.add("already exists"); + errors.add("could not create unique index"); + errors.add("has no default operator class"); + errors.add("does not support"); + errors.add("does not support casting"); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("insufficient columns in UNIQUE constraint definition"); + errors.add("invalid input syntax for"); + errors.add("must be type "); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("out of range"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("functions in index expression must be marked IMMUTABLE"); + errors.add("result of range difference would not be contiguous"); + errors.add("which is part of the partition key"); + MaterializeCommon.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java new file mode 100644 index 000000000..01015514b --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java @@ -0,0 +1,106 @@ +package sqlancer.materialize.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeExpression; + +public final class MaterializeInsertGenerator { + + private MaterializeInsertGenerator() { + } + + public static SQLQueryAdapter insert(MaterializeGlobalState globalState) { + MaterializeTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("cannot insert into column"); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonInsertUpdateErrors(errors); + MaterializeCommon.addCommonExpressionErrors(errors); + errors.add("multiple assignments to same column"); + errors.add("violates foreign key constraint"); + errors.add("value too long for type character varying"); + errors.add("conflicting key value violates exclusion constraint"); + errors.add("violates not-null constraint"); + errors.add("current transaction is aborted"); + errors.add("bit string too long"); + errors.add("new row violates check option for view"); + errors.add("reached maximum value of sequence"); + errors.add("but expression is of type"); + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES"); + + if (globalState.getDbmsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { + StringBuilder sbRowValue = new StringBuilder(); + sbRowValue.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sbRowValue.append(", "); + } + sbRowValue.append(MaterializeVisitor.asString(MaterializeExpressionGenerator + .generateConstant(globalState.getRandomly(), columns.get(i).getType()))); + } + sbRowValue.append(")"); + + int n = (int) Randomly.getNotCachedInteger(100, 1000); + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(sbRowValue); + } + } else { + int n = Randomly.smallNumber() + 1; + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + insertRow(globalState, sb, columns); + } + } + errors.add("duplicate key value violates unique constraint"); + errors.add("identity column defined as GENERATED ALWAYS"); + errors.add("out of range"); + errors.add("violates check constraint"); + errors.add("no partition of relation"); + errors.add("invalid input syntax"); + errors.add("division by zero"); + errors.add("violates foreign key constraint"); + errors.add("data type unknown"); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private static void insertRow(MaterializeGlobalState globalState, StringBuilder sb, + List columns) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + MaterializeExpression generateConstant; + if (Randomly.getBoolean()) { + generateConstant = MaterializeExpressionGenerator.generateConstant(globalState.getRandomly(), + columns.get(i).getType()); + } else { + generateConstant = new MaterializeExpressionGenerator(globalState) + .generateExpression(columns.get(i).getType()); + } + sb.append(MaterializeVisitor.asString(generateConstant)); + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java new file mode 100644 index 000000000..16f4f1be8 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java @@ -0,0 +1,64 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.ForClause; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.SelectType; + +public final class MaterializeRandomQueryGenerator { + + private MaterializeRandomQueryGenerator() { + } + + public static MaterializeSelect createRandomQuery(int nrColumns, MaterializeGlobalState globalState) { + List columns = new ArrayList<>(); + MaterializeTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState) + .setColumns(tables.getColumns()); + for (int i = 0; i < nrColumns; i++) { + columns.add(gen.generateExpression(0)); + } + MaterializeSelect select = new MaterializeSelect(); + select.setSelectType(SelectType.getRandom()); + if (select.getSelectOption() == SelectType.DISTINCT && Randomly.getBoolean()) { + select.setDistinctOnClause(gen.generateExpression(0)); + } + select.setFromList(tables.getTables().stream().map(t -> new MaterializeFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return select; + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeTableGenerator.java b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java new file mode 100644 index 000000000..f132f7370 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java @@ -0,0 +1,127 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public class MaterializeTableGenerator { + + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + @SuppressWarnings("unused") + private boolean isTemporaryTable; + @SuppressWarnings("unused") + private final MaterializeSchema newSchema; + private final List columnsToBeAdded = new ArrayList<>(); + protected final ExpectedErrors errors = new ExpectedErrors(); + private final MaterializeTable table; + private final boolean generateOnlyKnown; + private final MaterializeGlobalState globalState; + + public MaterializeTableGenerator(String tableName, MaterializeSchema newSchema, boolean generateOnlyKnown, + MaterializeGlobalState globalState) { + this.tableName = tableName; + this.newSchema = newSchema; + this.generateOnlyKnown = generateOnlyKnown; + this.globalState = globalState; + table = new MaterializeTable(tableName, columnsToBeAdded, null, null, null, false, false); + errors.add("invalid input syntax for"); + errors.add("is not unique"); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("cannot create partitioned table as inheritance child"); + errors.add("does not support casting"); + errors.add("ERROR: functions in index expression must be marked IMMUTABLE"); + errors.add("functions in partition key expression must be marked IMMUTABLE"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("has no default operator class for access method"); + errors.add("does not exist for access method"); + errors.add("does not accept data type"); + errors.add("but default expression is of type text"); + errors.add("has pseudo-type unknown"); + errors.add("no collation was derived for partition key column"); + errors.add("inherits from generated column but specifies identity"); + errors.add("inherits from generated column but specifies default"); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonTableErrors(errors); + } + + public static SQLQueryAdapter generate(String tableName, MaterializeSchema newSchema, boolean generateOnlyKnown, + MaterializeGlobalState globalState) { + return new MaterializeTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); + } + + protected SQLQueryAdapter generate() { + sb.append("CREATE"); + sb.append(" TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + createStandard(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void createStandard() throws AssertionError { + sb.append("("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + String name = DBMSCommon.createColumnName(i); + createColumn(name); + } + sb.append(")"); + } + + private void createColumn(String name) throws AssertionError { + sb.append(name); + sb.append(" "); + MaterializeDataType type = MaterializeDataType.getRandomType(); + MaterializeCommon.appendDataType(type, sb, true, generateOnlyKnown, globalState.getCollates()); + MaterializeColumn c = new MaterializeColumn(name, type); + c.setTable(table); + columnsToBeAdded.add(c); + sb.append(" "); + if (Randomly.getBoolean()) { + createColumnConstraint(type); + } + } + + private enum ColumnConstraint { + DEFAULT + }; + + private void createColumnConstraint(MaterializeDataType type) { + List constraintSubset = Randomly.nonEmptySubset(ColumnConstraint.values()); + for (ColumnConstraint c : constraintSubset) { + sb.append(" "); + switch (c) { + case DEFAULT: + sb.append("DEFAULT"); + sb.append(" ("); + sb.append(MaterializeVisitor + .asString(MaterializeExpressionGenerator.generateExpression(globalState, type))); + sb.append(")"); + // CREATE TEMPORARY TABLE t1(c0 smallint DEFAULT ('566963878')); + errors.add("out of range"); + errors.add("is a generated column"); + break; + default: + throw new AssertionError(sb); + } + } + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java new file mode 100644 index 000000000..1b7e69208 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java @@ -0,0 +1,78 @@ +package sqlancer.materialize.gen; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeExpression; + +public final class MaterializeUpdateGenerator extends AbstractUpdateGenerator { + + private final MaterializeGlobalState globalState; + private MaterializeTable randomTable; + + private MaterializeUpdateGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; + errors.addAll(Arrays.asList("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "does not support casting", "must be type boolean", + "is not unique", " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable")); + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + return new MaterializeUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + sb.append("UPDATE "); + sb.append(randomTable.getName()); + sb.append(" SET "); + errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced + // table multiple times + errors.add("new row violates check option for view"); + MaterializeCommon.addCommonInsertUpdateErrors(errors); + updateColumns(columns); + errors.add("invalid input syntax for "); + errors.add("operator does not exist: text = boolean"); + errors.add("violates check constraint"); + errors.add("could not determine which collation to use for string comparison"); + errors.add("but expression is of type"); + MaterializeCommon.addCommonExpressionErrors(errors); + if (!Randomly.getBooleanWithSmallProbability()) { + sb.append(" WHERE "); + MaterializeExpression where = MaterializeExpressionGenerator.generateExpression(globalState, + randomTable.getColumns(), MaterializeDataType.BOOLEAN); + sb.append(MaterializeVisitor.asString(where)); + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + @Override + protected void updateValue(MaterializeColumn column) { + if (!Randomly.getBoolean()) { + MaterializeExpression constant = MaterializeExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(MaterializeVisitor.asString(constant)); + } else { + sb.append("("); + MaterializeExpression expr = MaterializeExpressionGenerator.generateExpression(globalState, + randomTable.getColumns(), column.getType()); + // caused by casts + sb.append(MaterializeVisitor.asString(expr)); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeViewGenerator.java b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java new file mode 100644 index 000000000..c8cd0e93f --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java @@ -0,0 +1,73 @@ +package sqlancer.materialize.gen; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeSelect; + +public final class MaterializeViewGenerator { + + private MaterializeViewGenerator() { + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE"); + @SuppressWarnings("unused") + boolean materialized; + @SuppressWarnings("unused") + boolean recursive = false; + if (Randomly.getBoolean()) { + sb.append(" MATERIALIZED"); + materialized = true; + } else { + if (Randomly.getBoolean()) { + sb.append(" OR REPLACE"); + } + materialized = false; + } + sb.append(" VIEW "); + int i = 0; + String[] name = new String[1]; + while (true) { + name[0] = "v" + i++; + if (globalState.getSchema().getDatabaseTables().stream() + .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { + break; + } + } + sb.append(name[0]); + sb.append("("); + int nrColumns = Randomly.smallNumber() + 1; + for (i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(DBMSCommon.createColumnName(i)); + } + sb.append(")"); + sb.append(" AS ("); + MaterializeSelect select = MaterializeRandomQueryGenerator.createRandomQuery(nrColumns, globalState); + sb.append(MaterializeVisitor.asString(select)); + sb.append(")"); + MaterializeCommon.addGroupingErrors(errors); + errors.add("already exists"); + errors.add("cannot drop columns from view"); + errors.add("non-integer constant in ORDER BY"); // TODO + errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO + errors.add("cannot change data type of view column"); + errors.add("specified more than once"); // TODO + errors.add("materialized views must not use temporary tables or views"); + errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); + errors.add("is not a view"); + errors.add("non-integer constant in DISTINCT ON"); + errors.add("unable to parse column reference in DISTINCT ON clause"); + errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + MaterializeCommon.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java new file mode 100644 index 000000000..fbfbcb132 --- /dev/null +++ b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java @@ -0,0 +1,176 @@ +package sqlancer.materialize.oracle; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSelect.SelectType; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; +import sqlancer.materialize.oracle.tlp.MaterializeTLPBase; + +public class MaterializeNoRECOracle extends NoRECBase + implements TestOracle { + + private final MaterializeSchema s; + + public MaterializeNoRECOracle(MaterializeGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonFetchErrors(errors); + } + + @Override + public void check() throws SQLException { + MaterializeTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + MaterializeExpression randomWhereCondition = getRandomWhereCondition(columns); + List tables = randomTables.getTables(); + + List joinStatements = getJoinStatements(state, columns, tables); + List fromTables = tables.stream() + .map(t -> new MaterializeFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); + int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, + firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + public static List getJoinStatements(MaterializeGlobalState globalState, + List columns, List tables) { + List joinStatements = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeTable table = Randomly.fromList(tables); + tables.remove(table); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, + options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + private MaterializeExpression getRandomWhereCondition(List columns) { + return new MaterializeExpressionGenerator(state).setColumns(columns) + .generateExpression(MaterializeDataType.BOOLEAN); + } + + private int getUnoptimizedQueryCount(List fromTables, + MaterializeExpression randomWhereCondition, List joinStatements) throws SQLException { + MaterializeSelect select = new MaterializeSelect(); + MaterializeCastOperation isTrue = new MaterializeCastOperation(randomWhereCondition, + MaterializeCompoundDataType.create(MaterializeDataType.INT)); + MaterializePostfixText asText = new MaterializePostfixText(isTrue, " as count", null, MaterializeDataType.INT); + select.setFetchColumns(Arrays.asList(asText)); + select.setFromList(fromTables); + select.setSelectType(SelectType.ALL); + select.setJoinClauses(joinStatements); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + MaterializeVisitor.asString(select) + ") as res"; + if (options.logEachSelect()) { + logger.writeCurrent(unoptimizedQueryString); + } + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGet(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getOptimizedQueryCount(List randomTables, List columns, + MaterializeExpression randomWhereCondition, List joinStatements) throws SQLException { + MaterializeSelect select = new MaterializeSelect(); + MaterializeColumnValue allColumns = new MaterializeColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(Arrays.asList(allColumns)); + select.setFromList(randomTables); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions( + new MaterializeExpressionGenerator(state).setColumns(columns).generateOrderBy()); + } + select.setSelectType(SelectType.ALL); + select.setJoinClauses(joinStatements); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = MaterializeVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + + @Override + public String getLastQueryString() { + return optimizedQueryString; + } +} diff --git a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..07bfb470f --- /dev/null +++ b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java @@ -0,0 +1,151 @@ +package sqlancer.materialize.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeRowValue; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public class MaterializePivotedQuerySynthesisOracle extends + PivotedQuerySynthesisBase { + + private List fetchColumns; + + public MaterializePivotedQuerySynthesisOracle(MaterializeGlobalState globalState) throws SQLException { + super(globalState); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonFetchErrors(errors); + } + + @Override + public SQLQueryAdapter getRectifiedQuery() throws SQLException { + MaterializeTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); + + MaterializeSelect selectStatement = new MaterializeSelect(); + selectStatement.setSelectType(Randomly.fromOptions(MaterializeSelect.SelectType.values())); + List columns = randomFromTables.getColumns(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + + fetchColumns = columns; + selectStatement.setFromList(randomFromTables.getTables().stream().map(t -> new MaterializeFromTable(t, false)) + .collect(Collectors.toList())); + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new MaterializeColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + MaterializeExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + MaterializeExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + MaterializeExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + List orderBy = new MaterializeExpressionGenerator(globalState).setColumns(columns) + .generateOrderBy(); + selectStatement.setOrderByExpressions(orderBy); + return new SQLQueryAdapter(MaterializeVisitor.asString(selectStatement)); + } + + /* + * Prevent name collisions by aliasing the column. + */ + private MaterializeColumn getFetchValueAliasedColumn(MaterializeColumn c) { + MaterializeColumn aliasedColumn = new MaterializeColumn( + c.getName() + " AS " + c.getTable().getName() + c.getName(), c.getType()); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + private List generateGroupByClause(List columns, MaterializeRowValue rw) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> MaterializeColumnValue.create(c, rw.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private MaterializeConstant generateLimit() { + if (Randomly.getBoolean()) { + return MaterializeConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private MaterializeExpression generateOffset() { + if (Randomly.getBoolean()) { + return MaterializeConstant.createIntConstant(0); + } else { + return null; + } + } + + private MaterializeExpression generateRectifiedExpression(List columns, MaterializeRowValue rw) { + MaterializeExpression expr = new MaterializeExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) + .generateExpressionWithExpectedResult(MaterializeDataType.BOOLEAN); + MaterializeExpression result; + if (expr.getExpectedValue().isNull()) { + result = MaterializePostfixOperation.create(expr, PostfixOperator.IS_NULL); + } else { + result = MaterializePostfixOperation.create(expr, + expr.getExpectedValue().cast(MaterializeDataType.BOOLEAN).asBoolean() ? PostfixOperator.IS_TRUE + : PostfixOperator.IS_FALSE); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions + sb.append(query.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (MaterializeColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + @Override + protected String getExpectedValues(MaterializeExpression expr) { + return MaterializeVisitor.asExpectedValues(expr); + } + +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java new file mode 100644 index 000000000..4f5f67294 --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java @@ -0,0 +1,187 @@ +package sqlancer.materialize.oracle.tlp; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.postgresql.util.PSQLException; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; +import sqlancer.materialize.ast.MaterializeAlias; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializePrefixOperation.PrefixOperator; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.gen.MaterializeCommon; + +public class MaterializeTLPAggregateOracle extends MaterializeTLPBase implements TestOracle { + private String generatedQueryString; + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public MaterializeTLPAggregateOracle(MaterializeGlobalState state) { + super(state); + MaterializeCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() throws SQLException { + MaterializeAggregateFunction aggregateFunction = Randomly.fromOptions(MaterializeAggregateFunction.MAX, + MaterializeAggregateFunction.MIN, MaterializeAggregateFunction.SUM, + MaterializeAggregateFunction.BIT_AND, MaterializeAggregateFunction.BIT_OR, + MaterializeAggregateFunction.BOOL_AND, MaterializeAggregateFunction.BOOL_OR, + MaterializeAggregateFunction.COUNT); + MaterializeAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), + aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + originalQuery = MaterializeVisitor.asString(select); + generatedQueryString = originalQuery; + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + String queryFormatString = "-- %s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + String assertionMessage = String.format("the results mismatch!\n%s\n%s", firstQueryString, + secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + private String createMetamorphicUnionQuery(MaterializeSelect select, MaterializeAggregate aggregate, + List from) { + String metamorphicQuery; + MaterializeExpression whereClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeExpression negatedClause = new MaterializePrefixOperation(whereClause, PrefixOperator.NOT); + MaterializeExpression notNullClause = new MaterializePostfixOperation(whereClause, PostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + MaterializeSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); + MaterializeSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); + MaterializeSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += MaterializeVisitor.asString(leftSelect) + " UNION ALL " + + MaterializeVisitor.asString(middleSelect) + " UNION ALL " + MaterializeVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + } catch (PSQLException e) { + throw new AssertionError(queryString, e); + } + return resultString; + } + + private List mapped(MaterializeAggregate aggregate) { + switch (aggregate.getFunction()) { + case SUM: + case COUNT: + case BIT_AND: + case BIT_OR: + case BOOL_AND: + case BOOL_OR: + case MAX: + case MIN: + return aliasArgs(Arrays.asList(aggregate)); + default: + throw new AssertionError(aggregate.getFunction()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (MaterializeExpression expr : originalAggregateArgs) { + args.add(new MaterializeAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(MaterializeAggregate aggregate) { + switch (aggregate.getFunction()) { + case COUNT: + return MaterializeAggregateFunction.SUM.toString() + "(agg0)"; + default: + return aggregate.getFunction().toString() + "(agg0)"; + } + } + + private MaterializeSelect getSelect(List aggregates, List from, + MaterializeExpression whereClause, List joinList) { + MaterializeSelect leftSelect = new MaterializeSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinClauses(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java new file mode 100644 index 000000000..4aa2196d1 --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java @@ -0,0 +1,123 @@ +package sqlancer.materialize.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.ForClause; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; +import sqlancer.materialize.oracle.MaterializeNoRECOracle; + +public class MaterializeTLPBase + extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + protected MaterializeSchema s; + protected MaterializeTables targetTables; + protected MaterializeExpressionGenerator gen; + protected MaterializeSelect select; + + public MaterializeTLPBase(MaterializeGlobalState state) { + super(state); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonFetchErrors(errors); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + List tables = targetTables.getTables(); + List joins = getJoinStatements(state, targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + protected List getJoinStatements(MaterializeGlobalState globalState, + List columns, List tables) { + return MaterializeNoRECOracle.getJoinStatements(state, columns, tables); + // TODO joins + } + + protected void generateSelectBase(List tables, List joins) { + List tableList = tables.stream() + .map(t -> new MaterializeFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + gen = new MaterializeExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new MaterializeSelect(); + select.setFetchColumns(generateFetchColumns()); + select.setFromList(tableList); + select.setWhereClause(null); + select.setJoinClauses(joins); + if (Randomly.getBoolean()) { + select.setForClause(ForClause.getRandom()); + } + } + + List generateFetchColumns() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new MaterializeColumnValue(MaterializeColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(targetTables.getColumns()); + for (MaterializeColumn c : targetColumns) { + fetchColumns.add(new MaterializeColumnValue(c, null)); + } + return fetchColumns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + + public static MaterializeSubquery createSubquery(MaterializeGlobalState globalState, String name, + MaterializeTables tables) { + List columns = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState) + .setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + MaterializeSelect select = new MaterializeSelect(); + select.setFromList(tables.getTables().stream().map(t -> new MaterializeFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return new MaterializeSubquery(select, name); + } +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java new file mode 100644 index 000000000..46d6c6d74 --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java @@ -0,0 +1,83 @@ +package sqlancer.materialize.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.gen.MaterializeCommon; + +public class MaterializeTLPHavingOracle extends MaterializeTLPBase { + private String generatedQueryString; + + public MaterializeTLPHavingOracle(MaterializeGlobalState state) { + super(state); + MaterializeCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + havingCheck(); + } + + protected void havingCheck() throws SQLException { + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(MaterializeDataType.BOOLEAN)); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = MaterializeVisitor.asString(select); + generatedQueryString = originalQueryString; + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + // See https://github.com/MaterializeInc/materialize/issues/18346, have to check if predicate errors by putting + // it in SELECT first + List originalColumns = select.getFetchColumns(); + List checkColumns = new ArrayList<>(); + checkColumns.add(predicate); + select.setFetchColumns(checkColumns); + String errorCheckQueryString = MaterializeVisitor.asString(select); + ComparatorHelper.getResultSetFirstColumnAsString(errorCheckQueryString, errors, state); + select.setFetchColumns(originalColumns); + + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + select.setHavingClause(predicate); + String firstQueryString = MaterializeVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = MaterializeVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = MaterializeVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + protected MaterializeExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List generateFetchColumns() { + List expressions = gen.allowAggregates(true) + .generateExpressions(Randomly.smallNumber() + 1); + gen.allowAggregates(false); + return expressions; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java new file mode 100644 index 000000000..6af0ac2cc --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java @@ -0,0 +1,52 @@ +package sqlancer.materialize.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeVisitor; + +public class MaterializeTLPWhereOracle extends MaterializeTLPBase { + private String generatedQueryString; + + public MaterializeTLPWhereOracle(MaterializeGlobalState state) { + super(state); + } + + @Override + public void check() throws SQLException { + super.check(); + whereCheck(); + } + + protected void whereCheck() throws SQLException { + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + String originalQueryString = MaterializeVisitor.asString(select); + generatedQueryString = originalQueryString; + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setOrderByExpressions(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = MaterializeVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = MaterializeVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = MaterializeVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/test/sqlancer/dbms/TestMaterialize.java b/test/sqlancer/dbms/TestMaterialize.java new file mode 100644 index 000000000..1818aa513 --- /dev/null +++ b/test/sqlancer/dbms/TestMaterialize.java @@ -0,0 +1,24 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMaterialize { + + String materializeAvailable = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materializeAvailable != null && materializeAvailable.equalsIgnoreCase("true"); + + @Test + public void test() { + assumeTrue(materializeIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--username", "materialize", + "materialize", "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/dbms/TestMaterializePQS.java b/test/sqlancer/dbms/TestMaterializePQS.java new file mode 100644 index 000000000..5d4d795e4 --- /dev/null +++ b/test/sqlancer/dbms/TestMaterializePQS.java @@ -0,0 +1,25 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMaterializePQS { + + String materializeAvailable = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materializeAvailable != null && materializeAvailable.equalsIgnoreCase("true"); + + @Test + public void test() { + assumeTrue(materializeIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "--username", "materialize", "materialize", "--oracle", "pqs", + "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQPG.java b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java new file mode 100644 index 000000000..0564d048c --- /dev/null +++ b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java @@ -0,0 +1,23 @@ +package sqlancer.qpg.cockroachdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestMaterializeQPG { + + @Test + public void testMaterializeQPG() { + String materialize = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materialize != null && materialize.equalsIgnoreCase("true"); + assumeTrue(materializeIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "--username", + "materialize", "materialize", "--oracle", "QUERY_PARTITIONING", "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java new file mode 100644 index 000000000..08e402037 --- /dev/null +++ b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java @@ -0,0 +1,49 @@ +package sqlancer.qpg.materialize; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeOptions; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeProvider; + +public class TestMaterializeQueryPlan { + + @Test + void testMaterializeQueryPlan() throws Exception { + String materialize = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materialize != null && materialize.equalsIgnoreCase("true"); + assumeTrue(materializeIsAvailable); + + String databaseName = "materialize"; + MaterializeProvider provider = new MaterializeProvider(); + MaterializeGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + MaterializeOptions materializeOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(materializeOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals( + "Return;Union;Get l0;Project (#2, #3, #0);Union;Negate;Project (#2);Get materialize.public.t2;Get materialize.public.t2;With;Get materialize.public.t1;Get materialize.public.t2;;Source materialize.public.t1;", + queryPlan); + } + +} From 63b03fb1e22be6c7e1b2a411da2ebbbeab9bfc61 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Apr 2023 14:11:57 +0200 Subject: [PATCH 0764/1284] Mz: Fix typo in expected error (#775) Caused by: org.postgresql.util.PSQLException: ERROR: result exceeds max size of 1073.7 MB --- src/sqlancer/materialize/gen/MaterializeCommon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/gen/MaterializeCommon.java b/src/sqlancer/materialize/gen/MaterializeCommon.java index 0c4847bed..7d722e102 100644 --- a/src/sqlancer/materialize/gen/MaterializeCommon.java +++ b/src/sqlancer/materialize/gen/MaterializeCommon.java @@ -31,7 +31,7 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("non-integer constant in GROUP BY"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY position"); - errors.add("results exceed max size of"); + errors.add("results exceeds max size of"); errors.add("does not exist"); errors.add("aggregate functions are not allowed in"); From 28e6c15086c3e81b93dc2a3980c957932cd383d7 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 12 Apr 2023 09:18:51 +0200 Subject: [PATCH 0765/1284] Materialize: Use Serializable isolation level (#778) --- src/sqlancer/materialize/MaterializeProvider.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java index dc802d2aa..3e6e1c59f 100644 --- a/src/sqlancer/materialize/MaterializeProvider.java +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -216,6 +216,12 @@ public SQLConnection createDatabase(MaterializeGlobalState globalState) throws S globalState.getState().logStatement(String.format("\\c %s;", databaseName)); con = DriverManager.getConnection("jdbc:" + testURL, username, password); + // Serializable transaction isolation is much faster than Strict + // Serializable and should guarantee enough for SQLancer: + // https://materialize.com/docs/overview/isolation-level/ + try (Statement s = con.createStatement()) { + s.execute("SET transaction_isolation = 'SERIALIZABLE'"); + } return new SQLConnection(con); } From ba2059740dcc6afe65cbd49d66f29e9afe4c4f15 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 13 Apr 2023 23:00:41 +0800 Subject: [PATCH 0766/1284] tidb: support tidb_enable_non_prepared_plan_cache (#776) * tidb: support tidb_enable_non_prepared_plan_cache Signed-off-by: Weizhen Wang * update Signed-off-by: Weizhen Wang --------- Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBOptions.java | 3 +++ src/sqlancer/tidb/TiDBProvider.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 764c4bfd9..6fbab6ba8 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -32,6 +32,9 @@ public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracle = Arrays.asList(TiDBOracleFactory.QUERY_PARTITIONING); + @Parameter(names = "--enable-non-prepared-plan-cache") + public boolean nonPreparePlanCache; + public enum TiDBOracleFactory implements OracleFactory { HAVING { @Override diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index c4f13a207..91223a246 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -156,6 +156,9 @@ public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLExcep globalState.getState().logStatement("USE " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); + if (globalState.getDbmsSpecificOptions().nonPreparePlanCache) { + s.execute("set global tidb_enable_non_prepared_plan_cache=ON;"); + } } try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); From b4cd36e5c47242837a8ee41f558e74bef99cb369 Mon Sep 17 00:00:00 2001 From: Rachael Harding Date: Wed, 26 Apr 2023 08:39:00 -0700 Subject: [PATCH 0767/1284] Update CockroachDB to v23.1.0-beta.2 (#779) Disable ignored errors from CockroachDB bugs that have been fixed. --- .github/workflows/main.yml | 12 +++++----- src/sqlancer/cockroachdb/CockroachDBBugs.java | 24 ++++++++++--------- .../cockroachdb/CockroachDBErrors.java | 4 ++++ .../gen/CockroachDBTableGenerator.java | 1 + 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1892285b2..d4166e941 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -122,11 +122,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.3.linux-amd64.tgz | tar xvz - cd cockroach-v22.2.3.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-beta.2.linux-amd64.tgz | tar xvz + cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -145,11 +145,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.3.linux-amd64.tgz | tar xvz - cd cockroach-v22.2.3.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-beta.2.linux-amd64.tgz | tar xvz + cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 0d708c9a2..55045f637 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -18,41 +18,43 @@ public final class CockroachDBBugs { public static boolean bug83874 = true; // https://github.com/cockroachdb/cockroach/issues/83973 - public static boolean bug83973 = true; + public static boolean bug83973; // https://github.com/cockroachdb/cockroach/issues/83976 - public static boolean bug83976 = true; + public static boolean bug83976; + // The following bug is closed, but leave it enabled until + // the underlying interval issue is resolved. + // https://github.com/cockroachdb/cockroach/issues/84078 // https://github.com/cockroachdb/cockroach/issues/84154 public static boolean bug84154 = true; // https://github.com/cockroachdb/cockroach/issues/85356 - public static boolean bug85356 = true; + public static boolean bug85356; // https://github.com/cockroachdb/cockroach/issues/85371 - public static boolean bug85371 = true; + public static boolean bug85371; // https://github.com/cockroachdb/cockroach/issues/85389 - public static boolean bug85389 = true; + public static boolean bug85389; // https://github.com/cockroachdb/cockroach/issues/85390 - public static boolean bug85390 = true; + public static boolean bug85390; // https://github.com/cockroachdb/cockroach/issues/85393 - public static boolean bug85393 = true; + public static boolean bug85393; // https://github.com/cockroachdb/cockroach/issues/85394 public static boolean bug85394 = true; // https://github.com/cockroachdb/cockroach/issues/85441 - public static boolean bug85441 = true; + public static boolean bug85441; // https://github.com/cockroachdb/cockroach/issues/85499 - public static boolean bug85499 = true; + public static boolean bug85499; // https://github.com/cockroachdb/cockroach/issues/88037 - // TODO: This should be fixed in v22.2.1. - public static boolean bug88037 = true; + public static boolean bug88037; private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index bbbda7baf..e282981dc 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -74,6 +74,10 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add(" unsupported comparison operator: NOT LIKE != "); errors.add("expected DEFAULT expression to have type bytes"); + errors.add("expected DEFAULT (in CREATE TABLE) expression to have type bytes"); + errors.add("expected DEFAULT (in CREATE VIEW) expression to have type bytes"); + errors.add("expected DEFAULT (in SET DEFAULT) expression to have type bytes"); + errors.add("expected DEFAULT (in ADD COLUMN) expression to have type bytes"); errors.add("value type string doesn't match type bytes of column"); errors.add("as decimal, found type: int"); errors.add("to be of type decimal, found type float"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index c32fa76eb..c8bfa3c6a 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -38,6 +38,7 @@ public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { @Override public void buildStatement() { errors.add("and thus is not indexable"); // array types are not indexable + errors.add("context-dependent operators are not allowed in STORED COMPUTED COLUMN"); if (globalState.getDbmsSpecificOptions().testTempTables) { errors.add("constraints on temporary tables may reference only temporary tables"); errors.add("constraints on permanent tables may reference only permanent tables"); From 18b35964733949593c2d6ffb771b245a5ee7d2aa Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 27 Apr 2023 19:32:01 +0200 Subject: [PATCH 0768/1284] Materialize: Typo in error message (#782) Caused by: org.postgresql.util.PSQLException: ERROR: result exceeds max size of 1073.7 MB --- src/sqlancer/materialize/gen/MaterializeCommon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/gen/MaterializeCommon.java b/src/sqlancer/materialize/gen/MaterializeCommon.java index 7d722e102..7dee348f4 100644 --- a/src/sqlancer/materialize/gen/MaterializeCommon.java +++ b/src/sqlancer/materialize/gen/MaterializeCommon.java @@ -31,7 +31,7 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("non-integer constant in GROUP BY"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY position"); - errors.add("results exceeds max size of"); + errors.add("result exceeds max size of"); errors.add("does not exist"); errors.add("aggregate functions are not allowed in"); From 9e17ede91412199f2633d8b1c724e1ee37f76de1 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Sat, 20 May 2023 21:43:51 +0800 Subject: [PATCH 0769/1284] fix: partly fix ci error in databend (#784) * fix: fix ci error https://github.com/sqlancer/sqlancer/actions/runs/4822718555/jobs/8590290712 * style: run mvn format * fix: fix bool type size * fix: Nullable type * fix: add FLOAT64 * refactor: change the sequence correspond to https://databend.rs/doc/sql-reference/data-types/#general-data-types --- src/sqlancer/databend/DatabendSchema.java | 47 +++++++++++++++-------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 2e8e3c2bf..1400d9d5e 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -232,48 +232,65 @@ public DatabendTables getRandomTableNonEmptyAndViewTables() { } private static DatabendCompositeDataType getColumnType(String typeString) { - DatabendDataType primitiveType; - int size = -1; if (typeString.startsWith("DECIMAL")) { // Ugly hack return new DatabendCompositeDataType(DatabendDataType.FLOAT, 8); } - switch (typeString) { - case "INT": + if (typeString.startsWith("Nullable")) { // Ugly hack + String substring = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + return getColumnTypeNormalCases(substring); + } + return getColumnTypeNormalCases(typeString); + } + + private static DatabendCompositeDataType getColumnTypeNormalCases(String typeString) { + DatabendDataType primitiveType; + int size = -1; + switch (typeString.toUpperCase()) { + case "BOOLEAN": + case "BOOL": + primitiveType = DatabendDataType.BOOLEAN; + size = 1; + break; + case "TINYINT": + case "INT8": primitiveType = INT; - size = 4; + size = 1; break; case "SMALLINT": + case "INT16": primitiveType = INT; size = 2; break; - case "BIGINT": + case "INT": + case "INT32": primitiveType = INT; - size = 8; + size = 4; break; - case "TINYINT": + case "BIGINT": + case "INT64": primitiveType = INT; - size = 1; - break; - case "VARCHAR": - primitiveType = DatabendDataType.VARCHAR; + size = 8; break; case "FLOAT": + case "FLOAT32": primitiveType = DatabendDataType.FLOAT; size = 4; break; case "DOUBLE": + case "FLOAT64": primitiveType = DatabendDataType.FLOAT; size = 8; break; - case "BOOLEAN": - primitiveType = DatabendDataType.BOOLEAN; - break; // case "DATE": // primitiveType = DatabendDataType.DATE; // break; // case "TIMESTAMP": // primitiveType = DatabendDataType.TIMESTAMP; // break; + case "VARCHAR": + case "STRING": + primitiveType = DatabendDataType.VARCHAR; + break; case "NULL": primitiveType = DatabendDataType.NULL; break; From e696c6e93c6f88e9edf173b482cff65ff82184c8 Mon Sep 17 00:00:00 2001 From: huang xu Date: Mon, 22 May 2023 11:29:53 +0800 Subject: [PATCH 0770/1284] Support Apache Doris (#758) * support doris init commit * support doris complete Signed-off-by: ChaseHuangxu * code format & correct github workflow main.yml about doris & add doris to README.md support dbms * correct README.md support dbms * change sql union method to union all in TLPDistinct check Signed-off-by: ChaseHuangxu * Limit the data format inserted by decimal type, and support Doris function call Signed-off-by: ChaseHuangxu * correct github/workflows/DBMS Tests(doris) Signed-off-by: ChaseHuangxu --------- Signed-off-by: ChaseHuangxu --- .github/workflows/main.yml | 43 +- README.md | 3 +- src/check_names.py | 1 + src/sqlancer/doris/DorisBugs.java | 45 ++ src/sqlancer/doris/DorisErrors.java | 64 ++ src/sqlancer/doris/DorisOptions.java | 194 +++++ src/sqlancer/doris/DorisProvider.java | 155 ++++ src/sqlancer/doris/DorisSchema.java | 611 ++++++++++++++++ .../doris/ast/DorisAggregateOperation.java | 39 + .../doris/ast/DorisBetweenOperation.java | 39 + .../ast/DorisBinaryArithmeticOperation.java | 143 ++++ .../ast/DorisBinaryComparisonOperation.java | 120 +++ .../ast/DorisBinaryLogicalOperation.java | 130 ++++ .../doris/ast/DorisCaseOperation.java | 38 + .../doris/ast/DorisCastOperation.java | 46 ++ src/sqlancer/doris/ast/DorisColumnValue.java | 53 ++ src/sqlancer/doris/ast/DorisConstant.java | 681 ++++++++++++++++++ src/sqlancer/doris/ast/DorisExpression.java | 13 + .../doris/ast/DorisFunctionOperation.java | 282 ++++++++ src/sqlancer/doris/ast/DorisInOperation.java | 53 ++ src/sqlancer/doris/ast/DorisJoin.java | 108 +++ .../doris/ast/DorisLikeOperation.java | 85 +++ src/sqlancer/doris/ast/DorisOrderByTerm.java | 10 + src/sqlancer/doris/ast/DorisSelect.java | 41 ++ .../doris/ast/DorisUnaryPostfixOperation.java | 87 +++ .../doris/ast/DorisUnaryPrefixOperation.java | 112 +++ .../doris/gen/DorisAlterTableGenerator.java | 50 ++ .../doris/gen/DorisDeleteGenerator.java | 31 + .../doris/gen/DorisDropTableGenerator.java | 28 + .../doris/gen/DorisDropViewGenerator.java | 27 + .../doris/gen/DorisIndexGenerator.java | 46 ++ .../doris/gen/DorisInsertGenerator.java | 56 ++ .../gen/DorisNewExpressionGenerator.java | 427 +++++++++++ .../gen/DorisRandomQuerySynthesizer.java | 79 ++ .../doris/gen/DorisTableGenerator.java | 114 +++ .../doris/gen/DorisUpdateGenerator.java | 56 ++ .../doris/gen/DorisViewGenerator.java | 38 + .../doris/oracle/DorisNoRECOracle.java | 137 ++++ .../DorisPivotedQuerySynthesisOracle.java | 155 ++++ ...DorisQueryPartitioningAggregateTester.java | 194 +++++ .../tlp/DorisQueryPartitioningBase.java | 98 +++ .../DorisQueryPartitioningDistinctTester.java | 45 ++ .../DorisQueryPartitioningGroupByTester.java | 55 ++ .../DorisQueryPartitioningHavingTester.java | 73 ++ .../DorisQueryPartitioningWhereTester.java | 53 ++ .../doris/utils/DorisNumberUtils.java | 111 +++ .../visitor/DorisExpectedValueVisitor.java | 155 ++++ .../doris/visitor/DorisExprToNode.java | 25 + .../doris/visitor/DorisToStringVisitor.java | 175 +++++ test/sqlancer/dbms/TestDoris.java | 54 ++ 50 files changed, 5476 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/doris/DorisBugs.java create mode 100644 src/sqlancer/doris/DorisErrors.java create mode 100644 src/sqlancer/doris/DorisOptions.java create mode 100644 src/sqlancer/doris/DorisProvider.java create mode 100644 src/sqlancer/doris/DorisSchema.java create mode 100644 src/sqlancer/doris/ast/DorisAggregateOperation.java create mode 100644 src/sqlancer/doris/ast/DorisBetweenOperation.java create mode 100644 src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java create mode 100644 src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java create mode 100644 src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java create mode 100644 src/sqlancer/doris/ast/DorisCaseOperation.java create mode 100644 src/sqlancer/doris/ast/DorisCastOperation.java create mode 100644 src/sqlancer/doris/ast/DorisColumnValue.java create mode 100644 src/sqlancer/doris/ast/DorisConstant.java create mode 100644 src/sqlancer/doris/ast/DorisExpression.java create mode 100644 src/sqlancer/doris/ast/DorisFunctionOperation.java create mode 100644 src/sqlancer/doris/ast/DorisInOperation.java create mode 100644 src/sqlancer/doris/ast/DorisJoin.java create mode 100644 src/sqlancer/doris/ast/DorisLikeOperation.java create mode 100644 src/sqlancer/doris/ast/DorisOrderByTerm.java create mode 100644 src/sqlancer/doris/ast/DorisSelect.java create mode 100644 src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java create mode 100644 src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java create mode 100644 src/sqlancer/doris/gen/DorisAlterTableGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisDeleteGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisDropTableGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisDropViewGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisIndexGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisInsertGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisNewExpressionGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java create mode 100644 src/sqlancer/doris/gen/DorisTableGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisUpdateGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisViewGenerator.java create mode 100644 src/sqlancer/doris/oracle/DorisNoRECOracle.java create mode 100644 src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/doris/utils/DorisNumberUtils.java create mode 100644 src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java create mode 100644 src/sqlancer/doris/visitor/DorisExprToNode.java create mode 100644 src/sqlancer/doris/visitor/DorisToStringVisitor.java create mode 100644 test/sqlancer/dbms/TestDoris.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d4166e941..c03e3908c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -442,11 +442,52 @@ jobs: - name: Setup Yugabyte run: | docker pull yugabytedb/yugabyte:latest - docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false + docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false sleep 5 - name: Run Tests run: YUGABYTE_AVAILABLE=true mvn -Dtest=TestYugabyte test + + doris: + name: DBMS Tests (Apache Doris) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: install mysql client + run: | + sudo apt update + sudo apt install mysql-client --assume-yes + - name: Set up Apache Doris + run: | + sudo sysctl -w vm.max_map_count=2000000 + LASTEST_TAG=$(curl -s GET https://api.github.com/repos/apache/doris/releases | jq -r '.[].tag_name' | sed -n 1p) + LASTEST_TAG_BIG_VERSION=$(echo ${LASTEST_TAG} | awk '{split($1, arr, "."); print arr[1]"."arr[2]}') + curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz" + curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz" + curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz" + mkdir ./doris + tar xf apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-fe-${LASTEST_TAG}-bin-x86_64 doris/fe + tar xf apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-be-${LASTEST_TAG}-bin-x86_64 doris/be + tar xf apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64 doris/dependencies + cp doris/dependencies/*.jar doris/be/lib/ + doris/fe/bin/start_fe.sh --daemon + doris/be/bin/start_be.sh --daemon + sudo apt install libnet-ifconfig-wrapper-perl --assume-yes + IP=$(ifconfig eth0 | grep inet | grep -v inet6 | awk '{print $2}') + mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';" + mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: DORIS_AVAILABLE=true mvn -Dtest=TestDoris test + + java13: name: Java 13 Compatibility (DuckDB) runs-on: ubuntu-latest diff --git a/README.md b/README.md index 59f69cce2..f22b00f4d 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,8 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | Databend | Working | Typed | | | QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | | CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | -| Materialize |Working | Typed | | +| Materialize |Working | Typed | +| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. # Using SQLancer diff --git a/src/check_names.py b/src/check_names.py index 6dd4f1a72..1b110370b 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -52,4 +52,5 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["SQLite3"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "sqlite3")) name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes + name_to_files["Doris"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "doris")) verify_all_dbs(name_to_files) diff --git a/src/sqlancer/doris/DorisBugs.java b/src/sqlancer/doris/DorisBugs.java new file mode 100644 index 000000000..080aa0450 --- /dev/null +++ b/src/sqlancer/doris/DorisBugs.java @@ -0,0 +1,45 @@ +package sqlancer.doris; + +public final class DorisBugs { + // https://github.com/apache/doris/issues/17697 + // Logical bug about where true not in (columns) + public static boolean bug17697 = true; + + // https://github.com/apache/doris/issues/17700 + // Cannot use between and in boolean column + public static boolean bug17700 = true; + + // https://github.com/apache/doris/issues/17701 + // Wrong result of `where column not in (values)` + public static boolean bug17701 = true; + + // https://github.com/apache/doris/issues/17705 + // Different result caused by `where` split and union all + public static boolean bug17705 = true; + + // https://github.com/apache/doris/issues/19370 + // Internal Error occur in GroupBy&Having sql + // fixed by https://github.com/apache/doris/pull/19559 + public static boolean bug19370 = true; + + // https://github.com/apache/doris/issues/19374 + // Different result of having not ($value in column) and having ($value not in column) + // fixed by https://github.com/apache/doris/pull/19471 + public static boolean bug19374 = true; + + // https://github.com/apache/doris/issues/19611 + // ERROR occur in nested subqueries with same column name and union + public static boolean bug19611 = true; + + // https://github.com/apache/doris/issues/19613 + // Wrong result when right outer join and where false + public static boolean bug19613 = true; + + // https://github.com/apache/doris/issues/19614 + // Wrong result when value like column from table_join + public static boolean bug19614 = true; + + private DorisBugs() { + + } +} diff --git a/src/sqlancer/doris/DorisErrors.java b/src/sqlancer/doris/DorisErrors.java new file mode 100644 index 000000000..678601635 --- /dev/null +++ b/src/sqlancer/doris/DorisErrors.java @@ -0,0 +1,64 @@ +package sqlancer.doris; + +import sqlancer.common.query.ExpectedErrors; + +public final class DorisErrors { + + private DorisErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + // SQL syntax error + errors.add("Syntax error"); + errors.add("Please check your sql, we meet an error when parsing"); + errors.add("but returns type"); + errors.add("is not a number"); + + // Not in line with Doris' logic + errors.add("Unexpected exception: null"); + errors.add("Cross join can't be used with ON clause"); + errors.add("BetweenPredicate needs to be rewritten into a CompoundPredicate"); + errors.add("can't be assigned to some PlanNode"); + errors.add("can not cast from origin type"); + errors.add("not produced by aggregation output"); + errors.add("cannot combine"); // cannot combine SELECT DISTINCT with aggregate functions or GROUP BY + errors.add("Invalid type"); + errors.add("cannot be cast to"); + + // functions + errors.add("No matching function with signature"); + errors.add("Invalid number format"); + errors.add("group_concat requires"); + errors.add("function's argument should be"); + errors.add("requires a numeric parameter"); + errors.add("out of bounds"); + errors.add("function do not support"); + errors.add("parameter must be"); + errors.add("Not supported input arguments types"); + errors.add("No matching function with signature"); + errors.add("function"); + errors.add("Invalid"); + errors.add("Incorrect"); + + // regex + + // To avoid bugs + if (DorisBugs.bug19370) { + errors.add("failed to initialize storage"); + } + if (DorisBugs.bug19374) { + errors.add("the size of the result sets mismatch"); + } + if (DorisBugs.bug19611) { + errors.add("Duplicated inline view column alias"); + } + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.add("Insert has filtered data in strict mode"); + errors.add("Only value columns of unique table could be updated"); + errors.add("Only unique olap table could be updated"); + errors.add("Number out of range"); + } + +} diff --git a/src/sqlancer/doris/DorisOptions.java b/src/sqlancer/doris/DorisOptions.java new file mode 100644 index 000000000..a2a05b95e --- /dev/null +++ b/src/sqlancer/doris/DorisOptions.java @@ -0,0 +1,194 @@ +package sqlancer.doris; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.doris.DorisOptions.DorisOracleFactory; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.oracle.DorisNoRECOracle; +import sqlancer.doris.oracle.DorisPivotedQuerySynthesisOracle; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningAggregateTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningDistinctTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningGroupByTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningHavingTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningWhereTester; + +@Parameters(commandDescription = "Apache Doris (default port: " + DorisOptions.DEFAULT_PORT + ", default host: " + + DorisOptions.DEFAULT_HOST + ")") +public class DorisOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 9030; + + @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables/views that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + + @Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1) + public boolean testDefaultValues = true; + + @Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1) + public boolean testNotNullConstraints = true; + + @Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1) + public boolean testFunctions; + + @Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1) + public boolean testCasts = true; + + @Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions", arity = 1) + public boolean testBetween = true; + + @Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1) + public boolean testIn = true; + + @Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1) + public boolean testCase = true; + + @Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1) + public boolean testBinaryLogicals = true; + + @Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1) + public boolean testIntConstants = true; + + @Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1) + public boolean testFloatConstants = true; + + @Parameter(names = "--test-decimal-constants", description = "Allow generating DECIMAL constants", arity = 1) + public boolean testDecimalConstants = true; + + @Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testDateConstants = true; + + @Parameter(names = "--test-datetime-constants", description = "Allow generating DATETIME constants", arity = 1) + public boolean testDateTimeConstants = true; + + @Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1) + public boolean testStringConstants = true; + + @Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1) + public boolean testBooleanConstants = true; + + @Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1) + public boolean testBinaryComparisons = true; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates; + + @Parameter(names = "--max-num-table-alters", description = "The maximum number of ALTER TABLE statements that are issued for a database", arity = 1) + public int maxNumTableAlters; + + @Parameter(names = "--test-engine-type", description = "The engine type in Doris, only consider OLAP now", arity = 1) + public String testEngineType = "OLAP"; // skip now + + @Parameter(names = "--test-indexes", description = "Allow explicit indexes, Doris only supports creating indexes on single-column BITMAP", arity = 1) + public boolean testIndexes = true; // skip now + + @Parameter(names = "--test-column-aggr", description = "Allow test column aggregation (sum, min, max, replace, replace_if_not_null, hll_union, bitmap_untion)", arity = 1) + public boolean testColumnAggr = true; + + @Parameter(names = "--test-datemodel", description = "Allow generating Doris’s data model in tables. (Aggregate、Uniqe、Duplicate)", arity = 1) + public boolean testDataModel = true; + + @Parameter(names = "--test-distribution", description = "Allow generating data distribution in tables.", arity = 1) + public boolean testDistribution = true; // must have it, skip now + + @Parameter(names = "--test-rollup", description = "Allow generating rollups in tables.", arity = 1) + public boolean testRollup = true; // skip now + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(DorisOracleFactory.NOREC); + + public enum DorisOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisNoRECOracle(globalState); + } + + }, + HAVING { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); + oracles.add(new DorisQueryPartitioningWhereTester(globalState)); + oracles.add(new DorisQueryPartitioningHavingTester(globalState)); + oracles.add(new DorisQueryPartitioningAggregateTester(globalState)); + oracles.add(new DorisQueryPartitioningDistinctTester(globalState)); + oracles.add(new DorisQueryPartitioningGroupByTester(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + PQS { + @Override + public TestOracle create(DorisGlobalState globalState) throws Exception { + return new DorisPivotedQuerySynthesisOracle(globalState); + } + }, + ALL { + @Override + public TestOracle create(DorisGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(new DorisNoRECOracle(globalState)); + oracles.add(new DorisQueryPartitioningWhereTester(globalState)); + oracles.add(new DorisQueryPartitioningHavingTester(globalState)); + oracles.add(new DorisQueryPartitioningAggregateTester(globalState)); + oracles.add(new DorisQueryPartitioningDistinctTester(globalState)); + oracles.add(new DorisQueryPartitioningGroupByTester(globalState)); + oracles.add(new DorisPivotedQuerySynthesisOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + + } + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/doris/DorisProvider.java b/src/sqlancer/doris/DorisProvider.java new file mode 100644 index 000000000..3f64231f1 --- /dev/null +++ b/src/sqlancer/doris/DorisProvider.java @@ -0,0 +1,155 @@ +package sqlancer.doris; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.gen.DorisAlterTableGenerator; +import sqlancer.doris.gen.DorisDeleteGenerator; +import sqlancer.doris.gen.DorisDropTableGenerator; +import sqlancer.doris.gen.DorisDropViewGenerator; +import sqlancer.doris.gen.DorisIndexGenerator; +import sqlancer.doris.gen.DorisInsertGenerator; +import sqlancer.doris.gen.DorisTableGenerator; +import sqlancer.doris.gen.DorisUpdateGenerator; +import sqlancer.doris.gen.DorisViewGenerator; + +@AutoService(DatabaseProvider.class) +public class DorisProvider extends SQLProviderAdapter { + + public DorisProvider() { + super(DorisGlobalState.class, DorisOptions.class); + } + + public enum Action implements AbstractAction { + CREATE_TABLE(DorisTableGenerator::createRandomTableStatement), CREATE_VIEW(DorisViewGenerator::getQuery), + CREATE_INDEX(DorisIndexGenerator::getQuery), INSERT(DorisInsertGenerator::getQuery), + DELETE(DorisDeleteGenerator::generate), UPDATE(DorisUpdateGenerator::getQuery), + ALTER_TABLE(DorisAlterTableGenerator::getQuery), + TRUNCATE((g) -> new SQLQueryAdapter( + "TRUNCATE TABLE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), + DROP_TABLE(DorisDropTableGenerator::dropTable), DROP_VIEW(DorisDropViewGenerator::dropView); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(DorisGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(DorisGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes); + case UPDATE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates); + case ALTER_TABLE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumTableAlters); + case TRUNCATE: + return r.getInteger(0, 2); + case CREATE_TABLE: + case CREATE_INDEX: + case CREATE_VIEW: + case DROP_TABLE: + case DROP_VIEW: + return 0; + default: + throw new AssertionError(a); + } + } + + public static class DorisGlobalState extends SQLGlobalState { + + @Override + protected DorisSchema readSchema() throws SQLException { + return DorisSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(DorisGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success = false; + do { + SQLQueryAdapter qt = new DorisTableGenerator().getQuery(globalState); + if (qt != null) { + success = globalState.executeStatement(qt); + } + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + DorisProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(DorisGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + if (password.equals("\"\"")) { + password = ""; + } + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = DorisOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = DorisOptions.DEFAULT_PORT; + } + String databaseName = globalState.getDatabaseName(); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "doris"; + } + +} diff --git a/src/sqlancer/doris/DorisSchema.java b/src/sqlancer/doris/DorisSchema.java new file mode 100644 index 000000000..d452f784a --- /dev/null +++ b/src/sqlancer/doris/DorisSchema.java @@ -0,0 +1,611 @@ +package sqlancer.doris; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.ast.DorisConstant; + +public class DorisSchema extends AbstractSchema { + + public enum DorisTableDataModel { + UNIQUE, AGGREGATE, DUPLICATE; + + public static DorisTableDataModel getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum DorisColumnAggrType { + SUM, MIN, MAX, REPLACE, REPLCAE_IF_NOT_NULL, BITMAP_UNION, HLL_UNION, NULL; + + public static DorisColumnAggrType getRandom(DorisCompositeDataType columnDataType) { + // if (columnDataType.getPrimitiveDataType() == DorisSchema.DorisDataType.BITMAP) { + // return DorisColumnAggrType.BITMAP_UNION; + // } + // if (columnDataType.getPrimitiveDataType() == DorisSchema.DorisDataType.HLL) { + // return DorisColumnAggrType.HLL_UNION; + // } + + return Randomly.fromOptions(SUM, MIN, MAX, REPLACE, REPLCAE_IF_NOT_NULL); + } + } + + public enum DorisDataType { + INT, FLOAT, DECIMAL, DATE, DATETIME, VARCHAR, BOOLEAN, NULL; + // HLL, BITMAP, ARRAY; + + private int decimalScale; + private int decimalPrecision; + private int varcharLength; + + public static DorisDataType getRandomWithoutNull() { + DorisDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DorisDataType.NULL); + return dt; + } + + public int getDecimalScale() { + return decimalScale; + } + + public void setDecimalScale(int decimalScale) { + this.decimalScale = decimalScale; + } + + public int getDecimalPrecision() { + return decimalPrecision; + } + + public void setDecimalPrecision(int decimalPrecision) { + this.decimalPrecision = decimalPrecision; + } + + public int getVarcharLength() { + return varcharLength; + } + + public void setVarcharLength(int varcharLength) { + this.varcharLength = varcharLength; + } + } + + public static class DorisCompositeDataType { + + private final DorisDataType dataType; + + private final int size; + + public DorisCompositeDataType(DorisDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public DorisDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static DorisCompositeDataType getRandomWithoutNull() { + DorisDataType type = DorisDataType.getRandomWithoutNull(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8, 16); + break; + case FLOAT: + size = Randomly.fromOptions(4, 12); + break; + case DECIMAL: + size = Randomly.fromOptions(1, 3); // DECIMAL or DECIMALV3 + break; + case DATE: + case DATETIME: + case VARCHAR: + case BOOLEAN: + // case HLL: + // case BITMAP: + // case ARRAY: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new DorisCompositeDataType(type, size); + } + + public void initColumnArgs() { + Randomly r = new Randomly(); + int scale; + int precision; + int varcharLength; + switch (getPrimitiveDataType()) { + case DECIMAL: + if (getPrimitiveDataType().getDecimalPrecision() != 0) { + break; + } + if (size == 1) { + scale = r.getInteger(0, 9); + precision = r.getInteger(scale + 1, scale + 18); + getPrimitiveDataType().setDecimalPrecision(precision); + getPrimitiveDataType().setDecimalScale(scale); + } else if (size == 3) { + precision = r.getInteger(1, 38); + scale = r.getInteger(0, precision); + getPrimitiveDataType().setDecimalPrecision(precision); + getPrimitiveDataType().setDecimalScale(scale); + } else { + throw new AssertionError(size); + } + break; + case VARCHAR: + if (getPrimitiveDataType().getVarcharLength() != 0) { + break; + } + varcharLength = r.getInteger(1, 255); + getPrimitiveDataType().setVarcharLength(varcharLength); + break; + default: + // pass + } + + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 16: + return "LARGEINT"; + case 8: + return "BIGINT"; + case 4: + return "INT"; + case 2: + return "SMALLINT"; + case 1: + return "TINYINT"; + default: + throw new AssertionError(size); + } + case FLOAT: + switch (size) { + case 12: + return "DOUBLE"; + case 4: + return "FLOAT"; + default: + throw new AssertionError(size); + } + case DECIMAL: + switch (size) { + case 1: + return "DECIMAL(" + getPrimitiveDataType().getDecimalPrecision() + "," + + getPrimitiveDataType().getDecimalScale() + ")"; + case 3: + return "DECIMALV3(" + getPrimitiveDataType().getDecimalPrecision() + "," + + getPrimitiveDataType().getDecimalScale() + ")"; + default: + throw new AssertionError(size); + } + case DATE: + return "DATEV2"; + case DATETIME: + return Randomly.fromOptions("DATETIME", "DATETIMEV2"); + case VARCHAR: + return Randomly.fromOptions("VARCHAR", "CHAR") + "(" + getPrimitiveDataType().getVarcharLength() + ")"; + case BOOLEAN: + return "BOOLEAN"; + // case HLL: + // return "HLL"; + // case BITMAP: + // return "BITMAP"; + // case ARRAY: + // return "ARRAY"; + case NULL: + return Randomly.fromOptions("NULL"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + public boolean canBeKey() { + switch (dataType) { + // case HLL: + // case BITMAP: + // case ARRAY: + case FLOAT: + return false; + default: + return true; + } + } + + } + + public static class DorisColumn extends AbstractTableColumn { + + private final boolean isKey; + private final boolean isNullable; + private final DorisColumnAggrType aggrType; + private final boolean hasDefaultValue; + private final String defaultValue; + + public DorisColumn(String name, DorisCompositeDataType type, boolean isKey, boolean isNullable, + DorisColumnAggrType aggrType, boolean hasDefaultValue, String defaultValue) { + super(name, null, type); + this.isKey = isKey; + this.isNullable = isNullable; + this.aggrType = aggrType; + this.hasDefaultValue = hasDefaultValue; + this.defaultValue = defaultValue; + } + + public DorisColumn(String name, DorisCompositeDataType type, boolean isKey, boolean isNullable) { + super(name, null, type); + this.isKey = isKey; + this.isNullable = isNullable; + this.aggrType = DorisColumnAggrType.NULL; + this.hasDefaultValue = false; + this.defaultValue = ""; + } + + public boolean isKey() { + return isKey; + } + + public boolean isNullable() { + return isNullable; + } + + public boolean hasDefaultValue() { + return hasDefaultValue; + } + + @Override + public String toString() { + String ret = this.getName() + " " + this.getType(); + if (aggrType != DorisColumnAggrType.NULL) { + ret += " " + aggrType.name(); + } + if (!isNullable) { + ret += " NOT NULL"; + } + if (hasDefaultValue) { + ret += " DEFAULT " + defaultValue; + } + return ret; + } + + @Override + public int compareTo(AbstractTableColumn o) { + // To sort columns + DorisColumn other = (DorisColumn) o; + if (isKey != other.isKey) { + return isKey ? 1 : -1; + } + return getName().compareTo(other.getName()); + } + } + + public static class DorisTables extends AbstractTables { + + public DorisTables(List tables) { + super(tables); + } + + public DorisRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String rowValueQuery = String.format("SELECT %s FROM %s ORDER BY 1 LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet rs = s.executeQuery(rowValueQuery); + if (!rs.next()) { + throw new IgnoreMeException(); + // throw new AssertionError("could not find random row " + rowValueQuery + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + DorisColumn column = getColumns().get(i); + int columnIndex = rs.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + DorisConstant constant; + if (rs.getString(columnIndex) == null) { + constant = DorisConstant.createNullConstant(); + } else { + switch (column.getType().getPrimitiveDataType()) { + case INT: + constant = DorisConstant.createIntConstant(rs.getLong(columnIndex)); + break; + case FLOAT: + case DECIMAL: + constant = DorisConstant.createFloatConstant(rs.getDouble(columnIndex)); + break; + case DATE: + constant = DorisConstant.createDateConstant(rs.getString(columnIndex)); + break; + case DATETIME: + constant = DorisConstant.createDatetimeConstant(rs.getString(columnIndex)); + break; + case VARCHAR: + constant = DorisConstant.createStringConstant(rs.getString(columnIndex)); + break; + case BOOLEAN: + constant = DorisConstant.createBooleanConstant(rs.getBoolean(columnIndex)); + break; + case NULL: + constant = DorisConstant.createNullConstant(); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !rs.next(); + return new DorisRowValue(this, values); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } + + } + + public static class DorisRowValue extends AbstractRowValue { + + DorisRowValue(DorisTables tables, Map values) { + super(tables, values); + } + + } + + public DorisSchema(List databaseTables) { + super(databaseTables); + } + + public DorisTables getRandomTableNonEmptyTables() { + return new DorisTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public DorisTables getRandomTableNonEmptyAndViewTables() { + List tables = getDatabaseTables().stream().filter(t -> !t.isView()).collect(Collectors.toList()); + tables = Randomly.nonEmptySubset(tables); + return new DorisTables(tables); + } + + public int getIndexCount() { + int count = 0; + for (DorisTable table : getDatabaseTables()) { + count += table.getIndexes().size(); + } + return count; + } + + private static DorisCompositeDataType getColumnType(String typeString) { + DorisDataType primitiveType; + int size = -1; + + if (typeString.startsWith("DECIMALV3")) { + primitiveType = DorisDataType.DECIMAL; + String precisionAndScale = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + String[] split = precisionAndScale.split(","); + assert split.length == 2; + primitiveType.setDecimalPrecision(Integer.parseInt(split[0].trim())); + primitiveType.setDecimalScale(Integer.parseInt(split[1].trim())); + size = 3; + } else if (typeString.startsWith("DECIMAL")) { + primitiveType = DorisDataType.DECIMAL; + String precisionAndScale = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + String[] split = precisionAndScale.split(","); + assert split.length == 2; + primitiveType.setDecimalPrecision(Integer.parseInt(split[0].trim())); + primitiveType.setDecimalScale(Integer.parseInt(split[1].trim())); + size = 1; + } else if (typeString.startsWith("DATEV2")) { + primitiveType = DorisDataType.DATE; + size = 2; + } else if (typeString.startsWith("DATE")) { + primitiveType = DorisDataType.DATE; + size = 1; + } else if (typeString.startsWith("DATETIMEV2")) { + primitiveType = DorisDataType.DATETIME; + size = 2; + } else if (typeString.startsWith("DATETIME")) { + primitiveType = DorisDataType.DATETIME; + size = 1; + } else if (typeString.startsWith("CHAR") || typeString.startsWith("VARCHAR")) { + primitiveType = DorisDataType.VARCHAR; + String varcharLength = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + primitiveType.setVarcharLength(Integer.parseInt(varcharLength.trim())); + } else { + switch (typeString) { + case "LARGEINT": + primitiveType = DorisDataType.INT; + size = 16; + break; + case "BIGINT": + primitiveType = DorisDataType.INT; + size = 8; + break; + case "INT": + primitiveType = DorisDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = DorisDataType.INT; + size = 2; + break; + case "TINYINT": + primitiveType = DorisDataType.INT; + size = 1; + break; + case "DOUBLE": + primitiveType = DorisDataType.FLOAT; + size = 12; + break; + case "FLOAT": + primitiveType = DorisDataType.FLOAT; + size = 4; + break; + case "DECIMAL": + case "DECIMAL(*,*)": + primitiveType = DorisDataType.DECIMAL; + size = 1; + break; + case "DECIMALV3": + case "DECIMALV3(*,*)": + primitiveType = DorisDataType.DECIMAL; + size = 3; + break; + case "CHAR": + case "CHAR(*)": + case "VARCHAR": + case "VARCHAR(*)": + primitiveType = DorisDataType.VARCHAR; + break; + case "DATE": + primitiveType = DorisDataType.DATE; + size = 1; + break; + case "DATEV2": + primitiveType = DorisDataType.DATE; + size = 2; + break; + case "DATETIME": + primitiveType = DorisDataType.DATETIME; + size = 1; + break; + case "DATETIMEV2": + primitiveType = DorisDataType.DATETIME; + size = 2; + break; + case "BOOLEAN": + primitiveType = DorisDataType.BOOLEAN; + break; + // case "HLL": + // primitiveType = DorisDataType.HLL; + // break; + // case "BITMAP": + // primitiveType = DorisDataType.BITMAP; + // break; + case "NULL": + primitiveType = DorisDataType.NULL; + break; + default: + throw new AssertionError(typeString); + } + } + return new DorisCompositeDataType(primitiveType, size); + } + + public static class DorisTable extends AbstractRelationalTable { + + public DorisTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + public List getRandomNonEmptyInsertColumns() { + List columns = getColumns(); + List retColumns = new ArrayList<>(); + List remainColumns = new ArrayList<>(); + for (DorisColumn column : columns) { + if (!column.hasDefaultValue() && !column.isNullable) { + retColumns.add(column); + } else { + remainColumns.add(column); + } + } + if (retColumns.isEmpty()) { + retColumns.addAll(Randomly.nonEmptySubset(remainColumns)); + } else { + retColumns.addAll(Randomly.subset(remainColumns)); + } + return retColumns; + } + + } + + public static DorisSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + DorisTable t = new DorisTable(tableName, databaseColumns, isView); + for (DorisColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new DorisSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES")) { + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("DESCRIBE " + tableName)) { + while (rs.next()) { + String columnName = rs.getString("Field"); + String dataType = rs.getString("Type"); + String isNullString = rs.getString("Null"); + assert isNullString.contentEquals("Yes") || isNullString.contentEquals("No"); + boolean isNullable = isNullString.contentEquals("Yes"); + String isKeyString = rs.getString("Key"); + assert isKeyString.contentEquals("true") || isKeyString.contentEquals("false"); + boolean isKey = isKeyString.contentEquals("true"); + String defaultValue = rs.getString("Default"); + boolean hasDefaultValue = defaultValue != null; + DorisColumn c = new DorisColumn(columnName, getColumnType(dataType), isKey, isNullable, + DorisColumnAggrType.NULL, hasDefaultValue, defaultValue); + columns.add(c); + } + } + } + return columns; + } + +} diff --git a/src/sqlancer/doris/ast/DorisAggregateOperation.java b/src/sqlancer/doris/ast/DorisAggregateOperation.java new file mode 100644 index 000000000..de818c26d --- /dev/null +++ b/src/sqlancer/doris/ast/DorisAggregateOperation.java @@ -0,0 +1,39 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisAggregateOperation extends + NewFunctionNode implements DorisExpression { + + public DorisAggregateOperation(List args, DorisAggregateFunction func) { + super(DorisExprToNode.casts(args), func); + } + + public enum DorisAggregateFunction { + COLLECT_SET(1), MIN(1), STDDEV_SAMP(1), AVG(1), AVG_WEIGHTED(2), PERCENTILE(1), PERCENTILE_ARRAY(2), + HLL_UNION_AGG(1), TOPN(2), TOPN_ARRAY(2), TOPN_WEIGHTED(3), COUNT(1), SUM(1), MAX_BY(2), BITMAP_UNION(1), + GROUP_BITMAP_XOR(1), GROUP_BIT_ADD(1), GROUP_BIT_OR(1), GROUP_BIT_XOR(1), PERCENTILE_APPROX(2), STDDEV(1), + STDDEV_POP(1), GROUP_CONCAT(1), COLLECT_LIST(1), MIN_BY(2), MAX(1), ANY_VALUE(1), VAR_SAMP(1), VARIANCE_SAMP(1), + APPROX_COUNT_DISTINCT(1), VARIANCE(1), VAR_POP(1), VARIANCE_POP(1), GROUPING(1), GROUPING_ID(1); + // RETENTION(1), SEQUENCE_MATCH(1), SEQUENCE_COUNT(1), // TODO，not currently considered + + private int nrArgs; + + DorisAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static DorisAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } +} diff --git a/src/sqlancer/doris/ast/DorisBetweenOperation.java b/src/sqlancer/doris/ast/DorisBetweenOperation.java new file mode 100644 index 000000000..76488393b --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBetweenOperation.java @@ -0,0 +1,39 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisBetweenOperation extends NewBetweenOperatorNode implements DorisExpression { + public DorisBetweenOperation(DorisExpression left, DorisExpression middle, DorisExpression right, boolean isTrue) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(middle), DorisExprToNode.cast(right), isTrue); + } + + public DorisExpression getLeftExpr() { + return (DorisExpression) left; + } + + public DorisExpression getMiddleExpr() { + return (DorisExpression) middle; + } + + public DorisExpression getRightExpr() { + return (DorisExpression) right; + } + + @Override + public DorisConstant getExpectedValue() { + DorisBinaryComparisonOperation leftComparison = new DorisBinaryComparisonOperation(getMiddleExpr(), + getLeftExpr(), DorisBinaryComparisonOperation.DorisBinaryComparisonOperator.LESS_EQUALS); + DorisBinaryComparisonOperation rightComparison = new DorisBinaryComparisonOperation(getLeftExpr(), + getRightExpr(), DorisBinaryComparisonOperation.DorisBinaryComparisonOperator.LESS_EQUALS); + return new DorisBinaryLogicalOperation(leftComparison, rightComparison, + DorisBinaryLogicalOperation.DorisBinaryLogicalOperator.AND).getExpectedValue(); + } + + @Override + public DorisSchema.DorisDataType getExpectedType() { + return DorisSchema.DorisDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java new file mode 100644 index 000000000..244404569 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java @@ -0,0 +1,143 @@ +package sqlancer.doris.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisBinaryArithmeticOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisBinaryArithmeticOperation(DorisExpression left, DorisExpression right, BinaryOperatorNode.Operator op) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + } + + public enum DorisBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + ADDITION("+") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> l + r); + } + }, + SUBTRACTION("-") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + } + }, + MODULO("%") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + } + }, + CONCAT("||") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isBoolean() || !right.isBoolean()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> l == 1 || r == 1 ? 1.0 : 0.0); + } + }, + BIT_AND("&") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() & (int) r.doubleValue())); + } + }, + BIT_OR("|") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() | (int) r.doubleValue())); + } + }, + LSHIFT("<<") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() << (int) r.doubleValue())); + } + }, + RSHIFT(">>") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() >> (int) r.doubleValue())); + } + }; + + private final String textRepresentation; + + DorisBinaryArithmeticOperator(String text) { + textRepresentation = text; + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + public DorisConstant applyOperation(DorisConstant left, DorisConstant right, BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return DorisConstant.createNullConstant(); + } + double leftVal = left.cast(DorisDataType.FLOAT).asFloat(); + double rightVal = right.cast(DorisDataType.FLOAT).asFloat(); + return DorisConstant.createFloatConstant(op.apply(leftVal, rightVal)); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + + public DorisExpression getLeftExpr() { + return (DorisExpression) super.getLeft(); + } + + public DorisExpression getRightExpr() { + return (DorisExpression) super.getRight(); + } + + public DorisBinaryArithmeticOperator getOp() { + return (DorisBinaryArithmeticOperator) op; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftValue = getLeftExpr().getExpectedValue(); + DorisConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.FLOAT; + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java new file mode 100644 index 000000000..894d2ab9b --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java @@ -0,0 +1,120 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisBinaryComparisonOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisBinaryComparisonOperation(DorisExpression left, DorisExpression right, + DorisBinaryComparisonOperator op) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + } + + public DorisExpression getLeftExpression() { + return (DorisExpression) super.getLeft(); + } + + public DorisExpression getRightExpression() { + return (DorisExpression) super.getRight(); + } + + public DorisBinaryComparisonOperator getOp() { + return (DorisBinaryComparisonOperator) op; + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftExpectedValue = getLeftExpression().getExpectedValue(); + DorisConstant rightExpectedValue = getRightExpression().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + + public enum DorisBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return left.valueEquals(right); + } + }, + NOT_EQUALS("!=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean()) { + return DorisConstant.createBooleanConstant(!valueEquals.asBoolean()); + } + // maybe DorisNULLConstant or null object + return valueEquals; + } + }, + LESS("<") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return left.valueLessThan(right); + } + }, + LESS_EQUALS("<=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueLessThan = left.valueLessThan(right); + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean() && valueEquals.asBoolean()) { + return valueEquals; + } + return valueLessThan; + } + }, + GREATER(">") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueLessThan = left.valueLessThan(right); + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean() && valueEquals.asBoolean()) { + return DorisConstant.createBooleanConstant(false); + } + if (valueLessThan.isNull()) { + return valueLessThan; + } + return DorisConstant.createBooleanConstant(!valueLessThan.asBoolean()); + } + }, + GREATER_EQUALS(">=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueLessThan = left.valueLessThan(right); + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean() && valueEquals.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } + if (valueLessThan.isNull()) { + return valueLessThan; + } + return DorisConstant.createBooleanConstant(!valueLessThan.asBoolean()); + } + }; + + private final String textRepresentation; + + DorisBinaryComparisonOperator(String text) { + textRepresentation = text; + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java new file mode 100644 index 000000000..b62ccd61b --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java @@ -0,0 +1,130 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisBinaryLogicalOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisBinaryLogicalOperation(DorisExpression left, DorisExpression right, DorisBinaryLogicalOperator op) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + } + + public DorisExpression getLeftExpr() { + return (DorisExpression) super.getLeft(); + } + + public DorisExpression getRightExpr() { + return (DorisExpression) super.getRight(); + } + + public DorisBinaryLogicalOperator getOp() { + return (DorisBinaryLogicalOperator) op; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftValue = getLeftExpr().getExpectedValue(); + DorisConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + public enum DorisBinaryLogicalOperator implements BinaryOperatorNode.Operator { + /* + * null and false -> false null and true -> null null or false -> null null or true -> true + */ + AND("AND", "and") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant leftVal = left.cast(DorisDataType.BOOLEAN); + DorisConstant rightVal = right.cast(DorisDataType.BOOLEAN); + assert leftVal.isNull() || leftVal.isBoolean() : leftVal + "is not null or boolean"; + assert rightVal.isNull() || rightVal.isBoolean() : rightVal + "is not null or boolean"; + if (leftVal.isNull() && rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (leftVal.isNull()) { + if (!rightVal.asBoolean()) { + return DorisConstant.createBooleanConstant(false); + } else { + return DorisConstant.createNullConstant(); + } + } + if (rightVal.isNull()) { + if (!leftVal.asBoolean()) { + return DorisConstant.createBooleanConstant(false); + } else { + return DorisConstant.createNullConstant(); + } + } + if (leftVal.asBoolean() && right.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } + return DorisConstant.createBooleanConstant(false); + } + }, + OR("OR", "or") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant leftVal = left.cast(DorisDataType.BOOLEAN); + DorisConstant rightVal = right.cast(DorisDataType.BOOLEAN); + assert leftVal.isNull() || leftVal.isBoolean() : leftVal + "is not null or boolean"; + assert rightVal.isNull() || rightVal.isBoolean() : rightVal + "is not null or boolean"; + if (leftVal.isNull() && rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (leftVal.isNull()) { + if (rightVal.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } else { + return DorisConstant.createNullConstant(); + } + } + if (rightVal.isNull()) { + if (leftVal.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } else { + return DorisConstant.createNullConstant(); + } + } + if (leftVal.asBoolean() || right.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } + return DorisConstant.createBooleanConstant(false); + } + }; + + private final String[] textRepresentations; + + DorisBinaryLogicalOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + + public DorisBinaryLogicalOperator getRandomOp() { + return Randomly.fromOptions(values()); + } + + public static DorisBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + } + +} diff --git a/src/sqlancer/doris/ast/DorisCaseOperation.java b/src/sqlancer/doris/ast/DorisCaseOperation.java new file mode 100644 index 000000000..e852e36bd --- /dev/null +++ b/src/sqlancer/doris/ast/DorisCaseOperation.java @@ -0,0 +1,38 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.Node; + +public class DorisCaseOperation implements Node, DorisExpression { + + private final DorisExpression expr; + private final List conditions; + private final List thenClauses; + private final DorisExpression elseClause; + + public DorisCaseOperation(DorisExpression expr, List conditions, List thenClauses, + DorisExpression elseClause) { + this.expr = expr; + this.conditions = conditions; + this.thenClauses = thenClauses; + this.elseClause = elseClause; + } + + public DorisExpression getExpr() { + return expr; + } + + public List getConditions() { + return conditions; + } + + public List getThenClauses() { + return thenClauses; + } + + public DorisExpression getElseClause() { + return elseClause; + } + +} diff --git a/src/sqlancer/doris/ast/DorisCastOperation.java b/src/sqlancer/doris/ast/DorisCastOperation.java new file mode 100644 index 000000000..1aacdaf06 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisCastOperation.java @@ -0,0 +1,46 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisCastOperation implements Node, DorisExpression { + Node expr; + DorisDataType type; + + public DorisCastOperation(Node expr, DorisCompositeDataType type) { + this.expr = expr; + this.type = type.getPrimitiveDataType(); + } + + public DorisCastOperation(Node expr, DorisDataType type) { + this.expr = expr; + this.type = type; + } + + public Node getExpr() { + return expr; + } + + public DorisExpression getExpression() { + return (DorisExpression) expr; + } + + public DorisDataType getType() { + return type; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type); + } + + @Override + public DorisDataType getExpectedType() { + return type; + } +} diff --git a/src/sqlancer/doris/ast/DorisColumnValue.java b/src/sqlancer/doris/ast/DorisColumnValue.java new file mode 100644 index 000000000..fe3ec1448 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisColumnValue.java @@ -0,0 +1,53 @@ +package sqlancer.doris.ast; + +import java.util.Objects; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisColumnValue extends ColumnReferenceNode implements DorisExpression { + + private final DorisConstant expectedValue; + + public DorisColumnValue(DorisColumn column, DorisConstant value) { + super(column); + this.expectedValue = value; + } + + @Override + public DorisConstant getExpectedValue() { + return expectedValue; + } + + @Override + public DorisDataType getExpectedType() { + return getColumn().getType().getPrimitiveDataType(); + } + + public static DorisColumnValue create(DorisColumn column, DorisConstant value) { + return new DorisColumnValue(column, value); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DorisColumnValue that = (DorisColumnValue) o; + if (!this.getColumn().getName().equals(that.getColumn().getName())) { + return false; + } + return Objects.equals(expectedValue, that.expectedValue); + } + + @Override + public int hashCode() { + String nameAndValue = this.getColumn().getName(); + nameAndValue += expectedValue == null ? "NULL" : expectedValue.toString(); + return Objects.hash(nameAndValue); + } +} diff --git a/src/sqlancer/doris/ast/DorisConstant.java b/src/sqlancer/doris/ast/DorisConstant.java new file mode 100644 index 000000000..37374a4d7 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisConstant.java @@ -0,0 +1,681 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.utils.DorisNumberUtils; + +public abstract class DorisConstant implements Node, DorisExpression { + + private DorisConstant() { + } + + public boolean isNull() { + return false; + } + + public boolean isInt() { + return false; + } + + public boolean isBoolean() { + return false; + } + + public boolean isString() { + return false; + } + + public boolean isFloat() { + return false; + } + + public boolean isNum() { + // for INT, FLOAT, BOOLEAN + return false; + } + + public boolean isDate() { + return false; + } + + public boolean isDatetime() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asFloat() { + throw new UnsupportedOperationException(this.toString()); + } + + public abstract DorisConstant cast(DorisDataType dataType); + + public abstract DorisConstant valueEquals(DorisConstant rightVal); + + public abstract DorisConstant valueLessThan(DorisConstant rightVal); + + public static class DorisNullConstant extends DorisConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + return DorisConstant.createNullConstant(); + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + return DorisConstant.createNullConstant(); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + return DorisConstant.createNullConstant(); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.NULL; + } + } + + public static class DorisIntConstant extends DorisConstant { + + private final long value; + + public DorisIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public boolean isNum() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + return this; + case FLOAT: + case DECIMAL: + return new DorisFloatConstant(value); + case VARCHAR: + return new DorisTextConstant(String.valueOf(value)); + case BOOLEAN: + return new DorisBooleanConstant(value != 0); + default: + return DorisConstant.createNullConstant(); + } + } + + @Override + public long asInt() { + return value; + } + + @Override + public boolean asBoolean() { + return value != 0; + } + + @Override + public double asFloat() { + return value; + } + + @Override + public String asString() { + return String.valueOf(value); + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isNum()) { + return DorisConstant.createBooleanConstant(value == rightVal.asFloat()); + } + + throw new AssertionError(rightVal); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isNum()) { + return DorisConstant.createBooleanConstant(value < rightVal.asFloat()); + } + + throw new AssertionError(rightVal); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.INT; + } + + } + + public static class DorisFloatConstant extends DorisConstant { + + private final double value; + + public DorisFloatConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public boolean isFloat() { + return true; + } + + @Override + public boolean isNum() { + return true; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "3.40282347e+38"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-3.40282347e+38"; + } + + return String.valueOf(value); + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + return new DorisIntConstant((long) value); + case FLOAT: + case DECIMAL: + return this; + case VARCHAR: + return new DorisTextConstant(String.valueOf(value)); + case BOOLEAN: + return new DorisBooleanConstant(value >= 1); + default: + return null; + } + } + + @Override + public double asFloat() { + return value; + } + + @Override + public String asString() { + return toString(); + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DorisConstant.createBooleanConstant(value == rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DorisConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DorisConstant.createBooleanConstant(value < rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DorisConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + + } + + public static class DorisTextConstant extends DorisConstant { + + private final String value; + + public DorisTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; + } + + @Override + public String asString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + // Currently only supports conversion of int text to int, not float text, see + // https://github.com/apache/doris/issues/18227 + if (DorisNumberUtils.isNumber(value)) { + long val = (long) Double.parseDouble(value); + return new DorisIntConstant(val); + } + return new DorisNullConstant(); + case FLOAT: + case DECIMAL: + if (DorisNumberUtils.isNumber(value)) { + return new DorisFloatConstant(Double.parseDouble(value)); + } + return new DorisNullConstant(); + case DATE: + if (DorisNumberUtils.isDate(value)) { + return new DorisDateConstant(value); + } + return new DorisNullConstant(); + case DATETIME: + if (DorisNumberUtils.isDatetime(value)) { + return new DorisDatetimeConstant(value); + } + return new DorisNullConstant(); + case VARCHAR: + return this; + case BOOLEAN: + if ("false".contentEquals(value.toLowerCase())) { + return new DorisBooleanConstant(false); + } + if ("true".contentEquals(value.toLowerCase())) { + return new DorisBooleanConstant(true); + } + return new DorisNullConstant(); + default: + return new DorisNullConstant(); + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isString()) { + return DorisConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } + if (DorisNumberUtils.isNumber(value) && rightVal.isNum()) { + return DorisConstant.createBooleanConstant(Double.parseDouble(value) == rightVal.asFloat()); + } + // Doris currently does not support judgment between string types and other types, such date, datetime + return DorisConstant.createBooleanConstant(false); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isString()) { + return DorisConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } + if (DorisNumberUtils.isNumber(value) && rightVal.isNum()) { + return DorisConstant.createBooleanConstant(Double.parseDouble(value) < rightVal.asFloat()); + } + // Doris currently does not support judgment between string types and other types, such date, datetime + return DorisConstant.createBooleanConstant(false); + } + + } + + public static class DorisDateConstant extends DorisConstant { + + public String textRepr; + + public DorisDateConstant(long val) { + textRepr = DorisNumberUtils.timestampToDateText(val); + } + + public DorisDateConstant(String textRepr) { + this.textRepr = textRepr; + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + @Override + public String asString() { + return textRepr; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case VARCHAR: + return new DorisTextConstant(textRepr); + case DATE: + return this; + case DATETIME: + return new DorisDatetimeConstant(DorisNumberUtils.dateTextToDatetimeText(textRepr)); + default: + return new DorisNullConstant(); + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && rightVal.asString().contentEquals("CURRENT_TIMESTAMP")) { + return DorisConstant.createBooleanConstant(false); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant.createBooleanConstant(DorisNumberUtils.dateEqual(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && rightVal.asString().contentEquals("CURRENT_TIMESTAMP")) { + return DorisConstant.createBooleanConstant( + DorisNumberUtils.dateLessThan(textRepr, DorisNumberUtils.getCurrentTimeText())); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant + .createBooleanConstant(DorisNumberUtils.dateLessThan(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public boolean isDate() { + return true; + } + } + + public static class DorisDatetimeConstant extends DorisConstant { + + public String textRepr; + + public DorisDatetimeConstant(long val) { + textRepr = DorisNumberUtils.timestampToDatetimeText(val); + } + + public DorisDatetimeConstant(String textRepr) { + this.textRepr = textRepr; + } + + public DorisDatetimeConstant() { + textRepr = "CURRENT_TIMESTAMP"; + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + @Override + public String asString() { + return textRepr; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case VARCHAR: + return new DorisTextConstant(textRepr); + case DATE: + return new DorisDatetimeConstant(DorisNumberUtils.datetimeTextToDateText(textRepr)); + case DATETIME: + return this; + default: + return new DorisNullConstant(); + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && (rightVal.asString().contentEquals("CURRENT_TIMESTAMP") + || textRepr.contentEquals("CURRENT_TIMESTAMP"))) { + boolean isEq = rightVal.asString().contentEquals("CURRENT_TIMESTAMP") + && textRepr.contentEquals("CURRENT_TIMESTAMP"); + return DorisConstant.createBooleanConstant(isEq); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant + .createBooleanConstant(DorisNumberUtils.datetimeEqual(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && (rightVal.asString().contentEquals("CURRENT_TIMESTAMP") + || textRepr.contentEquals("CURRENT_TIMESTAMP"))) { + String leftText = textRepr; + String rightText = rightVal.asString(); + if (leftText.contentEquals(rightText)) { + return DorisConstant.createBooleanConstant(false); + } + if (leftText.contentEquals("CURRENT_TIMESTAMP")) { + leftText = DorisNumberUtils.getCurrentTimeText(); + } + if (rightText.contentEquals("CURRENT_TIMESTAMP")) { + rightText = DorisNumberUtils.getCurrentTimeText(); + } + boolean lessThan = DorisNumberUtils.dateLessThan(leftText, rightText); + return DorisConstant.createBooleanConstant(lessThan); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant + .createBooleanConstant(DorisNumberUtils.datetimeLessThan(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public boolean isDatetime() { + return true; + } + + } + + public static class DorisBooleanConstant extends DorisConstant { + + private final boolean value; + + public DorisBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public String asString() { + return toString(); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public boolean isNum() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + return new DorisIntConstant(value ? 1 : 0); + case FLOAT: + case DECIMAL: + return new DorisFloatConstant(value ? 1 : 0); + case BOOLEAN: + return this; + case VARCHAR: + return new DorisTextConstant(value ? "1" : "0"); + default: + return null; + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isBoolean()) { + return DorisConstant.createBooleanConstant(value == rightVal.asBoolean()); + } + if (rightVal.isNum() || rightVal.isString() && DorisNumberUtils.isNumber(rightVal.asString())) { + return DorisConstant.createBooleanConstant((value ? 1 : 0) == rightVal.asFloat()); + } + throw new AssertionError(rightVal); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isBoolean()) { + return DorisConstant.createBooleanConstant(value == rightVal.asBoolean()); + } + if (rightVal.isNum() || rightVal.isString() && DorisNumberUtils.isNumber(rightVal.asString())) { + return DorisConstant.createBooleanConstant((value ? 1 : 0) == rightVal.asFloat()); + } + throw new AssertionError(rightVal); + } + } + + public static DorisConstant createStringConstant(String text) { + return new DorisTextConstant(text); + } + + public static DorisConstant createFloatConstant(double val) { + return new DorisFloatConstant(val); + } + + public static DorisConstant createIntConstant(long val) { + return new DorisIntConstant(val); + } + + public static DorisConstant createNullConstant() { + return new DorisNullConstant(); + } + + public static DorisConstant createBooleanConstant(boolean val) { + return new DorisBooleanConstant(val); + } + + public static DorisConstant createDateConstant(long integer) { + return new DorisDateConstant(integer); + } + + public static DorisConstant createDateConstant(String date) { + return new DorisDateConstant(date); + } + + public static DorisConstant createDatetimeConstant(long integer) { + return new DorisDatetimeConstant(integer); + } + + public static DorisConstant createDatetimeConstant(String datetime) { + return new DorisDatetimeConstant(datetime); + } + + public static DorisConstant createDatetimeConstant() { + // use CURRENT_TIMESTAMP + return new DorisDatetimeConstant(); + } + +} diff --git a/src/sqlancer/doris/ast/DorisExpression.java b/src/sqlancer/doris/ast/DorisExpression.java new file mode 100644 index 000000000..559ecd8f3 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisExpression.java @@ -0,0 +1,13 @@ +package sqlancer.doris.ast; + +import sqlancer.doris.DorisSchema; + +public interface DorisExpression { + default DorisSchema.DorisDataType getExpectedType() { + return null; + } + + default DorisConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/doris/ast/DorisFunctionOperation.java b/src/sqlancer/doris/ast/DorisFunctionOperation.java new file mode 100644 index 000000000..0a75892d0 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisFunctionOperation.java @@ -0,0 +1,282 @@ +package sqlancer.doris.ast; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.gen.DorisNewExpressionGenerator; + +public class DorisFunctionOperation implements Node, DorisExpression { + + private DorisFunction function; + private List args; + + // https://doris.apache.org/zh-CN/docs/dev/summary/basic-summary + public enum DorisFunction { + + // Array functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/array-functions/array + // Skip now + + // Date functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/date-time-functions/convert_tz/ + CONVERT_TZ(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CURDATE(false, DorisDataType.DATE), CURRENT_DATE(false, DorisDataType.DATE), + CURTIME(false, DorisDataType.VARCHAR), CURRENT_TIME(false, DorisDataType.VARCHAR), + CURRENT_TIMESTAMP(false, DorisDataType.DATETIME), LOCALTIME(false, DorisDataType.DATETIME), + LOCALTIMESTAMP(false, DorisDataType.DATETIME), NOW(false, DorisDataType.DATETIME), + YEAR(false, DorisDataType.INT, DorisDataType.DATETIME), + QUARTER(false, DorisDataType.INT, DorisDataType.DATETIME), + MONTH(false, DorisDataType.INT, DorisDataType.DATETIME), DAY(false, DorisDataType.INT, DorisDataType.DATETIME), + DAYOFYEAR(false, DorisDataType.INT, DorisDataType.DATETIME), + DAYOFMONTH(false, DorisDataType.INT, DorisDataType.DATETIME), + DAYOFWEEK(false, DorisDataType.INT, DorisDataType.DATETIME), WEEK(false, DorisDataType.INT, DorisDataType.DATE), + WEEKDAY(false, DorisDataType.INT, DorisDataType.DATE), + WEEKOFYEAR(false, DorisDataType.INT, DorisDataType.DATETIME), + YEARWEEK(false, DorisDataType.INT, DorisDataType.DATE), + DAYNAME(false, DorisDataType.VARCHAR, DorisDataType.DATETIME), + MONTHNAME(false, DorisDataType.VARCHAR, DorisDataType.DATETIME), + HOUR(false, DorisDataType.INT, DorisDataType.DATETIME), + MINUTE(false, DorisDataType.INT, DorisDataType.DATETIME), + SECOND(false, DorisDataType.INT, DorisDataType.DATETIME), + FROM_DAYS(false, DorisDataType.DATE, DorisDataType.INT), + LAST_DAYS(false, DorisDataType.DATE, DorisDataType.DATETIME), + TO_MONDAY(false, DorisDataType.DATE, DorisDataType.DATETIME), + FROM_UNIXTIME(false, DorisDataType.DATETIME, DorisDataType.INT), + UNIX_TIMESTAMP(false, DorisDataType.INT, DorisDataType.DATETIME), UTC_TIMESTAMP(false, DorisDataType.DATETIME), + TO_DATE(false, DorisDataType.DATE, DorisDataType.DATETIME), + TO_DAYS(false, DorisDataType.INT, DorisDataType.DATETIME), + TIME_TO_SEC(false, DorisDataType.INT, DorisDataType.DATETIME), + // EXTRACT(1), // select extract(year from '2022-09-22 17:01:30') as year, currently not considered + MAKEDATE(false, DorisDataType.DATE, DorisDataType.INT, DorisDataType.INT), + STR_TO_DATE(false, DorisDataType.DATETIME, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + TIME_ROUND(false, DorisDataType.DATETIME, DorisDataType.DATETIME), + TIME_DIFF(false, DorisDataType.VARCHAR, DorisDataType.DATETIME, DorisDataType.DATETIME), + TIMESTAMPADD(false, DorisDataType.DATETIME, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.DATETIME), + TIMESTAMPDIFF(false, DorisDataType.VARCHAR, DorisDataType.DATETIME, DorisDataType.DATETIME), + DATE_ADD(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATE_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATE_TRUNC(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATE_FORMAT(false, DorisDataType.VARCHAR, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATEDIFF(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.DATETIME), + // MICROSECONDS_ADD(false), + MINUTES_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + MINUTES_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + MINUTES_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + SECONDS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + SECONDS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + SECONDS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + HOURS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + HOURS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + HOURS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + DAYS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + DAYS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + DAYS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + WEEKS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + WEEKS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + WEEKS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + MONTHS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + MONTHS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + MONTHS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + YEARS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + YEARS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + YEARS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + + // GIS functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/spatial-functions/st_x + // Skip now + + // String functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/string-functions/to_base64 + TO_BASE64(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + FROM_BASE64(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + ASCII(false, DorisDataType.INT, DorisDataType.VARCHAR), LENGTH(false, DorisDataType.INT, DorisDataType.VARCHAR), + BIT_LENGTH(false, DorisDataType.INT, DorisDataType.VARCHAR), + CHAR_LENGTH(false, DorisDataType.INT, DorisDataType.VARCHAR), + LPAD(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.VARCHAR), + RPAD(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.VARCHAR), + LOWER(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LCASE(false, DorisDataType.INT, DorisDataType.VARCHAR), + UPPER(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + UCASE(false, DorisDataType.INT, DorisDataType.VARCHAR), + INITCAP(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + REPEAT(false, DorisDataType.VARCHAR, DorisDataType.INT), + REVERSE(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CHAR(true, DorisDataType.VARCHAR, DorisDataType.INT), + CONCAT(true, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CONCAT_WS(true, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + SUBSTR(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.INT), + SUBSTRING(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + SUB_REPLACE(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + APPEND_TRAILING_CHAR_IF_ABSENT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + ENDS_WITH(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + STARTS_WITH(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + TRIM(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LTRIM(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + RTRIM(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + NULL_OR_EMPTY(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR), + NOT_NULL_OR_EMPTY(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR), + HEX(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + UNHEX(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + ELT(true, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.VARCHAR), + INSTR(false, DorisDataType.INT, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LOCATE(false, DorisDataType.INT, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + // FIELD(1, true), + FIND_IN_SET(false, DorisDataType.INT, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + REPLACE(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LEFT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + RIGHT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + STRLEFT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + STRRIGHT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + SPLIT_PART(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + // SPLIT_BY_STRING(2), + SUBSTRING_INDEX(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + MONEY_FORMAT(false, DorisDataType.VARCHAR, DorisDataType.DECIMAL), + PARSE_URL(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CONVERT_TO(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + EXTRACT_URL_PARAMETER(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + UUID(false, DorisDataType.VARCHAR), SPACE(false, DorisDataType.VARCHAR, DorisDataType.INT), + // SLEEP(1), + ESQUERY(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + MASK(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + MASK_FIRST_N(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + MASK_LAST_N(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + // MULTI_SEARCH_ALL_POSITIONS(2), + // MULTI_MATCH_ANY(2), + + // BITMAP functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/bitmap-functions/to_bitmap + // skip now + + // Bitwise functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/bitwise-functions/bitand + BITAND(false, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BITOR(false, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BITXOR(false, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BITNOT(false, DorisDataType.INT, DorisDataType.INT), + + // condition funtions + // case(), + COALESCE(true, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + IF(false, DorisDataType.VARCHAR, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + IFNULL(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + NVL(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + NULLIF(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + + // JSON Functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/json-functions/jsonb_parse + // skip now + + // Hash functions, + // https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/hash-functions/murmur_hash3_32 + MURMUR_HASH3_32(true, DorisDataType.INT, DorisDataType.VARCHAR), + MURMUR_HASH3_64(true, DorisDataType.INT, DorisDataType.VARCHAR), + + // HLL functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/hll-functions/hll_cardinality + // skip now + + // Math functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/math-functions/conv + CONV(false, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BIN(false, DorisDataType.VARCHAR, DorisDataType.INT), SIN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + COS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), TAN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + ASIN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), ACOS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + ATAN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), E(false, DorisDataType.FLOAT), + PI(false, DorisDataType.FLOAT), EXP(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + LOG(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT), + LOG2(false, DorisDataType.FLOAT, DorisDataType.FLOAT), LN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + LOG10(false, DorisDataType.FLOAT, DorisDataType.FLOAT), CEIL(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + FLOOR(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + PMOD(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT), + ROUND(false, DorisDataType.INT, DorisDataType.FLOAT), + ROUND_BANKERS(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.INT), + TRUNCATE(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.INT), + ABS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), SQRT(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + CBRT(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + POW(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT), + DEGREES(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + RADIANS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), SIGN(false, DorisDataType.INT, DorisDataType.FLOAT), + POSTIVE(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + NEGATIVE(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + GREATEST(true, DorisDataType.FLOAT, DorisDataType.FLOAT), LEAST(true, DorisDataType.FLOAT, DorisDataType.FLOAT), + RANDOM(false, DorisDataType.FLOAT), MOD(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT); + + // encrypt-digest-functions, + // https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/encrypt-digest-functions/aes + // skip now + + private boolean isVariadic; // If isVALid is true, then treat the last argumentTypes as an infinite type + private DorisDataType returnType; + private DorisDataType[] argumentTypes; + private String functionName; + + DorisFunction(String functionName, boolean isVariadic, DorisDataType returnType, + DorisDataType... argumentTypes) { + this.functionName = functionName; + this.isVariadic = isVariadic; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + DorisFunction(boolean isVariadic, DorisDataType returnType, DorisDataType... argumentTypes) { + this.functionName = toString(); + this.isVariadic = isVariadic; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + DorisFunction(boolean isVariadic, DorisDataType returnType) { + this.functionName = toString(); + this.isVariadic = isVariadic; + this.returnType = returnType; + this.argumentTypes = null; + } + + public String getFunctionName() { + return functionName; + } + + public static DorisFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public boolean isVariadic() { + return isVariadic; + } + + public boolean isCompatibleWithReturnType(DorisDataType returnType) { + return this.returnType == returnType; + } + + public DorisDataType[] getArgumentTypes() { + if (argumentTypes == null) { + return null; + } + return argumentTypes.clone(); + } + + public DorisFunctionOperation getCall(DorisDataType returnType, DorisNewExpressionGenerator gen, int depth) { + List arguments = new ArrayList<>(); + if (getArgumentTypes() != null) { + Stream.of(getArgumentTypes()).forEach(arg -> arguments.add(gen.generateExpression(arg, depth + 1))); + } + return new DorisFunctionOperation(this, arguments); + } + + public static List getFunctionsCompatibleWith(DorisDataType returnType) { + return Stream.of(values()).filter(f -> f.isCompatibleWithReturnType(returnType)) + .collect(Collectors.toList()); + } + + } + + public DorisFunctionOperation(DorisFunction function, List args) { + this.function = function; + this.args = args; + } + + public List getArgs() { + return args; + } + + public DorisFunction getFunction() { + return function; + } + +} diff --git a/src/sqlancer/doris/ast/DorisInOperation.java b/src/sqlancer/doris/ast/DorisInOperation.java new file mode 100644 index 000000000..31dcbe1ec --- /dev/null +++ b/src/sqlancer/doris/ast/DorisInOperation.java @@ -0,0 +1,53 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisInOperation extends NewInOperatorNode implements DorisExpression { + + private final DorisExpression leftExpr; + private final List rightExpr; + + public DorisInOperation(DorisExpression left, List right, boolean isNegated) { + super(DorisExprToNode.cast(left), DorisExprToNode.casts(right), isNegated); + this.leftExpr = left; + this.rightExpr = right; + } + + @Override + public DorisSchema.DorisDataType getExpectedType() { + return DorisSchema.DorisDataType.BOOLEAN; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftValue = leftExpr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return DorisConstant.createNullConstant(); + } + boolean containNull = false; + for (DorisExpression expr : rightExpr) { + DorisConstant rightValue = expr.getExpectedValue(); + if (rightValue == null) { + return null; + } + if (rightValue.isNull()) { + containNull = true; + } else if (rightValue.valueEquals(leftValue).isBoolean() && rightValue.valueEquals(leftValue).asBoolean()) { + return DorisConstant.createBooleanConstant(!isNegated()); + } + } + + if (containNull) { + return DorisConstant.createNullConstant(); + } + // should return false when not considering isNegated op + return DorisConstant.createBooleanConstant(isNegated()); + } +} diff --git a/src/sqlancer/doris/ast/DorisJoin.java b/src/sqlancer/doris/ast/DorisJoin.java new file mode 100644 index 000000000..01274449f --- /dev/null +++ b/src/sqlancer/doris/ast/DorisJoin.java @@ -0,0 +1,108 @@ +package sqlancer.doris.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + + public enum JoinType { + INNER, STRAIGHT, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public DorisJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + public static List> getJoins(List> tableList, + DorisGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + DorisNewExpressionGenerator joinGen = new DorisNewExpressionGenerator(globalState).setColumns(columns); + switch (DorisJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(DorisJoin.createInnerJoin(leftTable, rightTable, + DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + break; + case STRAIGHT: + joinExpressions.add(DorisJoin.createStraightJoin(leftTable, rightTable, + DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + break; + case LEFT: + joinExpressions.add(DorisJoin.createLeftOuterJoin(leftTable, rightTable, + DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + break; + case RIGHT: + joinExpressions.add(DorisJoin.createRightOuterJoin(leftTable, rightTable, + DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static DorisJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DorisJoin(left, right, JoinType.INNER, predicate); + } + + public static DorisJoin createStraightJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DorisJoin(left, right, JoinType.STRAIGHT, predicate); + } + + public static DorisJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DorisJoin(left, right, JoinType.RIGHT, predicate); + } + + public static DorisJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DorisJoin(left, right, JoinType.LEFT, predicate); + } +} diff --git a/src/sqlancer/doris/ast/DorisLikeOperation.java b/src/sqlancer/doris/ast/DorisLikeOperation.java new file mode 100644 index 000000000..a3251ca7f --- /dev/null +++ b/src/sqlancer/doris/ast/DorisLikeOperation.java @@ -0,0 +1,85 @@ +package sqlancer.doris.ast; + +import sqlancer.LikeImplementationHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisLikeOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisLikeOperation(DorisExpression left, DorisExpression right, DorisLikeOperator op) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + public DorisExpression getLeftExpr() { + return (DorisExpression) super.getLeft(); + } + + public DorisExpression getRightExpr() { + return (DorisExpression) super.getRight(); + } + + public DorisLikeOperator getOp() { + return (DorisLikeOperator) op; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftVal = getLeftExpr().getExpectedValue(); + DorisConstant rightVal = getRightExpr().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } + return getOp().apply(leftVal, rightVal); + } + + public enum DorisLikeOperator implements BinaryOperatorNode.Operator { + LIKE_OPERATOR("LIKE", "like") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (left == null || right == null) { + return null; + } + if (left.isNull() || right.isNull()) { + return DorisConstant.createNullConstant(); + } + boolean result = LikeImplementationHelper.match(left.asString(), right.asString(), 0, 0, true); + return DorisConstant.createBooleanConstant(result); + } + }, + NOT_LIKE("NOT LIKE", "not like") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (left == null || right == null) { + return null; + } + if (left.isNull() || right.isNull()) { + return DorisConstant.createNullConstant(); + } + boolean result = LikeImplementationHelper.match(left.asString(), right.asString(), 0, 0, true); + return DorisConstant.createBooleanConstant(!result); + } + }; + + private final String[] textRepresentations; + + DorisLikeOperator(String... text) { + textRepresentations = text.clone(); + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + @Override + public String getTextRepresentation() { + return " " + Randomly.fromOptions(textRepresentations) + " "; + } + } + +} diff --git a/src/sqlancer/doris/ast/DorisOrderByTerm.java b/src/sqlancer/doris/ast/DorisOrderByTerm.java new file mode 100644 index 000000000..6207f4af2 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisOrderByTerm.java @@ -0,0 +1,10 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.Node; + +public class DorisOrderByTerm extends NewOrderingTerm implements DorisExpression { + public DorisOrderByTerm(Node expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/doris/ast/DorisSelect.java b/src/sqlancer/doris/ast/DorisSelect.java new file mode 100644 index 000000000..283d474cc --- /dev/null +++ b/src/sqlancer/doris/ast/DorisSelect.java @@ -0,0 +1,41 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class DorisSelect extends SelectBase> implements Node { + + public enum DorisSelectDistinctType { + + ALL, DISTINCT, DISTINCTROW, NULL; + + public static DorisSelectDistinctType getRandomWithoutNull() { + DorisSelectDistinctType sft; + do { + sft = Randomly.fromOptions(values()); + } while (sft == DorisSelectDistinctType.NULL); + return sft; + } + } + + private DorisSelectDistinctType selectDistinctType = DorisSelectDistinctType.ALL; + + public void setDistinct(boolean isDistinct) { + if (isDistinct) { + this.selectDistinctType = DorisSelectDistinctType.DISTINCT; + } else { + this.selectDistinctType = DorisSelectDistinctType.ALL; + } + } + + public void setDistinct(DorisSelectDistinctType type) { + this.selectDistinctType = type; + } + + public boolean isDistinct() { + return this.selectDistinctType == DorisSelectDistinctType.DISTINCT + || this.selectDistinctType == DorisSelectDistinctType.DISTINCTROW; + } + +} diff --git a/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java new file mode 100644 index 000000000..9a34e2c54 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java @@ -0,0 +1,87 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements DorisExpression { + + public DorisUnaryPostfixOperation(DorisExpression expr, DorisUnaryPostfixOperator op) { + super(DorisExprToNode.cast(expr), op); + } + + public DorisExpression getExpression() { + return (DorisExpression) getExpr(); + } + + public DorisUnaryPostfixOperator getOp() { + return (DorisUnaryPostfixOperator) op; + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } + + public enum DorisUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public DorisDataType[] getInputDataTypes() { + return DorisDataType.values(); + } + + @Override + public DorisConstant apply(DorisConstant value) { + return DorisConstant.createBooleanConstant(value.isNull()); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public DorisDataType[] getInputDataTypes() { + return DorisDataType.values(); + } + + @Override + public DorisConstant apply(DorisConstant value) { + return DorisConstant.createBooleanConstant(!value.isNull()); + } + }; + + private final String textRepresentations; + + DorisUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static DorisUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract DorisDataType[] getInputDataTypes(); + + public abstract DorisConstant apply(DorisConstant value); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + +} diff --git a/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java new file mode 100644 index 000000000..e7f7d91b0 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java @@ -0,0 +1,112 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements DorisExpression { + + public DorisUnaryPrefixOperation(DorisExpression expr, DorisUnaryPrefixOperator op) { + super(DorisExprToNode.cast(expr), op); + } + + public DorisExpression getExpression() { + return (DorisExpression) getExpr(); + } + + public DorisUnaryPrefixOperator getOp() { + return (DorisUnaryPrefixOperator) op; + } + + @Override + public DorisDataType getExpectedType() { + return getOp().getExpressionType(getExpression()); + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } + + public enum DorisUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", DorisDataType.BOOLEAN, DorisDataType.INT) { + @Override + public DorisDataType getExpressionType(DorisExpression expr) { + return DorisDataType.BOOLEAN; + } + + @Override + protected DorisConstant apply(DorisConstant value) { + if (value.isNull()) { + return DorisConstant.createNullConstant(); + } else { + return DorisConstant.createBooleanConstant(!value.cast(DorisDataType.BOOLEAN).asBoolean()); + } + } + }, + + UNARY_PLUS("+", DorisDataType.INT) { + @Override + public DorisDataType getExpressionType(DorisExpression expr) { + return expr.getExpectedType(); + } + + @Override + protected DorisConstant apply(DorisConstant value) { + return value; + } + }, + UNARY_MINUS("-", DorisDataType.INT) { + @Override + public DorisDataType getExpressionType(DorisExpression expr) { + return expr.getExpectedType(); + } + + @Override + protected DorisConstant apply(DorisConstant value) { + if (value.isNull()) { + return DorisConstant.createNullConstant(); + } + try { + if (value.isInt()) { + return DorisConstant.createIntConstant(-value.asInt()); + } + if (value.isFloat()) { + return DorisConstant.createFloatConstant(-value.asFloat()); + } + return null; + } catch (UnsupportedOperationException e) { + return null; + } + } + }; + + private String textRepresentation; + private DorisDataType[] dataTypes; + + DorisUnaryPrefixOperator(String textRepresentation, DorisDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract DorisDataType getExpressionType(DorisExpression expr); + + public DorisDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); + } + + protected abstract DorisConstant apply(DorisConstant value); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + } + +} diff --git a/src/sqlancer/doris/gen/DorisAlterTableGenerator.java b/src/sqlancer/doris/gen/DorisAlterTableGenerator.java new file mode 100644 index 000000000..bfac074fc --- /dev/null +++ b/src/sqlancer/doris/gen/DorisAlterTableGenerator.java @@ -0,0 +1,50 @@ +package sqlancer.doris.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisTable; + +public final class DorisAlterTableGenerator { + + private DorisAlterTableGenerator() { + } + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(DorisCompositeDataType.getRandomWithoutNull().toString()); + break; + case ALTER_COLUMN: + sb.append("MODIFY COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" "); + sb.append(DorisCompositeDataType.getRandomWithoutNull().toString()); + break; + case DROP_COLUMN: + sb.append("DROP COLUMN "); + sb.append(table.getRandomColumn().getName()); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisDeleteGenerator.java b/src/sqlancer/doris/gen/DorisDeleteGenerator.java new file mode 100644 index 000000000..27f369aec --- /dev/null +++ b/src/sqlancer/doris/gen/DorisDeleteGenerator.java @@ -0,0 +1,31 @@ +package sqlancer.doris.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public final class DorisDeleteGenerator { + + private DorisDeleteGenerator() { + } + + public static SQLQueryAdapter generate(DorisGlobalState globalState) { + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + .setColumns(table.getColumns()).generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + DorisErrors.addExpressionErrors(errors); + } + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/doris/gen/DorisDropTableGenerator.java b/src/sqlancer/doris/gen/DorisDropTableGenerator.java new file mode 100644 index 000000000..c8bbc67d4 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisDropTableGenerator.java @@ -0,0 +1,28 @@ +package sqlancer.doris.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; + +public final class DorisDropTableGenerator { + + private DorisDropTableGenerator() { + } + + public static SQLQueryAdapter dropTable(DorisGlobalState globalState) { + if (globalState.getSchema().getTables(t -> !t.isView()).size() <= 1) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> !t.isView()).getName()); + if (Randomly.getBoolean()) { + sb.append(" FORCE "); + } + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisDropViewGenerator.java b/src/sqlancer/doris/gen/DorisDropViewGenerator.java new file mode 100644 index 000000000..81c1629c2 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisDropViewGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.doris.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; + +public final class DorisDropViewGenerator { + + private DorisDropViewGenerator() { + } + + public static SQLQueryAdapter dropView(DorisGlobalState globalState) { + if (globalState.getSchema().getTables(t -> t.isView()).size() == 0) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP VIEW "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + // TODO: DROP VIEW syntax: DROP MATERIALIZED VIEW [IF EXISTS] mv_name ON table_name; + // should record original table name in view table + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> t.isView()).getName()); + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisIndexGenerator.java b/src/sqlancer/doris/gen/DorisIndexGenerator.java new file mode 100644 index 000000000..5e56bb192 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisIndexGenerator.java @@ -0,0 +1,46 @@ +package sqlancer.doris.gen; + +import java.sql.SQLException; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; + +public final class DorisIndexGenerator { + + private DorisIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) throws SQLException { + if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } + ExpectedErrors errors = new ExpectedErrors(); + + DorisTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); + String indexName = globalState.getSchema().getFreeIndexName(); + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("INDEX "); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(indexName); + sb.append(" ON "); + sb.append(randomTable.getName()); + sb.append("("); + int nr = 1; // Doris Only support CREATE_INDEX on single column and index type is BITMAP; + List subset = Randomly.extractNrRandomColumns(randomTable.getColumns(), nr); + sb.append(subset.get(0).getName()); + sb.append(") "); + if (Randomly.getBoolean()) { + sb.append("USING BITMAP "); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java new file mode 100644 index 000000000..7ce4d2e60 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -0,0 +1,56 @@ +package sqlancer.doris.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisInsertGenerator extends AbstractInsertGenerator { + + private final DorisGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public DorisInsertGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + return new DorisInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyInsertColumns(); + sb.append(table.getName()); + sb.append(" ("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + DorisErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertValue(DorisColumn column) { + if (column.hasDefaultValue() && Randomly.getBooleanWithRatherLowProbability()) { + sb.append("DEFAULT"); + } else { + String value = DorisToStringVisitor + .asString(DorisExprToNode.cast(new DorisNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); // 生成一个与column相同的常量类型 + sb.append(value); + } + } + +} diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java new file mode 100644 index 000000000..5dcb82e8f --- /dev/null +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -0,0 +1,427 @@ +package sqlancer.doris.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.DorisSchema.DorisRowValue; +import sqlancer.doris.ast.DorisAggregateOperation; +import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; +import sqlancer.doris.ast.DorisBetweenOperation; +import sqlancer.doris.ast.DorisBinaryArithmeticOperation; +import sqlancer.doris.ast.DorisBinaryArithmeticOperation.DorisBinaryArithmeticOperator; +import sqlancer.doris.ast.DorisBinaryComparisonOperation; +import sqlancer.doris.ast.DorisBinaryComparisonOperation.DorisBinaryComparisonOperator; +import sqlancer.doris.ast.DorisBinaryLogicalOperation; +import sqlancer.doris.ast.DorisBinaryLogicalOperation.DorisBinaryLogicalOperator; +import sqlancer.doris.ast.DorisCaseOperation; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunctionOperation.DorisFunction; +import sqlancer.doris.ast.DorisInOperation; +import sqlancer.doris.ast.DorisLikeOperation; +import sqlancer.doris.ast.DorisOrderByTerm; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisNewExpressionGenerator extends TypedExpressionGenerator { + + private final DorisGlobalState globalState; + + private final int maxDepth; + private boolean allowAggregateFunctions; + private DorisRowValue rowValue; + + private Set columnOfLeafNode; + + public DorisNewExpressionGenerator setRowValue(DorisRowValue rowValue) { + this.rowValue = rowValue; + return this; + } + + public void setColumnOfLeafNode(Set columnOfLeafNode) { + this.columnOfLeafNode = columnOfLeafNode; + } + + public DorisNewExpressionGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + } + + @Override + public DorisExpression generateLeafNode(DorisDataType dataType) { + if (Randomly.getBoolean()) { + return generateConstant(dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(dataType); + } else { + return createColumnOfType(dataType); + } + } + } + + final List filterColumns(DorisDataType dataType) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType().getPrimitiveDataType() == dataType) + .collect(Collectors.toList()); + } + } + + private DorisExpression createColumnOfType(DorisDataType type) { + List columns = filterColumns(type); + DorisColumn column = Randomly.fromList(columns); + DorisConstant value = rowValue == null ? null : rowValue.getValues().get(column); + if (columnOfLeafNode != null) { + columnOfLeafNode.add(DorisColumnValue.create(column, value)); + } + return DorisColumnValue.create(column, value); + } + + public List> generateOrderBy() { + List randomColumns = Randomly.subset(columns); + return randomColumns.stream() + .map(c -> new DorisOrderByTerm(new DorisColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) + .collect(Collectors.toList()); + } + + @Override + public DorisExpression generateExpression(DorisDataType type, int depth) { + // todo: case operation should be add into generateExpression + + if (Randomly.getBooleanWithRatherLowProbability() || depth >= maxDepth) { + return generateLeafNode(type); + } + + if (globalState.getDbmsSpecificOptions().testFunctions && Randomly.getBooleanWithRatherLowProbability()) { + List applicableFunctions = DorisFunction.getFunctionsCompatibleWith(type); + if (!applicableFunctions.isEmpty()) { + DorisFunction function = Randomly.fromList(applicableFunctions); + return function.getCall(type, this, depth + 1); + } + } + if (globalState.getDbmsSpecificOptions().testCasts && Randomly.getBooleanWithRatherLowProbability()) { + return new DorisCastOperation(DorisExprToNode.cast(generateExpression(getRandomType(), depth + 1)), type); + } + if (globalState.getDbmsSpecificOptions().testCase && Randomly.getBooleanWithRatherLowProbability()) { + DorisExpression expr = generateExpression(DorisDataType.BOOLEAN, depth + 1); + List conditions = new ArrayList<>(); + List cases = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + conditions.add(generateExpression(DorisDataType.BOOLEAN, depth + 1)); + cases.add(generateExpression(type, depth + 1)); + } + DorisExpression elseExpr = null; + if (Randomly.getBoolean()) { + elseExpr = generateExpression(type, depth + 1); + } + return new DorisCaseOperation(expr, conditions, cases, elseExpr); + } + + switch (type) { + case INT: + return generateIntExpression(depth); + case BOOLEAN: + return generateBooleanExpression(depth); + case FLOAT: + case DECIMAL: + case DATE: + case DATETIME: + case VARCHAR: + case NULL: + return generateConstant(type); + default: + throw new AssertionError(); + } + } + + public List generateExpressions(int nr, DorisDataType type) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(type)); + } + return expressions; + } + + private enum IntExpression { + UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION + } + + private DorisExpression generateIntExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } + IntExpression intExpression = Randomly.fromOptions(IntExpression.values()); + switch (intExpression) { + case UNARY_OPERATION: + return new DorisUnaryPrefixOperation(generateExpression(DorisDataType.INT, depth + 1), + Randomly.getBoolean() ? DorisUnaryPrefixOperator.UNARY_PLUS : DorisUnaryPrefixOperator.UNARY_MINUS); + case BINARY_ARITHMETIC_OPERATION: + return new DorisBinaryArithmeticOperation(generateExpression(DorisDataType.INT, depth + 1), + generateExpression(DorisDataType.INT, depth + 1), + Randomly.fromOptions(DorisBinaryArithmeticOperator.values())); + default: + throw new AssertionError(); + } + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; + // SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; + } + + DorisExpression generateBooleanExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (!globalState.getDbmsSpecificOptions().testIn) { + validOptions.remove(BooleanExpression.IN_OPERATION); + } + if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { + validOptions.remove(BooleanExpression.BINARY_LOGICAL_OPERATOR); + } + if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { + validOptions.remove(BooleanExpression.BINARY_COMPARISON); + } + if (!globalState.getDbmsSpecificOptions().testBetween) { + validOptions.remove(BooleanExpression.BETWEEN); + } + + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + return getPostfix(depth + 1); + case NOT: + return getNOT(depth + 1); + case BETWEEN: + return getBetween(depth + 1); + case IN_OPERATION: + return getIn(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return getBinaryLogical(depth + 1, DorisDataType.BOOLEAN); + case BINARY_COMPARISON: + return getComparison(depth + 1); + case LIKE: + return getLike(depth + 1, DorisDataType.VARCHAR); + default: + throw new AssertionError(); + } + + } + + DorisExpression getPostfix(int depth) { + DorisUnaryPostfixOperator randomOp = DorisUnaryPostfixOperator.getRandom(); + return new DorisUnaryPostfixOperation( + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp); + } + + DorisExpression getNOT(int depth) { + DorisUnaryPrefixOperator op = DorisUnaryPrefixOperator.NOT; + return new DorisUnaryPrefixOperation(generateExpression(op.getRandomInputDataTypes(), depth), op); + } + + DorisExpression getBetween(int depth) { + DorisDataType dataType = Randomly.fromList(Arrays.asList(DorisDataType.values()).stream() + .filter(t -> t != DorisDataType.BOOLEAN).collect(Collectors.toList())); + + return new DorisBetweenOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + generateExpression(dataType, depth), Randomly.getBoolean()); + } + + DorisExpression getIn(int depth) { + DorisDataType dataType = Randomly.fromOptions(DorisDataType.values()); + DorisExpression leftExpr = generateExpression(dataType, depth); + List rightExprs = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + rightExprs.add(generateExpression(dataType, depth)); + } + return new DorisInOperation(leftExpr, rightExprs, Randomly.getBoolean()); + } + + DorisExpression getBinaryLogical(int depth, DorisDataType dataType) { + DorisExpression expr = generateExpression(dataType, depth); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + expr = new DorisBinaryLogicalOperation(expr, generateExpression(DorisDataType.BOOLEAN, depth), + DorisBinaryLogicalOperator.getRandom()); + } + return expr; + } + + DorisExpression getComparison(int depth) { + // 跳过boolean + DorisDataType dataType = Randomly.fromList(Arrays.asList(DorisDataType.values()).stream() + .filter(t -> t != DorisDataType.BOOLEAN).collect(Collectors.toList())); + DorisExpression leftExpr = generateExpression(dataType, depth); + DorisExpression rightExpr = generateExpression(dataType, depth); + return new DorisBinaryComparisonOperation(leftExpr, rightExpr, + Randomly.fromOptions(DorisBinaryComparisonOperator.values())); + } + + DorisExpression getLike(int depth, DorisDataType dataType) { + return new DorisLikeOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + DorisLikeOperation.DorisLikeOperator.LIKE_OPERATOR); + } + + public DorisExpression generateExpressionWithExpectedResult(DorisDataType type) { + DorisExpression expr; + do { + expr = this.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + @Override + public DorisExpression generatePredicate() { + return generateExpression(DorisDataType.BOOLEAN); + } + + @Override + public DorisExpression negatePredicate(DorisExpression predicate) { + return new DorisUnaryPrefixOperation(predicate, DorisUnaryPrefixOperator.NOT); + } + + @Override + public DorisExpression isNull(DorisExpression predicate) { + return new DorisUnaryPostfixOperation(predicate, DorisUnaryPostfixOperator.IS_NULL); + } + + public DorisExpression generateConstant(DorisDataType type, boolean isNullable) { + if (isNullable && Randomly.getBooleanWithSmallProbability()) { + createConstant(DorisDataType.NULL); + } + return createConstant(type); + } + + @Override + public DorisExpression generateConstant(DorisDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return DorisConstant.createNullConstant(); + } + return createConstant(type); + } + + public DorisExpression createConstant(DorisDataType type) { + Randomly r = globalState.getRandomly(); + long timestamp; + switch (type) { + case INT: + if (globalState.getDbmsSpecificOptions().testIntConstants) { + return DorisConstant.createIntConstant(r.getInteger()); + } + return DorisConstant.createNullConstant(); + case BOOLEAN: + if (globalState.getDbmsSpecificOptions().testBooleanConstants) { + return DorisConstant.createBooleanConstant(Randomly.getBoolean()); + } + return DorisConstant.createNullConstant(); + case DECIMAL: + if (globalState.getDbmsSpecificOptions().testDecimalConstants) { + double v = r.getDouble(); + while (v == Double.MAX_VALUE || v == -Double.MAX_VALUE || v == Double.POSITIVE_INFINITY + || v == Double.NEGATIVE_INFINITY) { + v = r.getDouble(); + } + + // e.g. format 1234.413232532 to qualify num 34.4132 + String formatter = "%." + type.getDecimalScale() + "f"; + String vStr = String.format(formatter, v); + int pointPos = vStr.indexOf('.'); + if (pointPos > type.getDecimalPrecision() - type.getDecimalScale()) { + vStr = vStr.substring(pointPos - (type.getDecimalPrecision() - type.getDecimalScale())); + } + return DorisConstant.createFloatConstant(Double.parseDouble(vStr)); + } + return DorisConstant.createNullConstant(); + case FLOAT: + if (globalState.getDbmsSpecificOptions().testFloatConstants) { + return DorisConstant.createFloatConstant((float) r.getDouble()); + } + return DorisConstant.createNullConstant(); + case DATE: + if (globalState.getDbmsSpecificOptions().testDateConstants) { + // [1970-01-01 08:00:00, 3000-01-01 00:00:00] + timestamp = globalState.getRandomly().getLong(0, 32503651200L); + return DorisConstant.createDateConstant(timestamp); + } + return DorisConstant.createNullConstant(); + case DATETIME: + if (globalState.getDbmsSpecificOptions().testDateTimeConstants) { + // [1970-01-01 08:00:00, 3000-01-01 00:00:00] + timestamp = globalState.getRandomly().getLong(0, 32503651200L); + return Randomly.fromOptions(DorisConstant.createDatetimeConstant(timestamp), + DorisConstant.createDatetimeConstant()); + } + return DorisConstant.createNullConstant(); + case VARCHAR: + if (globalState.getDbmsSpecificOptions().testStringConstants) { + String s = r.getString(); + if (s.length() > type.getVarcharLength()) { + s = s.substring(0, type.getVarcharLength()); + } + return DorisConstant.createStringConstant(s); + } + return DorisConstant.createNullConstant(); + case NULL: + return DorisConstant.createNullConstant(); + default: + throw new AssertionError(type); + } + } + + @Override + protected DorisExpression generateColumn(DorisDataType type) { + return null; + } + + @Override + protected DorisDataType getRandomType() { + return Randomly.fromOptions(DorisDataType.values()); + } + + @Override + protected boolean canGenerateColumnOfType(DorisDataType type) { + return false; + } + + public DorisExpression generateArgsForAggregate(DorisAggregateFunction aggregateFunction) { + DorisDataType dataType = Randomly.fromOptions(DorisDataType.values()); + return new DorisAggregateOperation(generateExpressions(aggregateFunction.getNrArgs(), dataType), + aggregateFunction); + } + + public DorisExpression generateAggregate() { + DorisAggregateFunction aggrFunc = DorisAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + public DorisExpression generateHavingClause() { + allowAggregateFunctions = true; + DorisExpression expression = generateExpression(DorisDataType.BOOLEAN); + allowAggregateFunctions = false; + return expression; + } + + public void setAllowAggregateFunctions(boolean allowAggregateFunctions) { + this.allowAggregateFunctions = allowAggregateFunctions; + } +} diff --git a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java new file mode 100644 index 000000000..3193594df --- /dev/null +++ b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java @@ -0,0 +1,79 @@ +package sqlancer.doris.gen; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.visitor.DorisExprToNode; + +public final class DorisRandomQuerySynthesizer { + + private DorisRandomQuerySynthesizer() { + } + + public static DorisSelect generateSelect(DorisGlobalState globalState, int nrColumns) { + DorisTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + List targetColumns = targetTables.getColumns(); + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState).setColumns(targetColumns); + DorisSelect select = new DorisSelect(); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); + int freeColumns = targetColumns.size(); + select.setDistinct(DorisSelect.DorisSelectDistinctType.getRandomWithoutNull()); + List> columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + Node column = null; + if (freeColumns > 0 && Randomly.getBoolean()) { + column = new DorisColumnValue(targetColumns.get(freeColumns - 1), null); + freeColumns -= 1; + columnOfLeafNode.add((DorisColumnValue) column); + } else { + column = DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); + } + columns.add(column); + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DorisJoin.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setHavingClause(DorisExprToNode.cast(gen.generateHavingClause())); + } + if (Randomly.getBoolean()) { + select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + } + + List> noExprColumns = new ArrayList<>(columnOfLeafNode); + + if (Randomly.getBoolean()) { + select.setOrderByExpressions(Randomly.nonEmptySubset(noExprColumns)); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(noExprColumns); + } + if (Randomly.getBoolean()) { + select.setLimitClause(DorisConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(DorisConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + return select; + } + +} diff --git a/src/sqlancer/doris/gen/DorisTableGenerator.java b/src/sqlancer/doris/gen/DorisTableGenerator.java new file mode 100644 index 000000000..8a98739f5 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisTableGenerator.java @@ -0,0 +1,114 @@ +package sqlancer.doris.gen; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisTableGenerator { + + // private final ExpectedErrors errors = new ExpectedErrors(); + + public static SQLQueryAdapter createRandomTableStatement(DorisGlobalState globalState) throws SQLException { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return new DorisTableGenerator().getQuery(globalState); + } + + public SQLQueryAdapter getQuery(DorisGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + DorisSchema.DorisTableDataModel dataModel = DorisSchema.DorisTableDataModel.getRandom(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(globalState); + Collections.sort(columns); + if (columns.isEmpty() || !columns.get(0).isKey()) { + return null; // ensure table has at least one key column + } + sb.append(columns.stream().map(DorisColumn::toString).collect(Collectors.joining(", "))); + sb.append(")"); + + List keysColumn = columns.stream().filter(DorisColumn::isKey).collect(Collectors.toList()); + if (globalState.getDbmsSpecificOptions().testDataModel && Randomly.getBoolean() && !keysColumn.isEmpty()) { + sb.append(" " + dataModel).append(" KEY("); + sb.append(keysColumn.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + } + sb.append(generateDistributionStr(globalState, dataModel, keysColumn)); + sb.append(" PROPERTIES (\"replication_num\" = \"1\")"); // now only consider this one parameter + DorisErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + public static String generateDistributionStr(DorisGlobalState globalState, + DorisSchema.DorisTableDataModel dataModel, List keysColumn) { + // DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num] + // DISTRIBUTED BY RANDOM [BUCKETS num] + StringBuilder sb = new StringBuilder(); + sb.append(" DISTRIBUTED BY"); + if (dataModel == DorisSchema.DorisTableDataModel.UNIQUE || Randomly.getBoolean()) { + sb.append(" HASH ("); + sb.append(Randomly.nonEmptySubset(keysColumn).stream().map(DorisColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + } else { + sb.append(" RANDOM"); + } + if (Randomly.getBoolean()) { + sb.append(" BUCKETS ").append(globalState.getRandomly().getInteger(1, 32)); + } + return sb.toString(); + } + + private static List getNewColumns(DorisGlobalState globalState) { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + DorisCompositeDataType columnType = DorisCompositeDataType.getRandomWithoutNull(); + columnType.initColumnArgs(); // set decimalAndVarchar + + boolean iskey = columnType.canBeKey() && Randomly.getBoolean(); + boolean isNullable = Randomly.getBoolean(); + if (!globalState.getDbmsSpecificOptions().testNotNullConstraints) { + isNullable = true; + } + // boolean isHllOrBitmap = (columnType.getPrimitiveDataType() == DorisSchema.DorisDataType.HLL) + // || (columnType.getPrimitiveDataType() == DorisSchema.DorisDataType.BITMAP); + boolean isHllOrBitmap = false; + DorisSchema.DorisColumnAggrType aggrType = DorisSchema.DorisColumnAggrType.NULL; + if (globalState.getDbmsSpecificOptions().testColumnAggr && (isHllOrBitmap || !iskey)) { + aggrType = DorisSchema.DorisColumnAggrType.getRandom(columnType); + } + + boolean hasDefaultValue = globalState.getDbmsSpecificOptions().testDefaultValues && Randomly.getBoolean() + && !isHllOrBitmap; + String defaultValue = ""; + if (hasDefaultValue) { + defaultValue = DorisToStringVisitor + .asString(DorisExprToNode.cast(new DorisNewExpressionGenerator(globalState) + .generateConstant(columnType.getPrimitiveDataType(), isNullable))); + } + columns.add(new DorisColumn(columnName, columnType, iskey, isNullable, aggrType, hasDefaultValue, + defaultValue)); + } + return columns; + } + +} diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java new file mode 100644 index 000000000..74fb9383d --- /dev/null +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -0,0 +1,56 @@ +package sqlancer.doris.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public final class DorisUpdateGenerator extends AbstractUpdateGenerator { + + private final DorisGlobalState globalState; + private DorisNewExpressionGenerator gen; + + private DorisUpdateGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + return new DorisUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new DorisNewExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + sb.append(" WHERE "); + sb.append(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + DorisErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void updateValue(DorisColumn column) { + if (Randomly.getBooleanWithSmallProbability()) { + DorisExpression expr = gen.generateExpression(column.getType().getPrimitiveDataType()); + sb.append(DorisToStringVisitor.asString(DorisExprToNode.cast(expr))); + } else { + DorisExpression expr = gen.generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()); + sb.append(DorisToStringVisitor.asString(expr)); + } + + } + +} diff --git a/src/sqlancer/doris/gen/DorisViewGenerator.java b/src/sqlancer/doris/gen/DorisViewGenerator.java new file mode 100644 index 000000000..480607378 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisViewGenerator.java @@ -0,0 +1,38 @@ +package sqlancer.doris.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public final class DorisViewGenerator { + + private DorisViewGenerator() { + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(DorisToStringVisitor.asString(DorisRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + DorisErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java new file mode 100644 index 000000000..dab6b53ae --- /dev/null +++ b/src/sqlancer/doris/oracle/DorisNoRECOracle.java @@ -0,0 +1,137 @@ +package sqlancer.doris.oracle; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisNoRECOracle extends NoRECBase implements TestOracle { + + private final DorisSchema s; + + public DorisNoRECOracle(DorisGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + DorisTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state).setColumns(columns); + Node randomWhereCondition = DorisExprToNode + .cast(gen.generateExpression(DorisDataType.BOOLEAN)); + List tables = randomTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DorisJoin.getJoins(tableList, state); + int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), + randomWhereCondition, joins); + int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, + randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getUnoptimizedQueryCount(List> tableList, + Node randomWhereCondition, List> joins) throws SQLException { + DorisSelect select = new DorisSelect(); + Node asText = new NewPostfixTextNode<>(new DorisCastOperation( + new NewPostfixTextNode(randomWhereCondition, + " IS NOT NULL AND " + DorisToStringVisitor.asString(randomWhereCondition)), + new DorisCompositeDataType(DorisDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + select.setFromList(tableList); + select.setJoinList(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + DorisToStringVisitor.asString(select) + ") as res"; + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getOptimizedQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, List> joins) + throws SQLException { + DorisSelect select = new DorisSelect(); + // select.setGroupByClause(groupBys); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + List> constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByExpressions(constants); + } + select.setJoinList(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = DorisToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..95c79393b --- /dev/null +++ b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java @@ -0,0 +1,155 @@ +package sqlancer.doris.oracle; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.DorisSchema.DorisRowValue; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.visitor.DorisExpectedValueVisitor; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + + public DorisPivotedQuerySynthesisOracle(DorisGlobalState globalState) { + super(globalState); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + protected Query getRectifiedQuery() throws Exception { + DorisTables randomTables = globalState.getSchema().getRandomTableNonEmptyAndViewTables(); + List columns = randomTables.getColumns(); + DorisSelect selectStatement = new DorisSelect(); + boolean isDistinct = Randomly.getBoolean(); + selectStatement.setDistinct(isDistinct); + pivotRow = randomTables.getRandomRowValue(globalState.getConnection()); + fetchColumns = columns; + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new DorisColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + selectStatement.setFromList(randomTables.getTables().stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList())); + DorisExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(DorisExprToNode.cast(whereClause)); + List> groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + Node limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + Node offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + gen.setColumns(columns); + if (!isDistinct) { + List> constants = new ArrayList<>(); + constants.add(new DorisConstant.DorisIntConstant( + Randomly.smallNumber() % selectStatement.getFetchColumns().size() + 1)); + selectStatement.setOrderByExpressions(constants); + } + return new SQLQueryAdapter(DorisToStringVisitor.asString(selectStatement), errors); + } + + private DorisExpression generateRectifiedExpression(List columns, DorisRowValue pivotRow) { + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState).setColumns(columns); + gen.setRowValue(pivotRow); + DorisExpression expr = gen.generateExpressionWithExpectedResult(DorisDataType.BOOLEAN); + DorisExpression result = null; + if (expr.getExpectedValue().isNull()) { + result = new DorisUnaryPostfixOperation(expr, DorisUnaryPostfixOperation.DorisUnaryPostfixOperator.IS_NULL); + } else if (!expr.getExpectedValue().cast(DorisDataType.BOOLEAN).asBoolean()) { + result = new DorisUnaryPrefixOperation(expr, DorisUnaryPrefixOperation.DorisUnaryPrefixOperator.NOT); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); + sb.append(pivotRowQuery.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (DorisColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL "); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).toString()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + private DorisColumn getFetchValueAliasedColumn(DorisColumn c) { + DorisColumn aliasedColumn = new DorisColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType(), false, false); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + @Override + protected String getExpectedValues(DorisExpression expr) { + return DorisExpectedValueVisitor.asExpectedValues(DorisExprToNode.cast(expr)); + } + + private List> generateGroupByClause(List columns, DorisRowValue rowValue) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> new DorisColumnValue(c, rowValue.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private Node generateLimit() { + if (Randomly.getBoolean()) { + return DorisConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private Node generateOffset() { + if (Randomly.getBoolean()) { + return DorisConstant.createIntConstant(0); + } else { + return null; + } + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java new file mode 100644 index 000000000..2e886afbe --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -0,0 +1,194 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.ast.DorisAggregateOperation; +import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; +import sqlancer.doris.ast.DorisBinaryArithmeticOperation; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; +import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningAggregateTester extends DorisQueryPartitioningBase + implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public DorisQueryPartitioningAggregateTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + DorisAggregateFunction aggregateFunction = Randomly.fromOptions(DorisAggregateFunction.MAX, + DorisAggregateFunction.MIN, DorisAggregateFunction.SUM, DorisAggregateFunction.COUNT, + DorisAggregateFunction.AVG); + NewFunctionNode aggregate = (DorisAggregateOperation) gen + .generateArgsForAggregate(aggregateFunction); + List> fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add((DorisAggregateOperation) gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List> constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByExpressions(constants); + } + originalQuery = DorisToStringVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResult == null && secondResult != null + || firstResult != null && (!firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { + throw new AssertionError(); + } + + } + + private String createMetamorphicUnionQuery(DorisSelect select, + NewFunctionNode aggregate, List> from) { + String metamorphicQuery; + Node whereClause = DorisExprToNode.cast(gen.generateExpression(DorisDataType.BOOLEAN)); + Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + DorisUnaryPrefixOperator.NOT); + Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + DorisUnaryPostfixOperator.IS_NULL); + List> mappedAggregate = mapped(aggregate); + DorisSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + DorisSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + DorisSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += DorisToStringVisitor.asString(leftSelect) + " UNION ALL " + + DorisToStringVisitor.asString(middleSelect) + " UNION ALL " + + DorisToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List> mapped(NewFunctionNode aggregate) { + + DorisCastOperation count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(Arrays.asList(aggregate)); + case AVG: + NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + DorisAggregateFunction.SUM); + count = new DorisCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), + new DorisCompositeDataType(DorisDataType.FLOAT, 8)); + return aliasArgs(Arrays.asList(sum, count)); + case STDDEV_POP: + NewFunctionNode sumSquared = new NewFunctionNode<>( + Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DorisBinaryArithmeticOperation.DorisBinaryArithmeticOperator.MULTIPLICATION)), + DorisAggregateFunction.SUM); + count = new DorisCastOperation( + new NewFunctionNode(aggregate.getArgs(), + DorisAggregateFunction.COUNT), + new DorisCompositeDataType(DorisDataType.FLOAT, 8)); + NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), + DorisAggregateFunction.AVG); + return aliasArgs(Arrays.asList(sumSquared, count, avg)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List> aliasArgs(List> originalAggregateArgs) { + List> args = new ArrayList<>(); + int i = 0; + for (Node expr : originalAggregateArgs) { + args.add(new NewAliasNode(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(NewFunctionNode aggregate) { + switch (aggregate.getFunc()) { + case STDDEV_POP: + return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; + case AVG: + return "SUM(agg0::FLOAT)/SUM(agg1)::FLOAT"; + case COUNT: + return DorisAggregateFunction.SUM.toString() + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private DorisSelect getSelect(List> aggregates, List> from, + Node whereClause, List> joinList) { + DorisSelect leftSelect = new DorisSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(groupByExpression); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java new file mode 100644 index 000000000..5db230854 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java @@ -0,0 +1,98 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.gen.DorisNewExpressionGenerator; + +public class DorisQueryPartitioningBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + DorisSchema s; + DorisTables targetTables; + DorisNewExpressionGenerator gen; + DorisSelect select; + + List> groupByExpression; + + public DorisQueryPartitioningBase(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + public static String canonicalizeResultValue(String value) { + if (value == null) { + return value; + } + + switch (value) { + case "-0.0": + return "0.0"; + case "-0": + return "0"; + default: + } + + return value; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new DorisNewExpressionGenerator(state).setColumns(targetTables.getColumns()); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); + initializeTernaryPredicateVariants(); + select = new DorisSelect(); + columnOfLeafNode.addAll(targetTables.getColumns().stream().map(c -> new DorisColumnValue(c, null)) + .collect(Collectors.toList())); + groupByExpression = new ArrayList<>(columnOfLeafNode); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DorisJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new DorisColumn("*", null, false, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..6cc15fdc8 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java @@ -0,0 +1,45 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningDistinctTester extends DorisQueryPartitioningBase { + + public DorisQueryPartitioningDistinctTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(DorisExprToNode.cast(predicate)); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + + String unionString = "SELECT DISTINCT * FROM (" + firstQueryString + " UNION ALL " + secondQueryString + + " UNION ALL " + thirdQueryString + ") tmpTable"; + combinedString.add(unionString); + List secondResultSet = ComparatorHelper.getResultSetFirstColumnAsString(unionString, errors, state); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DorisQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..0761fddb6 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java @@ -0,0 +1,55 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningGroupByTester extends DorisQueryPartitioningBase { + + public DorisQueryPartitioningGroupByTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setGroupByExpressions(groupByExpression); + select.setWhereClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(DorisExprToNode.cast(predicate)); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DorisQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + List> generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java new file mode 100644 index 000000000..7ce10dc09 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -0,0 +1,73 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningHavingTester extends DorisQueryPartitioningBase + implements TestOracle { + + public DorisQueryPartitioningHavingTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBoolean()) { + select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + } + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + List> constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByExpressions(constants); + } + select.setGroupByExpressions(groupByExpression); + select.setHavingClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setHavingClause(DorisExprToNode.cast(predicate)); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setHavingClause(DorisExprToNode.cast(negatedPredicate)); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setHavingClause(DorisExprToNode.cast(isNullPredicate)); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DorisQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + protected DorisExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List> generateFetchColumns() { + gen.setAllowAggregateFunctions(true); + List> expressions = DorisExprToNode + .casts(gen.generateExpressions(Randomly.smallNumber() + 1)); + gen.setAllowAggregateFunctions(false); + return expressions; + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java new file mode 100644 index 000000000..c45bc36fb --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -0,0 +1,53 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningWhereTester extends DorisQueryPartitioningBase { + + public DorisQueryPartitioningWhereTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + List> constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByExpressions(constants); + } + select.setWhereClause(DorisExprToNode.cast(predicate)); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DorisQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/doris/utils/DorisNumberUtils.java b/src/sqlancer/doris/utils/DorisNumberUtils.java new file mode 100644 index 000000000..01dd7238d --- /dev/null +++ b/src/sqlancer/doris/utils/DorisNumberUtils.java @@ -0,0 +1,111 @@ +package sqlancer.doris.utils; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class DorisNumberUtils { + private static Pattern numberPattern = Pattern.compile("-?[0-9]+(\\\\.[0-9]+)?"); + private static Pattern integerPattern = Pattern.compile("^[-\\+]?[\\d]*$"); + private static Pattern datePattern = Pattern + .compile("^([1-9]\\d{3}-)(([0]{0,1}[1-9]-)|([1][0-2]-))(([0-3]{0,1}[0-9]))$"); + private static Pattern datetimePattern = Pattern.compile( + "((([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})-(((0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01]))|((0[469]|11)-(0[1-9]|[12][0-9]|30))|(02-(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|((0[48]|[2468][048]|[3579][26])00))-02-29))\\\\s+([0-1]?[0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])\n"); + + private DorisNumberUtils() { + } + + public static boolean isNumber(String str) { + Matcher m = numberPattern.matcher(str); + return m.matches(); + } + + public static boolean isInteger(String str) { + Matcher m = integerPattern.matcher(str); + return m.matches(); + } + + public static boolean isDate(String str) { + Matcher m = datePattern.matcher(str); + return m.matches(); + } + + public static boolean isDatetime(String str) { + Matcher m = datetimePattern.matcher(str); + return m.matches(); + } + + public static String timestampToDateText(long ts) { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + return dateFormat.format(ts); + } + + public static String timestampToDatetimeText(long ts) { + SimpleDateFormat datetimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return datetimeFormat.format(ts); + } + + public static String dateTextToDatetimeText(String date) { + // '2021-03-12' -> '2021-03-12 00:00:00' + return date + " 00:00:00"; + } + + public static String datetimeTextToDateText(String datetime) { + // '2021-03-12 00:00:00' -> '2021-03-12' + return datetime.substring(0, 10); + } + + public static boolean datetimeEqual(String dt1, String dt2) { + String datetime1 = dt1; + String datetime2 = dt2; + if (isDate(dt1)) { + datetime1 = dateTextToDatetimeText(dt1); + } + if (isDate(dt2)) { + datetime2 = dateTextToDatetimeText(dt2); + } + return datetime1.contentEquals(datetime2); + } + + public static boolean dateEqual(String d1, String d2) { + String date1 = d1; + String date2 = d2; + if (isDatetime(d1)) { + date1 = datetimeTextToDateText(d1); + } + if (isDatetime(d2)) { + date2 = datetimeTextToDateText(d2); + } + return date1.contentEquals(date2); + } + + public static boolean dateLessThan(String d1, String d2) { + String date1 = d1; + String date2 = d2; + if (isDatetime(d1)) { + date1 = datetimeTextToDateText(d1); + } + if (isDatetime(d2)) { + date2 = datetimeTextToDateText(d2); + } + return date1.compareTo(date2) < 0; + } + + public static boolean datetimeLessThan(String dt1, String dt2) { + String datetime1 = dt1; + String datetime2 = dt2; + if (isDate(dt1)) { + datetime1 = dateTextToDatetimeText(dt1); + } + if (isDate(dt2)) { + datetime2 = dateTextToDatetimeText(dt2); + } + return datetime1.compareTo(datetime2) < 0; + } + + public static String getCurrentTimeText() { + SimpleDateFormat datetimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return datetimeFormat.format(new Date()); + } +} diff --git a/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java new file mode 100644 index 000000000..57cb29ae0 --- /dev/null +++ b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java @@ -0,0 +1,155 @@ +package sqlancer.doris.visitor; + +import java.util.List; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; + +public class DorisExpectedValueVisitor { + + protected final StringBuilder sb = new StringBuilder(); + + private void print(Node expr) { + sb.append(DorisToStringVisitor.asString(expr)); + sb.append(" -- "); + sb.append(((DorisExpression) expr).getExpectedValue()); + sb.append("\n"); + } + + @SuppressWarnings("unchecked") + public void visit(Node expr) { + assert expr != null; + if (expr instanceof ColumnReferenceNode) { + visit((ColumnReferenceNode) expr); + } else if (expr instanceof NewUnaryPostfixOperatorNode) { + visit((NewUnaryPostfixOperatorNode) expr); + } else if (expr instanceof NewUnaryPrefixOperatorNode) { + visit((NewUnaryPrefixOperatorNode) expr); + } else if (expr instanceof NewBinaryOperatorNode) { + visit((NewBinaryOperatorNode) expr); + } else if (expr instanceof TableReferenceNode) { + visit((TableReferenceNode) expr); + } else if (expr instanceof NewFunctionNode) { + visit((NewFunctionNode) expr); + } else if (expr instanceof NewBetweenOperatorNode) { + visit((NewBetweenOperatorNode) expr); + } else if (expr instanceof NewInOperatorNode) { + visit((NewInOperatorNode) expr); + } else if (expr instanceof NewOrderingTerm) { + visit((NewOrderingTerm) expr); + } else if (expr instanceof NewAliasNode) { + visit((NewAliasNode) expr); + } else if (expr instanceof NewPostfixTextNode) { + visit((NewPostfixTextNode) expr); + } else if (expr instanceof DorisConstant) { + visit((DorisConstant) expr); + } else if (expr instanceof DorisSelect) { + visit((DorisSelect) expr); + } else if (expr instanceof DorisJoin) { + visit((DorisJoin) expr); + } else { + throw new AssertionError(expr); + } + } + + public void visit(ColumnReferenceNode c) { + print(c); + } + + public void visit(NewUnaryPostfixOperatorNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewUnaryPrefixOperatorNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewBinaryOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(TableReferenceNode t) { + print(t); + } + + public void visit(NewFunctionNode fun) { + print(fun); + visit(fun.getArgs()); + } + + public void visit(List> expressions) { + for (Node expression : expressions) { + visit(expression); + } + } + + public void visit(NewBetweenOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getMiddle()); + visit(op.getRight()); + } + + public void visit(NewInOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(NewOrderingTerm op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewAliasNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewPostfixTextNode postFixText) { + print(postFixText); + visit(postFixText.getExpr()); + } + + public void visit(DorisConstant constant) { + print(constant); + } + + public void visit(DorisSelect select) { + print(select.getWhereClause()); + } + + public void visit(DorisJoin join) { + print(join.getOnCondition()); + } + + public String get() { + return sb.toString(); + } + + public static String asExpectedValues(Node expr) { + DorisExpectedValueVisitor v = new DorisExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + +} diff --git a/src/sqlancer/doris/visitor/DorisExprToNode.java b/src/sqlancer/doris/visitor/DorisExprToNode.java new file mode 100644 index 000000000..cec7aa061 --- /dev/null +++ b/src/sqlancer/doris/visitor/DorisExprToNode.java @@ -0,0 +1,25 @@ +package sqlancer.doris.visitor; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.ast.DorisExpression; + +public final class DorisExprToNode { + + private DorisExprToNode() { + + } + + @SuppressWarnings("unchecked") + public static Node cast(DorisExpression expression) { + return (Node) expression; + } + + @SuppressWarnings("unchecked") + public static List> casts(List expressions) { + return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/doris/visitor/DorisToStringVisitor.java b/src/sqlancer/doris/visitor/DorisToStringVisitor.java new file mode 100644 index 000000000..846955c83 --- /dev/null +++ b/src/sqlancer/doris/visitor/DorisToStringVisitor.java @@ -0,0 +1,175 @@ +package sqlancer.doris.visitor; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.ast.DorisCaseOperation; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunctionOperation; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; + +public class DorisToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof DorisConstant) { + visit((DorisConstant) expr); + } else if (expr instanceof DorisSelect) { + visit((DorisSelect) expr); + } else if (expr instanceof DorisJoin) { + visit((DorisJoin) expr); + } else if (expr instanceof DorisCastOperation) { + visit((DorisCastOperation) expr); + } else if (expr instanceof DorisCaseOperation) { + visit((DorisCaseOperation) expr); + } else if (expr instanceof DorisFunctionOperation) { + visit((DorisFunctionOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(DorisJoin join) { + sb.append(" "); + visit(join.getLeftTable()); + sb.append(" "); + switch (join.getJoinType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } else { + sb.append("CROSS "); + } + sb.append("JOIN "); + break; + case LEFT: + sb.append("LEFT "); + if (Randomly.getBoolean()) { + sb.append(" OUTER "); + } + sb.append("JOIN "); + break; + case RIGHT: + sb.append("RIGHT "); + if (Randomly.getBoolean()) { + sb.append(" OUTER "); + } + sb.append("JOIN "); + break; + case STRAIGHT: + sb.append("STRAIGHT_JOIN "); + break; + default: + throw new AssertionError(); + } + visit(join.getRightTable()); + sb.append(" "); + if (join.getOnCondition() != null) { + sb.append("ON "); + visit(join.getOnCondition()); + } + } + + private void visit(DorisConstant constant) { + sb.append(constant.toString()); + } + + private void visit(DorisCastOperation castExpr) { + sb.append("CAST("); + visit(castExpr.getExpr()); + sb.append(" AS "); + sb.append(castExpr.getType().toString()); + sb.append(") "); + } + + private void visit(DorisFunctionOperation func) { + sb.append(func.getFunction().getFunctionName()); + sb.append("("); + + if (func.getArgs() != null) { + for (int i = 0; i < func.getArgs().size(); i++) { + visit(DorisExprToNode.cast(func.getArgs().get(i))); + if (i != func.getArgs().size() - 1) { + sb.append(","); + } + } + } + sb.append(") "); + } + + private void visit(DorisCaseOperation cases) { + sb.append("CASE "); + visit(DorisExprToNode.cast(cases.getExpr())); + sb.append(" "); + for (int i = 0; i < cases.getConditions().size(); i++) { + DorisExpression predicate = cases.getConditions().get(i); + DorisExpression then = cases.getThenClauses().get(i); + sb.append(" WHEN "); + visit(DorisExprToNode.cast(predicate)); + sb.append(" THEN "); + visit(DorisExprToNode.cast(then)); + sb.append(" "); + } + if (cases.getElseClause() != null) { + sb.append("ELSE "); + visit(DorisExprToNode.cast(cases.getElseClause())); + sb.append(" "); + } + sb.append("END "); + } + + private void visit(DorisSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(Node expr) { + DorisToStringVisitor visitor = new DorisToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + public static String asString(DorisExpression expr) { + DorisToStringVisitor visitor = new DorisToStringVisitor(); + visitor.visit(DorisExprToNode.cast(expr)); + return visitor.get(); + } +} diff --git a/test/sqlancer/dbms/TestDoris.java b/test/sqlancer/dbms/TestDoris.java new file mode 100644 index 000000000..fd3c1e95a --- /dev/null +++ b/test/sqlancer/dbms/TestDoris.java @@ -0,0 +1,54 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDoris { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisNoREC() { + String dorisAvailable = System.getenv("DORIS_AVAILABLE"); + boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); + assumeTrue(dorisIsAvailable); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "NOREC")); + } + + @Test + public void testdorisPQS() { + String dorisAvailable = System.getenv("DORIS_AVAILABLE"); + boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); + assumeTrue(dorisIsAvailable); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "PQS")); + } + + @Test + public void testdorisTLPQueryPartitioning() { + String dorisAvailable = System.getenv("DORIS_AVAILABLE"); + boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); + assumeTrue(dorisIsAvailable); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "QUERY_PARTITIONING")); + } +} From d6263cd2cd3904854d24b12bfffcd0d7a4d086d4 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 22 May 2023 17:07:41 +0800 Subject: [PATCH 0771/1284] chore: upgrade versions in CI (#787) * chore: upgrade versions in CI * fix: typo and version * fix: add integers in mysql * fix: fix mysql Unknown system variable 'myisam_repair_threads' * fix: Set up typo --- .github/workflows/main.yml | 105 ++++++++---------- src/sqlancer/Randomly.java | 33 ++++++ src/sqlancer/mysql/gen/MySQLSetGenerator.java | 5 +- .../mysql/gen/MySQLTableGenerator.java | 8 +- 4 files changed, 88 insertions(+), 63 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c03e3908c..945d1bc18 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,18 +14,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - - name: Setup Python + - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.10' @@ -36,11 +36,11 @@ jobs: name: DBMS Tests (Citus) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -82,11 +82,11 @@ jobs: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -105,17 +105,15 @@ jobs: docker stop clickhouse-server docker rm clickhouse-server - - cockroachdb: name: DBMS Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -134,11 +132,11 @@ jobs: name: QPG Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -166,11 +164,11 @@ jobs: - 8000:8000 - 3307:3307 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -179,17 +177,16 @@ jobs: run: | DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test - duckdb: name: DBMS Tests (DuckDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build @@ -201,11 +198,11 @@ jobs: name: DBMS Tests (H2) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -215,14 +212,14 @@ jobs: mariadb: name: DBMS Tests (MariaDB) - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -243,7 +240,7 @@ jobs: name: DBMS Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Materialize @@ -254,7 +251,7 @@ jobs: # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -268,7 +265,7 @@ jobs: name: QPG Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Materialize @@ -279,7 +276,7 @@ jobs: # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -291,25 +288,20 @@ jobs: mysql: name: DBMS Tests (MySQL) - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up MySQL run: | - sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 - sudo apt-get remove mysql-* - wget -q https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar - tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar - sudo dpkg -i *.deb - sudo systemctl start mysql + sudo systemctl start mysql.service # MySQL 8.0.32-0ubuntu0.22.04.2 - name: Create SQLancer user run: mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - name: Run Tests @@ -317,16 +309,15 @@ jobs: MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP - postgres: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up PostgreSQL - uses: harmon758/postgresql-action@v1 + uses: harmon758/postgresql-action@v1.0.0 with: postgresql version: '12' postgresql user: 'sqlancer' @@ -346,11 +337,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build @@ -360,17 +351,16 @@ jobs: mvn -Dtest=TestSQLitePQS test mvn -Dtest=TestSQLite3 test - sqlite-qpg: name: QPG Tests (SQLite) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build @@ -379,16 +369,15 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test - tidb: name: DBMS Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -407,11 +396,11 @@ jobs: name: QPG Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -430,16 +419,16 @@ jobs: name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Setup Yugabyte + - name: Set up Yugabyte run: | docker pull yugabytedb/yugabyte:latest docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false @@ -493,11 +482,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 13 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 13 - name: Build @@ -510,11 +499,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 14 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 14 - name: Build @@ -527,11 +516,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 15 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 15-ea - name: Build diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index ae34ba1b1..e1250abb3 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -16,6 +16,7 @@ public final class Randomly { private static int cacheSize = 100; private final List cachedLongs = new ArrayList<>(); + private final List cachedIntegers = new ArrayList<>(); private final List cachedStrings = new ArrayList<>(); private final List cachedDoubles = new ArrayList<>(); private final List cachedBytes = new ArrayList<>(); @@ -30,6 +31,12 @@ private void addToCache(long val) { } } + private void addToCache(int val) { + if (useCaching && cachedIntegers.size() < cacheSize && !cachedIntegers.contains(val)) { + cachedIntegers.add(val); + } + } + private void addToCache(double val) { if (useCaching && cachedDoubles.size() < cacheSize && !cachedDoubles.contains(val)) { cachedDoubles.add(val); @@ -50,6 +57,14 @@ private Long getFromLongCache() { } } + private Integer getFromIntegerCache() { + if (!useCaching || cachedIntegers.isEmpty()) { + return null; + } else { + return Randomly.fromList(cachedIntegers); + } + } + private Double getFromDoubleCache() { if (!useCaching) { return null; @@ -368,6 +383,24 @@ public long getPositiveInteger() { return value; } + public int getPositiveIntegerInt() { + if (cacheProbability()) { + Integer value = getFromIntegerCache(); + if (value != null && value >= 0) { + return value; + } + } + int value; + if (smallBiasProbability()) { + value = Randomly.fromOptions(0, Integer.MAX_VALUE, 1); + } else { + value = getNextInt(0, Integer.MAX_VALUE); + } + addToCache(value); + assert value >= 0; + return value; + } + public double getFiniteDouble() { while (true) { double val = getDouble(); diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 2db20fee9..00a477714 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -66,8 +66,9 @@ private enum Action { MAX_SP_RECURSION_DEPTH("max_sp_recursion_depth", (r) -> r.getLong(0, 255), Scope.GLOBAL, Scope.SESSION), // MYISAM_DATA_POINTER_SIZE("myisam_data_pointer_size", (r) -> r.getLong(2, 7), Scope.GLOBAL), // MYISAM_MAX_SORT_FILE_SIZE("myisam_max_sort_file_size", (r) -> r.getLong(0, 9223372036854775807L), Scope.GLOBAL), // - MYISAM_REPAIR_THREADS("myisam_repair_threads", (r) -> r.getLong(1, Long.MAX_VALUE), Scope.GLOBAL, - Scope.SESSION), // + // MYISAM_REPAIR_THREADS("myisam_repair_threads", (r) -> r.getLong(1, Long.MAX_VALUE), Scope.GLOBAL, + // Scope.SESSION), // comment out this to avoid java.sql.SQLException: Unknown system variable + // 'myisam_repair_threads' MYISAM_SORT_BUFFER_SIZE("myisam_sort_buffer_size", (r) -> r.getLong(4096, Long.MAX_VALUE), Scope.GLOBAL, Scope.SESSION), // MYISAM_STATS_METHOD("myisam_stats_method", diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 579d9a4c9..427f5b7c3 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -176,9 +176,10 @@ private void appendTableOptions() { sb.append("AUTO_INCREMENT = "); sb.append(r.getPositiveInteger()); break; + // The valid range for avg_row_length is [0,4294967295] case AVG_ROW_LENGTH: sb.append("AVG_ROW_LENGTH = "); - sb.append(r.getPositiveInteger()); + sb.append(r.getLong(0, 4294967295L + 1)); break; case CHECKSUM: sb.append("CHECKSUM = 1"); @@ -212,9 +213,10 @@ private void appendTableOptions() { sb.append("INSERT_METHOD = "); sb.append(Randomly.fromOptions("NO", "FIRST", "LAST")); break; + // The valid range for key_block_size is [0,65535] case KEY_BLOCK_SIZE: sb.append("KEY_BLOCK_SIZE = "); - sb.append(r.getPositiveInteger()); + sb.append(r.getInteger(0, 65535 + 1)); break; case MAX_ROWS: sb.append("MAX_ROWS = "); @@ -336,7 +338,7 @@ private void appendTypeString(MySQLDataType randomType) { if (Randomly.getBoolean()) { sb.append("("); sb.append(Randomly.getNotCachedInteger(0, 255)); // Display width out of range for column 'c0' (max = - // 255) + // 255) sb.append(")"); } break; From 33e2865ff25293a20ce1c85b2995fdc21030760c Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 24 May 2023 02:00:29 +0200 Subject: [PATCH 0772/1284] Materialize: Fix duplicate table creations (#792) See explanation here: https://github.com/MaterializeInc/materialize/issues/19431#issuecomment-1559695135 Fixes the test issues described in https://github.com/sqlancer/sqlancer/pull/785 --- src/sqlancer/materialize/MaterializeProvider.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java index 3e6e1c59f..f35e476f3 100644 --- a/src/sqlancer/materialize/MaterializeProvider.java +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -216,11 +216,15 @@ public SQLConnection createDatabase(MaterializeGlobalState globalState) throws S globalState.getState().logStatement(String.format("\\c %s;", databaseName)); con = DriverManager.getConnection("jdbc:" + testURL, username, password); - // Serializable transaction isolation is much faster than Strict - // Serializable and should guarantee enough for SQLancer: - // https://materialize.com/docs/overview/isolation-level/ try (Statement s = con.createStatement()) { + // Serializable transaction isolation is much faster than Strict + // Serializable and should guarantee enough for SQLancer: + // https://materialize.com/docs/overview/isolation-level/ s.execute("SET transaction_isolation = 'SERIALIZABLE'"); + // Make sure tables still are visible immediately by not using an + // index for them, see + // https://github.com/MaterializeInc/materialize/issues/19431 + s.execute("SET auto_route_introspection_queries = false"); } return new SQLConnection(con); } From f0c7eedd1532fd9d56252bdd5e76a9ec2e504994 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 30 May 2023 09:29:31 +0800 Subject: [PATCH 0773/1284] Fix yugabyte tablegroups in colocated database (#795) * fix: yugabyte error: cannot use tablegroups in a colocated database * fix: delete useless import * fix: column index starts from 1 * fix: change check colocated logic when creating table * fix: add exception handle * style: check statement will not return an error * fix: remove useless null check * fix: PMD violation --- src/sqlancer/yugabyte/ysql/YSQLSchema.java | 12 ++++++++++++ src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java | 6 ++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/YSQLSchema.java b/src/sqlancer/yugabyte/ysql/YSQLSchema.java index 6aba5ac7d..400a34e21 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLSchema.java +++ b/src/sqlancer/yugabyte/ysql/YSQLSchema.java @@ -160,6 +160,18 @@ protected static List getTableColumns(SQLConnection con, String tabl return columns; } + public boolean getDatabaseIsColocated(SQLConnection con) { + try (Statement s = con.createStatement(); ResultSet rs = s.executeQuery("SELECT yb_is_database_colocated();")) { + rs.next(); + String result = rs.getString(1); + // The query will result in a 'f' for a non-colocated database + return !"f".equals(result); + + } catch (SQLException e) { + throw new AssertionError(e); + } + } + public YSQLTables getRandomTableNonEmptyTables() { return new YSQLTables(Randomly.nonEmptySubset(getDatabaseTables())); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java index 09b559e4c..231bf8008 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java @@ -159,8 +159,10 @@ public static void generateWith(StringBuilder sb, YSQLGlobalState globalState, E } } else if (Randomly.getBoolean()) { errors.add("Cannot use TABLEGROUP with TEMP table"); - sb.append(" TABLEGROUP tg").append( - Randomly.getNotCachedInteger(1, (int) YSQLTableGroupGenerator.UNIQUE_TABLEGROUP_COUNTER.get())); + if (!globalState.getSchema().getDatabaseIsColocated(globalState.getConnection())) { + sb.append(" TABLEGROUP tg").append( + Randomly.getNotCachedInteger(1, (int) YSQLTableGroupGenerator.UNIQUE_TABLEGROUP_COUNTER.get())); + } } } From a89d41e5161c272f4ac013adf815f70dc0e94620 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 31 May 2023 11:25:38 +0200 Subject: [PATCH 0774/1284] ClickHouse: set allow_suspicious_indices (#790) * ClickHouse: set allow_suspicious_indices * Fix table creation * It works locally * another try * add comment --- src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index 19ce78d27..3df5c5524 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -98,6 +98,8 @@ public void start() { columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); sb.append(ClickHouseToStringVisitor.asString(expr)); } + // Suppress index sanity checks https://github.com/sqlancer/sqlancer/issues/788 + sb.append(" SETTINGS allow_suspicious_indices=1"); // TODO: PRIMARY KEY } From 7d894b5afbb3e8e2b3c2494ed3ec472646366375 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 1 Jun 2023 23:24:08 -0400 Subject: [PATCH 0775/1284] Remove the redundancy in comparing floating-point values (#797) The canonicalization was first introduced as part of https://github.com/sqlancer/sqlancer/pull/497 and then copied to subsequent implementations. --- src/sqlancer/ComparatorHelper.java | 16 ++++++++++++++++ .../test/tlp/DatabendQueryPartitioningBase.java | 10 ---------- .../DatabendQueryPartitioningDistinctTester.java | 2 +- .../DatabendQueryPartitioningGroupByTester.java | 2 +- .../DatabendQueryPartitioningHavingTester.java | 2 +- .../DatabendQueryPartitioningWhereTester.java | 2 +- .../oracle/tlp/DorisQueryPartitioningBase.java | 16 ---------------- .../DorisQueryPartitioningDistinctTester.java | 2 +- .../tlp/DorisQueryPartitioningGroupByTester.java | 2 +- .../tlp/DorisQueryPartitioningHavingTester.java | 2 +- .../tlp/DorisQueryPartitioningWhereTester.java | 2 +- .../duckdb/test/DuckDBQueryPartitioningBase.java | 16 ---------------- .../DuckDBQueryPartitioningDistinctTester.java | 2 +- .../DuckDBQueryPartitioningGroupByTester.java | 2 +- .../DuckDBQueryPartitioningHavingTester.java | 2 +- .../test/DuckDBQueryPartitioningWhereTester.java | 2 +- .../test/QuestDBQueryPartitioningBase.java | 10 ---------- .../QuestDBQueryPartitioningWhereTester.java | 2 +- 18 files changed, 29 insertions(+), 65 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 880d3ae72..c2fc009ef 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -166,4 +166,20 @@ public static List getCombinedResultSetNoDuplicates(String firstQueryStr return secondResultSet; } + public static String canonicalizeResultValue(String value) { + if (value == null) { + return value; + } + + switch (value) { + case "-0.0": + return "0.0"; + case "-0": + return "0"; + default: + } + + return value; + } + } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 28d516f65..0042f83d6 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -43,15 +42,6 @@ public DatabendQueryPartitioningBase(DatabendGlobalState state) { DatabendErrors.addExpressionErrors(errors); } - public static String canonicalizeResultValue(String value) { - // Rule: -0.0 should be canonicalized to 0.0 - if (Objects.equals(value, "-0.0")) { - return "0.0"; - } - - return value; - } - @Override public void check() throws SQLException { s = state.getSchema(); diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java index 5756a00be..430c5cfc6 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java @@ -41,7 +41,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DatabendQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java index cbac6820b..6ec148ae8 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java @@ -42,7 +42,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DatabendQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java index a6a8bf47e..2eda63ab5 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java @@ -54,7 +54,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DatabendQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index cff5ff1dd..a9ac6a6c3 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -40,7 +40,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DatabendQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java index 5db230854..5a40e3134 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java @@ -41,22 +41,6 @@ public DorisQueryPartitioningBase(DorisGlobalState state) { DorisErrors.addInsertErrors(errors); } - public static String canonicalizeResultValue(String value) { - if (value == null) { - return value; - } - - switch (value) { - case "-0.0": - return "0.0"; - case "-0": - return "0"; - default: - } - - return value; - } - @Override public void check() throws SQLException { s = state.getSchema(); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java index 6cc15fdc8..3bce110ff 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java @@ -39,7 +39,7 @@ public void check() throws SQLException { combinedString.add(unionString); List secondResultSet = ComparatorHelper.getResultSetFirstColumnAsString(unionString, errors, state); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DorisQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java index 0761fddb6..00a0cf28a 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java @@ -43,7 +43,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DorisQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java index 7ce10dc09..2151e1da3 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -53,7 +53,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DorisQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java index c45bc36fb..e373e98b9 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -47,7 +47,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DorisQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 837247f62..2acc6a6d6 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -37,22 +37,6 @@ public DuckDBQueryPartitioningBase(DuckDBGlobalState state) { DuckDBErrors.addExpressionErrors(errors); } - public static String canonicalizeResultValue(String value) { - if (value == null) { - return value; - } - - switch (value) { - case "-0.0": - return "0.0"; - case "-0": - return "0"; - default: - } - - return value; - } - @Override public void check() throws SQLException { s = state.getSchema(); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java index d85d2a2d5..70cd731d6 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java @@ -38,7 +38,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java index fab480fed..86c26370d 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java @@ -41,7 +41,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index aef003fa5..e8988d867 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -47,7 +47,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java index 412ab6ffd..ca9cde670 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java @@ -39,7 +39,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index fa5283507..068abbd38 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -3,7 +3,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -48,15 +47,6 @@ List> generateFetchColumns() { return columns; } - public static String canonicalizeResultValue(String value) { - // Rule: -0.0 should be canonicalized to 0.0 - if (Objects.equals(value, "-0.0")) { - return "0.0"; - } - - return value; - } - @Override protected ExpressionGenerator> getGen() { return gen; diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java index fe5b5890d..ed89e7136 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java @@ -36,6 +36,6 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, false, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, QuestDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } From e4184896679662ce4da8e71aa34a4db56cba45fc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 2 Jun 2023 04:13:11 -0400 Subject: [PATCH 0776/1284] Fix the package declaration for the Materialize QPG test (#796) --- test/sqlancer/qpg/materialize/TestMaterializeQPG.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQPG.java b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java index 0564d048c..431468cee 100644 --- a/test/sqlancer/qpg/materialize/TestMaterializeQPG.java +++ b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java @@ -1,4 +1,4 @@ -package sqlancer.qpg.cockroachdb; +package sqlancer.qpg.materialize; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; From d013225e5d0d0637022c1d14f2d133f86bbb40a0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 6 Jun 2023 22:19:40 -0400 Subject: [PATCH 0777/1284] Add a workaround for issue 799 (#800) --- src/sqlancer/Main.java | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 29610f266..dfdc949d6 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -23,9 +23,32 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.JCommander.Builder; +import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.citus.CitusProvider; +import sqlancer.clickhouse.ClickHouseProvider; +import sqlancer.cnosdb.CnosDBProvider; +import sqlancer.cockroachdb.CockroachDBProvider; import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.cosmos.CosmosProvider; +import sqlancer.databend.DatabendProvider; +import sqlancer.doris.DorisProvider; +import sqlancer.duckdb.DuckDBProvider; +import sqlancer.h2.H2Provider; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.mariadb.MariaDBProvider; +import sqlancer.materialize.MaterializeProvider; +import sqlancer.mongodb.MongoDBProvider; +import sqlancer.mysql.MySQLProvider; +import sqlancer.oceanbase.OceanBaseProvider; +import sqlancer.postgres.PostgresProvider; +import sqlancer.questdb.QuestDBProvider; +import sqlancer.sqlite3.SQLite3Provider; +import sqlancer.tidb.TiDBProvider; +import sqlancer.timescaledb.TimescaleDBProvider; +import sqlancer.yugabyte.ycql.YCQLProvider; +import sqlancer.yugabyte.ysql.YSQLProvider; public final class Main { @@ -609,9 +632,41 @@ private boolean run(MainOptions options, ExecutorService execService, for (DatabaseProvider provider : loader) { providers.add(provider); } + checkForIssue799(providers); return providers; } + // see https://github.com/sqlancer/sqlancer/issues/799 + private static void checkForIssue799(List> providers) { + if (providers.isEmpty()) { + System.err.println( + "No DBMS implementations (i.e., instantiations of the DatabaseProvider class) were found. You likely ran into an issue described in https://github.com/sqlancer/sqlancer/issues/799. As a workaround, I now statically load all supported providers as of June 7, 2023."); + providers.add(new ArangoDBProvider()); + providers.add(new CitusProvider()); + providers.add(new ClickHouseProvider()); + providers.add(new CnosDBProvider()); + providers.add(new CockroachDBProvider()); + providers.add(new CosmosProvider()); + providers.add(new DatabendProvider()); + providers.add(new DorisProvider()); + providers.add(new DuckDBProvider()); + providers.add(new H2Provider()); + providers.add(new HSQLDBProvider()); + providers.add(new MariaDBProvider()); + providers.add(new MaterializeProvider()); + providers.add(new MongoDBProvider()); + providers.add(new MySQLProvider()); + providers.add(new OceanBaseProvider()); + providers.add(new PostgresProvider()); + providers.add(new QuestDBProvider()); + providers.add(new SQLite3Provider()); + providers.add(new TiDBProvider()); + providers.add(new TimescaleDBProvider()); + providers.add(new YCQLProvider()); + providers.add(new YSQLProvider()); + } + } + private static synchronized void startProgressMonitor() { if (progressMonitorStarted) { /* From a4e828b72c744d372b653b4154c2a0ed25d5a176 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 13 Jun 2023 11:52:21 +0800 Subject: [PATCH 0778/1284] fix: rename master to main, add certain constrains (#813) * fix: rename master to main, add certain constrains * feat: add constrains, rename --- .github/workflows/main.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 945d1bc18..3abeeb9e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,10 +3,16 @@ name: ci on: push: branches: - - master + - main + paths: + - 'src/**' + - '.github/workflows/*.yml' pull_request: branches: - - master + - main + paths: + - 'src/**' + - '.github/workflows/*.yml' jobs: misc: From e55fc4448cdbb8bcaf3cfea08d876d0b2b4b319d Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:40:53 +0800 Subject: [PATCH 0779/1284] refactor: refactor MySQLTableGenerator (#812) --- .../mysql/gen/MySQLTableGenerator.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 427f5b7c3..bc0533295 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -18,13 +18,11 @@ import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; public class MySQLTableGenerator { - private final StringBuilder sb = new StringBuilder(); private final boolean allowPrimaryKey; private boolean setPrimaryKey; private final String tableName; private final Randomly r; - private int columnId; private boolean tableHasNullableColumn; private MySQLEngine engine; private int keysSpecified; @@ -65,18 +63,18 @@ private SQLQueryAdapter create() { if (i != 0) { sb.append(", "); } - appendColumn(); + appendColumn(i); } sb.append(")"); sb.append(" "); appendTableOptions(); appendPartitionOptions(); - if ((tableHasNullableColumn || setPrimaryKey) && engine == MySQLEngine.CSV) { + if (engine == MySQLEngine.CSV && (tableHasNullableColumn || setPrimaryKey)) { if (true) { // TODO // results in an error throw new IgnoreMeException(); } - } else if ((tableHasNullableColumn || keysSpecified > 1) && engine == MySQLEngine.ARCHIVE) { + } else if (engine == MySQLEngine.ARCHIVE && (tableHasNullableColumn || keysSpecified > 1)) { errors.add("Too many keys specified; max 1 keys allowed"); errors.add("Table handler doesn't support NULL in given index"); addCommonErrors(errors); @@ -248,27 +246,21 @@ private void appendTableOptions() { } } - private void appendColumn() { + private void appendColumn(int columnId) { String columnName = DBMSCommon.createColumnName(columnId); columns.add(columnName); sb.append(columnName); appendColumnDefinition(); - columnId++; } private enum ColumnOptions { NULL_OR_NOT_NULL, UNIQUE, COMMENT, COLUMN_FORMAT, STORAGE, PRIMARY_KEY } - private void appendColumnDefinition() { - sb.append(" "); - MySQLDataType randomType = MySQLDataType.getRandom(globalState); - boolean isTextType = randomType == MySQLDataType.VARCHAR; - appendTypeString(randomType); - sb.append(" "); + private void appendColumnOption(MySQLDataType type) { + boolean isTextType = type == MySQLDataType.VARCHAR; boolean isNull = false; boolean columnHasPrimaryKey = false; - List columnOptions = Randomly.subset(ColumnOptions.values()); if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { tableHasNullableColumn = true; @@ -324,10 +316,17 @@ private void appendColumnDefinition() { throw new AssertionError(); } } + } + private void appendColumnDefinition() { + sb.append(" "); + MySQLDataType randomType = MySQLDataType.getRandom(globalState); + appendType(randomType); + sb.append(" "); + appendColumnOption(randomType); } - private void appendTypeString(MySQLDataType randomType) { + private void appendType(MySQLDataType randomType) { switch (randomType) { case DECIMAL: sb.append("DECIMAL"); From eb0a65d41a87dc5ad1c7c7a0863d06f8010e3c33 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Fri, 16 Jun 2023 11:56:41 +0800 Subject: [PATCH 0780/1284] tidb: optimize InsertGenerator (#823) * tidb: optimize InsertGenerator * tidb: add expected errors * tidb: add expected error --- src/sqlancer/tidb/TiDBSchema.java | 1 + .../tidb/gen/TiDBAlterTableGenerator.java | 2 ++ src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 2 ++ .../tidb/gen/TiDBInsertGenerator.java | 22 +++++++++---------- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 1 + 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 609d1dbe7..265cacbda 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -239,6 +239,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { break; case "double": case "double(8,6)": // workaround to address https://github.com/sqlancer/sqlancer/issues/669 + case "double(23,16)": size = 8; primitiveType = TiDBDataType.FLOATING; break; diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index b6f65a780..6a95fb9a9 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -30,6 +30,8 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("supported"); errors.add("SQL syntax"); errors.add("can't drop"); + errors.add("A PRIMARY must include all columns in the table's partitioning function"); + errors.add("key was too long"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index cb870a2df..1be2753b6 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -61,6 +61,8 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx } errors.add("Cannot decode index value, because"); // invalid value for generated column errors.add("index already exist"); + errors.add("Data truncation"); + errors.add("key was too long"); return new SQLQueryAdapter(sb.toString(), errors, true); } diff --git a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java index a78d6c2fb..363fa438b 100644 --- a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java @@ -26,11 +26,15 @@ public TiDBInsertGenerator(TiDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - return new TiDBInsertGenerator(globalState).get(); + TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + return new TiDBInsertGenerator(globalState).get(table); } - private SQLQueryAdapter get() { - TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState, TiDBTable table) { + return new TiDBInsertGenerator(globalState).get(table); + } + + private SQLQueryAdapter get(TiDBTable table) { gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); StringBuilder sb = new StringBuilder(); boolean isInsert = Randomly.getBoolean(); @@ -75,18 +79,14 @@ private void insertColumns(StringBuilder sb, List columns) { sb.append(", "); } sb.append("("); - for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { - if (nrColumn != 0) { + int i = 0; + for (TiDBColumn c : columns) { + if (i++ != 0) { sb.append(", "); } - insertValue(sb); + sb.append(TiDBVisitor.asString(gen.generateConstant(c.getType().getPrimitiveDataType()))); } sb.append(")"); } } - - private void insertValue(StringBuilder sb) { - sb.append(gen.generateConstant()); // TODO: try to insert valid data - } - } diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 4349b69ca..af9ad4457 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -33,6 +33,7 @@ public static SQLQueryAdapter createRandomTableStatement(TiDBGlobalState globalS public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { errors.add("Information schema is changed during the execution of the statement"); + errors.add("A CLUSTERED INDEX must include all columns in the table's partitioning function"); String tableName = globalState.getSchema().getFreeTableName(); int nrColumns = Randomly.smallNumber() + 1; allowPrimaryKey = Randomly.getBoolean(); From 0f9eef728108088615318aa33a821be22ea1177a Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Fri, 16 Jun 2023 16:30:53 +0800 Subject: [PATCH 0781/1284] feat: Add virtual db for statement reduction testing (#815) * feat: Add virtual db for statement reduction testing + Implement a prototype of `VirtualDB` so that statement reduction can be tested. + Write some test cases for statement reduction. + Simplify the code of StatementReducer + Add parameter for timeout and max steps of reducer * Correct code of VirtualDB. * Add an extra empty line for StatementReducer.java * Add GitHub Action for reducer tests * format * format 2 * Fix name issues * Do not run reducer test in "general" --------- Co-authored-by: Yichen Yan Co-authored-by: Yichen Yan --- .github/workflows/main.yml | 20 ++- src/sqlancer/MainOptions.java | 15 +++ src/sqlancer/StatementReducer.java | 69 ++++++++--- test/sqlancer/reducer/TestEnvironment.java | 114 ++++++++++++++++++ .../reducer/TestStatementReducer.java | 75 ++++++++++++ .../VirtualDB/VirtualDBConnection.java | 18 +++ .../reducer/VirtualDB/VirtualDBErrors.java | 12 ++ .../VirtualDB/VirtualDBGlobalState.java | 56 +++++++++ .../reducer/VirtualDB/VirtualDBOptions.java | 29 +++++ .../reducer/VirtualDB/VirtualDBProvider.java | 54 +++++++++ .../reducer/VirtualDB/VirtualDBQuery.java | 30 +++++ .../reducer/VirtualDB/VirtualDBSchema.java | 46 +++++++ 12 files changed, 522 insertions(+), 16 deletions(-) create mode 100644 test/sqlancer/reducer/TestEnvironment.java create mode 100644 test/sqlancer/reducer/TestStatementReducer.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3abeeb9e2..1bafaf3d1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,7 +30,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test + run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**,!sqlancer.reducer.**' test - name: Set up Python uses: actions/setup-python@v4 with: @@ -533,3 +533,21 @@ jobs: run: mvn -B package -DskipTests=true - name: Shortly run DuckDB run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb + + reducer: + name: Reducer Tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1.4.4 + with: + java-version: 11 + - name: Build + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + mvn -Dtest=TestStatementReducer test diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 5283ea2a6..c653a358f 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -10,6 +10,7 @@ @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { public static final int NO_SET_PORT = -1; + public static final int NO_REDUCE_LIMIT = -1; public static final MainOptions DEFAULT_OPTIONS = new MainOptions(); @Parameter(names = { "--help", "-h" }, description = "Lists all supported options and commands", help = true) @@ -125,6 +126,12 @@ public class MainOptions { @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") private boolean useReducer = false; // NOPMD + @Parameter(names = "--statement-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the statement reducer will do") + private long maxStatementReduceSteps = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--statement-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the statement reducer will do") + private long maxStatementReduceTime = NO_REDUCE_LIMIT; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -285,4 +292,12 @@ public boolean performConnectionTest() { public boolean useReducer() { return useReducer; } + + public long getMaxStatementReduceSteps() { + return maxStatementReduceSteps; + } + + public long getMaxStatementReduceTime() { + return maxStatementReduceTime; + } } diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java index 14db4ba5b..3d3d36711 100644 --- a/src/sqlancer/StatementReducer.java +++ b/src/sqlancer/StatementReducer.java @@ -1,9 +1,9 @@ package sqlancer; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.function.BiFunction; -import java.util.stream.Collectors; import sqlancer.common.query.Query; @@ -11,49 +11,85 @@ public class StatementReducer, O extends DBMSSpec implements Reducer { private final DatabaseProvider provider; private boolean observedChange; + private int partitionNum; public StatementReducer(DatabaseProvider provider) { this.provider = provider; } + private boolean hasNotReachedLimit(long curr, long limit) { + if (limit == MainOptions.NO_REDUCE_LIMIT) { + return true; + } + return curr < limit; + } + @SuppressWarnings("unchecked") @Override public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + long maxReduceTime = state.getOptions().getMaxStatementReduceTime(); + long maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); + List> knownToReproduceBugStatements = new ArrayList<>(); for (Query stat : state.getState().getStatements()) { knownToReproduceBugStatements.add((Query) stat); } + System.out.println("Starting query:"); printQueries(knownToReproduceBugStatements); System.out.println(); - do { + if (knownToReproduceBugStatements.size() <= 1) { + return; + } + + Instant timeOfReductionBegins = Instant.now(); + long currentReduceSteps = 0; + long currentReduceTime = 0; + partitionNum = 2; + + while (knownToReproduceBugStatements.size() >= 2 && hasNotReachedLimit(currentReduceSteps, maxReduceSteps) + && hasNotReachedLimit(currentReduceTime, maxReduceTime)) { observedChange = false; + knownToReproduceBugStatements = tryReduction(state, reproducer, newGlobalState, - knownToReproduceBugStatements, (candidateStatements, i) -> { - candidateStatements.remove((int) i); - return true; - }); - } while (observedChange); + knownToReproduceBugStatements); + if (!observedChange) { + if (partitionNum == knownToReproduceBugStatements.size()) { + break; + } + // increase the search granularity + partitionNum = Math.min(partitionNum * 2, knownToReproduceBugStatements.size()); + + currentReduceSteps++; + Instant currentInstant = Instant.now(); + currentReduceTime = Duration.between(currentInstant, timeOfReductionBegins).getSeconds(); + } + + } System.out.println("Reduced query:"); printQueries(knownToReproduceBugStatements); + newGlobalState.getState().setStatements(new ArrayList<>(knownToReproduceBugStatements)); } private List> tryReduction(G state, // NOPMD - Reproducer reproducer, G newGlobalState, List> knownToReproduceBugStatements, - BiFunction>, Integer, Boolean> reductionOperation) throws Exception { + Reproducer reproducer, G newGlobalState, List> knownToReproduceBugStatements) throws Exception { List> statements = knownToReproduceBugStatements; - for (int i = 0; i < statements.size(); i++) { + + int start = 0; + int subLength = statements.size() / partitionNum; + while (start < statements.size()) { + // newStatements = candidate[:start] + candidate[start+subLength:] + // in other word, remove [start, start+subLength) from candidates try (C con2 = provider.createDatabase(newGlobalState)) { newGlobalState.setConnection(con2); List> candidateStatements = new ArrayList<>(statements); - if (!reductionOperation.apply(candidateStatements, i)) { - continue; - } - newGlobalState.getState().setStatements(candidateStatements.stream().collect(Collectors.toList())); + candidateStatements.subList(start, start + subLength).clear(); + newGlobalState.getState().setStatements(new ArrayList<>(candidateStatements)); + for (Query s : candidateStatements) { try { s.execute(newGlobalState); @@ -65,6 +101,8 @@ private List> tryReduction(G state, // NOPMD if (reproducer.bugStillTriggers(newGlobalState)) { observedChange = true; statements = candidateStatements; + partitionNum = Math.max(partitionNum - 1, 2); + break; // reproducer.outputHook((SQLite3GlobalState) newGlobalState); // state.getLogger().logReduced(newGlobalState.getState()); } @@ -72,6 +110,7 @@ private List> tryReduction(G state, // NOPMD } } + start = start + subLength; } return statements; } diff --git a/test/sqlancer/reducer/TestEnvironment.java b/test/sqlancer/reducer/TestEnvironment.java new file mode 100644 index 000000000..3329597aa --- /dev/null +++ b/test/sqlancer/reducer/TestEnvironment.java @@ -0,0 +1,114 @@ +package sqlancer.reducer; + +import sqlancer.*; +import sqlancer.common.query.Query; +import sqlancer.reducer.VirtualDB.VirtualDBGlobalState; +import sqlancer.reducer.VirtualDB.VirtualDBProvider; +import sqlancer.reducer.VirtualDB.VirtualDBQuery; + +import java.util.ArrayList; +import java.util.List; +import java.util.ServiceLoader; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * TODO: Make Connection a generic type OR Fake a conn QUERY AND CONNECTION BOTH ARE FAKE. FAKE QUERY sub class + */ +public class TestEnvironment { + private final String databaseName = "virtual_db"; + private final MainOptions options = new MainOptions(); + private VirtualDBProvider provider = null; + private VirtualDBGlobalState state, newGlobalState; + + public TestEnvironment() throws Exception { + setUpTestingEnvironment(); + } + + /** + * @param queries: + * List of Query + * + * @return String of queries that appended together with '\n' separated (no '\n' at the last line) + */ + public static String getQueriesString(List> queries) { + return queries.stream().map(Query::getQueryString).collect(Collectors.joining("\n")); + } + + private VirtualDBGlobalState createGlobalState() { + try { + return provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @SuppressWarnings("rawtypes") + private void initVirtualDBProvider() { + try { + ServiceLoader loader = ServiceLoader.load(DatabaseProvider.class); + for (DatabaseProvider provider : loader) { + if (provider.getDBMSName().equals(databaseName)) { + this.provider = (VirtualDBProvider) provider; + break; + } + } + if (provider == null) { + throw new AssertionError("testing provider not registered"); + } + } catch (Exception e) { + throw new AssertionError(e); + } + + } + + private void setUpTestingEnvironment() throws Exception { + initVirtualDBProvider(); + state = createGlobalState(); + StateToReproduce stateToReproduce = provider.getStateToReproduce(databaseName); + + state.setState(stateToReproduce); + state.setDatabaseName(databaseName); + state.setMainOptions(options); + + // Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + // state.setStateLogger(logger); + + try (SQLConnection con = provider.createDatabase(state)) { + state.setConnection(con); + newGlobalState = createGlobalState(); + // Main.StateLogger newLogger = new Main.StateLogger(databaseName, provider, options); + // newGlobalState.setStateLogger(newLogger); + newGlobalState.setState(stateToReproduce); + newGlobalState.setDatabaseName(databaseName); + newGlobalState.setMainOptions(options); + } + } + + public void setInitialStatementsFromStrings(List statements) { + List> queries = new ArrayList<>(); + for (String s : statements) { + queries.add(new VirtualDBQuery(s)); + } + state.getState().setStatements(queries); + } + + public void setBugInducingCondition(Function>, Boolean> bugInducingCondition) { + state.setBugInducingCondition(bugInducingCondition); + newGlobalState.setBugInducingCondition(bugInducingCondition); + } + + public void runReduce() throws Exception { + Reducer reducer = new StatementReducer<>(provider); + Reproducer reproducer = provider.generateAndTestDatabase(newGlobalState); + reducer.reduce(state, reproducer, newGlobalState); + } + + public List> getReducedStatements() { + return newGlobalState.getState().getStatements(); + } + + public List> getInitialStatements() { + return state.getState().getStatements(); + } +} diff --git a/test/sqlancer/reducer/TestStatementReducer.java b/test/sqlancer/reducer/TestStatementReducer.java new file mode 100644 index 000000000..83c7de94e --- /dev/null +++ b/test/sqlancer/reducer/TestStatementReducer.java @@ -0,0 +1,75 @@ +package sqlancer.reducer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.query.Query; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestStatementReducer { + + @Test + void testSimple() throws Exception { + TestEnvironment env = new TestEnvironment(); + + String[] queriesStr = { "CREATE TABLE FAKE_TABLE;", "SELECT * FROM FAKE_TABLE;", "EXIT", }; + env.setInitialStatementsFromStrings(List.of(queriesStr)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("SELECT"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals(1, reducedResult.size()); + assertEquals("SELECT * FROM FAKE_TABLE;", reducedResult.get(0).toString()); + + } + + @Test + void testDeltaDebugging() throws Exception { + TestEnvironment env = new TestEnvironment(); + List fakeStatements = new ArrayList<>(); + for (int i = 0; i < 10000; i++) { + String statement = "Statement_" + i + ";"; + fakeStatements.add(statement); + } + + env.setInitialStatementsFromStrings(fakeStatements); + env.setBugInducingCondition(statements -> { + String queries = TestEnvironment.getQueriesString(statements); + return queries.contains("Statement_29;"); + }); + + env.runReduce(); + List> reducedQueries = env.getReducedStatements(); + String queriesString = TestEnvironment.getQueriesString(reducedQueries); + assertEquals(queriesString, "Statement_29;"); + } + + @Test + void testDeltaDebuggingWithStatementsCombination() throws Exception { + TestEnvironment env = new TestEnvironment(); + List fakeStatements = new ArrayList<>(); + + String pattern = "(.*\\n)*(Statement_2;)\\n(.*\\n)*(Statement_318);\\n(.*\\n)*(Statement_990;)(.*\\n)*.*"; + for (int i = 0; i < 1000; i++) { + String statement = "Statement_" + i + ";"; + fakeStatements.add(statement); + } + + env.setInitialStatementsFromStrings(fakeStatements); + env.setBugInducingCondition(queryList -> { + String queries = TestEnvironment.getQueriesString(queryList); + return Pattern.matches(pattern, queries); + }); + + env.runReduce(); + List> reducedQueries = env.getReducedStatements(); + String queriesString = TestEnvironment.getQueriesString(reducedQueries); + assertEquals(queriesString, "Statement_2;\nStatement_318;\nStatement_990;"); + } + +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java b/test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java new file mode 100644 index 000000000..6a562a90d --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java @@ -0,0 +1,18 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.SQLConnection; + +import java.sql.Connection; +import java.sql.SQLException; + +public class VirtualDBConnection extends SQLConnection { + + public VirtualDBConnection(Connection connection) { + super(connection); + } + + @Override + public void close() throws SQLException { + + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java b/test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java new file mode 100644 index 000000000..64c783c11 --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java @@ -0,0 +1,12 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.common.query.ExpectedErrors; + +public final class VirtualDBErrors { + public VirtualDBErrors() { + } + + public static void addErrors(ExpectedErrors errors) { + errors.add("Default error"); + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java b/test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java new file mode 100644 index 000000000..a0548495c --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java @@ -0,0 +1,56 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.Query; + +import java.util.List; +import java.util.function.Function; + +@SuppressWarnings("all") +public class VirtualDBGlobalState extends SQLGlobalState { + + private SQLConnection virtualConn = new SQLConnection(null); + private StringBuilder queriesStringBuilder = new StringBuilder(); + private Function>, Boolean> bugInducingCondition = null; + + public Function>, Boolean> getBugInducingCondition() { + return bugInducingCondition; + } + + public void setBugInducingCondition(Function>, Boolean> condition) { + bugInducingCondition = (condition); + } + + @Override + protected VirtualDBSchema readSchema() throws Exception { + return null; + } + + @Override + public SQLConnection getConnection() { + // It's a fake engine, so the connection would not be available :) + return virtualConn; + } + + @Override + public void setConnection(SQLConnection con) { + // A fake connection could also not be closed. + // So nothing would be done here. + // And reset the query String (Seems needless) + // queriesStringBuilder = new StringBuilder(); + } + + // public String getCurrentQueriesString() { + // return queriesStringBuilder.toString(); + // } + + @Override + public boolean executeStatement(Query q, String... fills) throws Exception { + if (queriesStringBuilder.length() != 0) { + queriesStringBuilder.append("\n"); + } + queriesStringBuilder.append(q.getQueryString()); + return true; + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java b/test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java new file mode 100644 index 000000000..72cbe918c --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java @@ -0,0 +1,29 @@ +package sqlancer.reducer.VirtualDB; + +import com.beust.jcommander.Parameters; +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.reducer.VirtualDB.VirtualDBOptions.VirtualDBFactory; + +import java.util.ArrayList; +import java.util.List; + +@Parameters(separators = "=", commandDescription = "VirtualDB (default port: " + "-1" + ", default host: " + "127.0.0.1" + + ")") +public class VirtualDBOptions implements DBMSSpecificOptions { + + List factories = new ArrayList<>(); + + @Override + public List getTestOracleFactory() { + return factories; + } + + public static class VirtualDBFactory implements OracleFactory { + @Override + public TestOracle create(VirtualDBGlobalState globalState) throws Exception { + return null; + } + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java b/test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java new file mode 100644 index 000000000..697e5c60e --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java @@ -0,0 +1,54 @@ +package sqlancer.reducer.VirtualDB; + +import com.google.auto.service.AutoService; +import sqlancer.DatabaseProvider; +import sqlancer.Reproducer; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; + +@AutoService(DatabaseProvider.class) +public class VirtualDBProvider extends SQLProviderAdapter { + + private Reproducer reproducerForTesting; + + public VirtualDBProvider() { + super(VirtualDBGlobalState.class, VirtualDBOptions.class); + } + + @Override + public SQLConnection createDatabase(VirtualDBGlobalState globalState) throws Exception { + return new VirtualDBConnection(null); + } + + @Override + public String getDBMSName() { + return "virtual_db"; + } + + @Override + public void generateDatabase(VirtualDBGlobalState globalState) throws Exception { + + } + + @Override + public Reproducer generateAndTestDatabase(VirtualDBGlobalState globalState) throws Exception { + return state -> { + if (globalState.getBugInducingCondition() == null) + return false; + return globalState.getBugInducingCondition().apply(globalState.getState().getStatements()); + }; + } + + @Override + public Class getGlobalStateClass() { + return super.getGlobalStateClass(); + } + + public Reproducer getReproducerForTesting() { + return reproducerForTesting; + } + + public void setReproducerForTesting(Reproducer reproducer) { + this.reproducerForTesting = reproducer; + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java new file mode 100644 index 000000000..37389109f --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java @@ -0,0 +1,30 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; + +import java.sql.SQLException; + +public class VirtualDBQuery extends SQLQueryAdapter { + + public VirtualDBQuery(String query) { + // Since the base class must check the format + // We judge if the statement could affect schema. A bit hacky tho. + super(query, (query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN"))); + } + + public VirtualDBQuery(String query, boolean couldAffectSchema) { + super(query, couldAffectSchema); + } + + @Override + public > boolean execute(G globalState, String... fills) + throws SQLException { + try { + return globalState.executeStatement(this, fills); + } catch (Exception e) { + throw new SQLException(e); + } + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java b/test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java new file mode 100644 index 000000000..9ea5b422e --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java @@ -0,0 +1,46 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.reducer.VirtualDB.VirtualDBSchema.VirtualDBTable; + +import java.util.List; + +public class VirtualDBSchema extends AbstractSchema { + + public VirtualDBSchema(List databaseTables) { + super(databaseTables); + } + + public static class VirtualDBTable extends AbstractTable { + protected VirtualDBTable(String name, List columns, List indexes, + boolean isView) { + super(name, columns, indexes, isView); + } + + @Override + public long getNrRows(VirtualDBGlobalState globalState) { + return 0; + } + } + + public static class VirtualDBIndex extends TableIndex { + + protected VirtualDBIndex(String indexName) { + super(indexName); + } + } + + public static class VirtualDBDataType { + + } + + public static class VirtualDBColumn extends AbstractTableColumn { + + public VirtualDBColumn(String name, VirtualDBTable table, VirtualDBDataType type) { + super(name, table, type); + } + } +} From 10d603f074293ee203d04461cebd848e3ae3a43a Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Jun 2023 11:39:29 +0800 Subject: [PATCH 0782/1284] tidb: support cross join and on condition (#827) * tidb: optimize InsertGenerator * tidb: add expected errors * tidb: add expected error * tidb: cross join and on condition * tidb: bypass bug 44747 --- src/sqlancer/tidb/TiDBBugs.java | 3 +++ src/sqlancer/tidb/TiDBErrors.java | 3 +++ src/sqlancer/tidb/ast/TiDBJoin.java | 21 ++++++++++++++++--- .../tidb/visitor/TiDBToStringVisitor.java | 14 ++++++------- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index d56629e44..4aef122e3 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -14,6 +14,9 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/38295 public static boolean bug38295 = true; + // https://github.com/pingcap/tidb/issues/44747 + public static boolean bug44747 = true; + private TiDBBugs() { } diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index c2db11757..2068f9f1e 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -52,6 +52,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (TiDBBugs.bug38295) { errors.add("assertion failed"); } + if (TiDBBugs.bug44747) { + errors.add("index out of range"); + } } public static void addExpressionHavingErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/tidb/ast/TiDBJoin.java b/src/sqlancer/tidb/ast/TiDBJoin.java index e22b34b6a..dc78b0020 100644 --- a/src/sqlancer/tidb/ast/TiDBJoin.java +++ b/src/sqlancer/tidb/ast/TiDBJoin.java @@ -12,12 +12,12 @@ public class TiDBJoin implements TiDBExpression { private final TiDBExpression leftTable; private final TiDBExpression rightTable; - private final JoinType joinType; - private final TiDBExpression onCondition; + private JoinType joinType; + private TiDBExpression onCondition; private NaturalJoinType outerType; public enum JoinType { - INNER, NATURAL, STRAIGHT, LEFT, RIGHT; + NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; public static JoinType getRandom() { return Randomly.fromOptions(values()); @@ -52,10 +52,18 @@ public JoinType getJoinType() { return joinType; } + public void setJoinType(JoinType joinType) { + this.joinType = joinType; + } + public TiDBExpression getOnCondition() { return onCondition; } + public static TiDBJoin createCrossJoin(TiDBExpression left, TiDBExpression right, TiDBExpression onClause) { + return new TiDBJoin(left, right, JoinType.CROSS, onClause); + } + public static TiDBJoin createNaturalJoin(TiDBExpression left, TiDBExpression right, NaturalJoinType type) { TiDBJoin tiDBJoin = new TiDBJoin(left, right, JoinType.NATURAL, null); tiDBJoin.setNaturalJoinType(type); @@ -110,6 +118,9 @@ public static List getJoins(List tableList, TiDB case RIGHT: joinExpressions.add(TiDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); break; + case CROSS: + joinExpressions.add(TiDBJoin.createCrossJoin(leftTable, rightTable, null)); + break; default: throw new AssertionError(); } @@ -117,4 +128,8 @@ public static List getJoins(List tableList, TiDB return joinExpressions; } + public void setOnCondition(TiDBExpression generateExpression) { + this.onCondition = generateExpression; + } + } diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index 8bd47ed70..e9c199407 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -95,11 +95,7 @@ public void visit(TiDBJoin join) { sb.append(" "); switch (join.getJoinType()) { case INNER: - if (Randomly.getBoolean()) { - sb.append("INNER "); - } else { - sb.append("CROSS "); - } + sb.append("INNER "); sb.append("JOIN "); break; case LEFT: @@ -135,13 +131,15 @@ public void visit(TiDBJoin join) { } sb.append("JOIN "); break; + case CROSS: + sb.append("CROSS JOIN "); + break; default: throw new AssertionError(); } visit(join.getRightTable()); - sb.append(" "); - if (join.getJoinType() != JoinType.NATURAL) { - sb.append("ON "); + if (join.getOnCondition() != null && join.getJoinType() != JoinType.NATURAL) { + sb.append(" ON "); visit(join.getOnCondition()); } } From 34355e6e32a24613a23459dc76dc5daf8b35beeb Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Jun 2023 22:57:55 +0800 Subject: [PATCH 0783/1284] mysql: bypass 111471 (#828) * mysql: bypass 111471 * formatting --- src/sqlancer/mysql/MySQLBugs.java | 3 +++ src/sqlancer/mysql/MySQLErrors.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 9cd87614d..ea44c83d4 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -19,6 +19,9 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=99135 public static boolean bug99135 = true; + // https://bugs.mysql.com/bug.php?id=111471 + public static boolean bug111471 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 39aa33e1e..540a5c5db 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -10,6 +10,10 @@ private MySQLErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) errors.add("is not valid for CHARACTER SET"); + + if (MySQLBugs.bug111471) { + errors.add("Memory capacity exceeded"); + } } } From 0de1c3757f96d9b5f9c5ecc2f1026bb5c8998cf3 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Jun 2023 15:28:16 +0800 Subject: [PATCH 0784/1284] tidb: enable non-empty tables (#829) * tidb: enable non-empty tables * typo --- src/sqlancer/tidb/TiDBProvider.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 91223a246..cf5421e7c 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -5,6 +5,8 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -21,6 +23,7 @@ import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; import sqlancer.tidb.gen.TiDBDeleteGenerator; @@ -84,6 +87,7 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { case CREATE_INDEX: return r.getInteger(0, 2); case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case TRUNCATE: case DELETE: case ADMIN_CHECKSUM_TABLE: @@ -213,4 +217,15 @@ protected void executeMutator(int index, TiDBGlobalState globalState) throws Exc globalState.executeStatement(queryMutateTable); } + @Override + public boolean addRowsToAllTables(TiDBGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (TiDBTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = TiDBInsertGenerator.getQuery(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } From 6ebbaae57b56f492ee728adb711a218631939aed Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Wed, 21 Jun 2023 19:51:45 +0800 Subject: [PATCH 0785/1284] tidb: cert init (#831) * tidb: utilities for CERT * tidb: CERT init * tidb: formatting * tidb: disable natural join for CERT * tidb: enum in CERT * rename editDistance * tidb: check oracle num * wrap editDistance --- src/sqlancer/common/DBMSCommon.java | 27 ++ src/sqlancer/common/ast/SelectBase.java | 8 + src/sqlancer/tidb/TiDBOptions.java | 12 + src/sqlancer/tidb/TiDBProvider.java | 18 ++ src/sqlancer/tidb/ast/TiDBJoin.java | 33 +++ src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 293 +++++++++++++++++++ 6 files changed, 391 insertions(+) create mode 100644 src/sqlancer/tidb/oracle/TiDBCERTOracle.java diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java index f302d48ad..4478be2b3 100644 --- a/src/sqlancer/common/DBMSCommon.java +++ b/src/sqlancer/common/DBMSCommon.java @@ -1,5 +1,6 @@ package sqlancer.common; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -39,4 +40,30 @@ public static int getMaxIndexInDoubleArray(double... doubleArray) { } return maxIndex; } + + public static boolean areQueryPlanSequencesSimilar(List list1, List list2) { + return editDistance(list1, list2) <= 1; + } + + public static int editDistance(List list1, List list2) { + int[][] dp = new int[list1.size() + 1][list2.size() + 1]; + for (int i = 0; i <= list1.size(); i++) { + for (int j = 0; j <= list2.size(); j++) { + if (i == 0) { + dp[i][j] = j; + } else if (j == 0) { + dp[i][j] = i; + } else { + dp[i][j] = Math.min(dp[i - 1][j - 1] + costOfSubstitution(list1.get(i - 1), list2.get(j - 1)), + Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1)); + } + } + } + return dp[list1.size()][list2.size()]; + } + + private static int costOfSubstitution(String string, String string2) { + return string.equals(string2) ? 0 : 1; + } + } diff --git a/src/sqlancer/common/ast/SelectBase.java b/src/sqlancer/common/ast/SelectBase.java index 341b00209..9e02a7f61 100644 --- a/src/sqlancer/common/ast/SelectBase.java +++ b/src/sqlancer/common/ast/SelectBase.java @@ -50,6 +50,10 @@ public void setGroupByExpressions(List groupByExpressions) { this.groupByExpressions = groupByExpressions; } + public void clearGroupByExpressions() { + this.groupByExpressions = Collections.emptyList(); + } + public List getGroupByExpressions() { assert groupByExpressions != null; return groupByExpressions; @@ -83,6 +87,10 @@ public T getHavingClause() { return havingClause; } + public void clearHavingClause() { + this.havingClause = null; + } + public void setLimitClause(T limitClause) { this.limitClause = limitClause; } diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 6fbab6ba8..7ff20027d 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -14,6 +14,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.oracle.TiDBCERTOracle; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @@ -56,6 +57,17 @@ public TestOracle create(TiDBGlobalState globalState) throws SQ oracles.add(new TiDBTLPHavingOracle(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + CERT { + @Override + public TestOracle create(TiDBGlobalState globalState) throws SQLException { + return new TiDBCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } }; } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index cf5421e7c..3701a4006 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -19,6 +19,7 @@ import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; @@ -35,6 +36,7 @@ import sqlancer.tidb.gen.TiDBTableGenerator; import sqlancer.tidb.gen.TiDBUpdateGenerator; import sqlancer.tidb.gen.TiDBViewGenerator; +import sqlancer.tidb.oracle.TiDBCERTOracle; @AutoService(DatabaseProvider.class) public class TiDBProvider extends SQLProviderAdapter { @@ -136,6 +138,22 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { throw new AssertionError(e); } } + + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 && globalState + .getDbmsSpecificOptions().getTestOracleFactory().get(0).create(globalState) instanceof TiDBCERTOracle) { + // Disable strict Group By constraints for ROW oracle + globalState.executeStatement(new SQLQueryAdapter( + "SET @@sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';")); + + // Enfore statistic collected for all tables + ExpectedErrors errors = new ExpectedErrors(); + TiDBErrors.addExpressionErrors(errors); + for (TiDBTable table : globalState.getSchema().getDatabaseTables()) { + if (!table.isView()) { + globalState.executeStatement(new SQLQueryAdapter("ANALYZE TABLE " + table.getName() + ";", errors)); + } + } + } } @Override diff --git a/src/sqlancer/tidb/ast/TiDBJoin.java b/src/sqlancer/tidb/ast/TiDBJoin.java index dc78b0020..e48c7a6f7 100644 --- a/src/sqlancer/tidb/ast/TiDBJoin.java +++ b/src/sqlancer/tidb/ast/TiDBJoin.java @@ -128,6 +128,39 @@ public static List getJoins(List tableList, TiDB return joinExpressions; } + public static List getJoinsWithoutNature(List tableList, + TiDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBoolean()) { + TiDBTableReference leftTable = (TiDBTableReference) tableList.remove(0); + TiDBTableReference rightTable = (TiDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + TiDBExpressionGenerator joinGen = new TiDBExpressionGenerator(globalState).setColumns(columns); + switch (TiDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(TiDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case STRAIGHT: + joinExpressions.add(TiDBJoin.createStraightJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case LEFT: + joinExpressions.add(TiDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions.add(TiDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + case CROSS: + joinExpressions.add(TiDBJoin.createCrossJoin(leftTable, rightTable, null)); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + public void setOnCondition(TiDBExpression generateExpression) { this.onCondition = generateExpression; } diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java new file mode 100644 index 000000000..d23789b26 --- /dev/null +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -0,0 +1,293 @@ +package sqlancer.tidb.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.tidb.TiDBErrors; +import sqlancer.tidb.TiDBExpressionGenerator; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBDataType; +import sqlancer.tidb.TiDBSchema.TiDBTables; +import sqlancer.tidb.ast.TiDBBinaryLogicalOperation; +import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; +import sqlancer.tidb.ast.TiDBColumnReference; +import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBJoin.JoinType; +import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableReference; +import sqlancer.tidb.visitor.TiDBVisitor; + +public class TiDBCERTOracle implements TestOracle { + private final TiDBGlobalState state; + private TiDBExpressionGenerator gen; + private final ExpectedErrors errors = new ExpectedErrors(); + private TiDBSelect select; + private List queryPlan1Sequences; + private List queryPlan2Sequences; + + public enum Mutator { + JOIN, WHERE, GROUPBY, HAVING, AND, OR, LIMIT; + + public static Mutator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public TiDBCERTOracle(TiDBGlobalState globalState) { + state = globalState; + TiDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + queryPlan1Sequences = new ArrayList<>(); + queryPlan2Sequences = new ArrayList<>(); + + // Randomly generate a query + TiDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new TiDBExpressionGenerator(state).setColumns(tables.getColumns()); + select = new TiDBSelect(); + + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new TiDBColumnReference(c)) + .collect(Collectors.toList())); + select.setFetchColumns(fetchColumns); + + List tableList = tables.getTables().stream().map(t -> new TiDBTableReference(t)) + .collect(Collectors.toList()); + List joins = TiDBJoin.getJoinsWithoutNature(tableList, state); + select.setJoinList(joins); + select.setFromList(tableList); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(select.getFetchColumns()); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBoolean()) { + select.setLimitClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(gen.generateExpression()); + } + + // Get the result of the first query + String queryString1 = TiDBVisitor.asString(select); + double rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + // Mutate the query + boolean increase = false; + + Mutator mutation = Mutator.getRandom(); + switch (mutation) { + case JOIN: + increase = mutateJoin(); + break; + case WHERE: + increase = mutateWhere(); + break; + case GROUPBY: + increase = mutateGroupBy(); + break; + case HAVING: + increase = mutateHaving(); + break; + case AND: + increase = mutateAnd(); + break; + case OR: + increase = mutateOr(); + break; + case LIMIT: + increase = mutateLimit(); + break; + default: + throw new AssertionError(); + } + + // Get the result of the second query + String queryString2 = TiDBVisitor.asString(select); + double rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + // Check structural equivalence + if (!DBMSCommon.areQueryPlanSequencesSimilar(queryPlan1Sequences, queryPlan2Sequences)) { + return; + } + + /* + * https://github.com/pingcap/tidb/issues/38474 A minor issue in TiDB that some operations would round the + * result while others would not. A false alarm happens when both queries have the same number of estimated rows + * but the restrictued one rounds up to a bigger number. To avoid this issue until it is fixed, we make sure the + * gap between two estimated rows is at least 1. + */ + // Check the results + if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { + throw new AssertionError("Mutator: " + mutation + ", Inconsistent result for query: EXPLAIN " + queryString1 + + "; --" + rowCount1 + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + } + } + + private boolean mutateJoin() { + if (select.getJoinList().isEmpty()) { + return false; + } + TiDBJoin join = (TiDBJoin) Randomly.fromList(select.getJoinList()); + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((TiDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((TiDBTableReference) join.getRightTable()).getTable().getColumns()); + TiDBExpressionGenerator joinGen2 = new TiDBExpressionGenerator(state).setColumns(columns); + join.setOnCondition(joinGen2.generateExpression()); + } + + JoinType newJoinType = TiDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation + // between LEFT and RIGHT + // join + newJoinType = Randomly.fromOptions(Arrays.stream(JoinType.values()) + .filter(j -> j != JoinType.LEFT && j != JoinType.RIGHT && j != JoinType.NATURAL) + .toArray(JoinType[]::new)); + } else { + newJoinType = Randomly.fromOptions(Arrays.stream(JoinType.values()) + .filter(j -> j != join.getJoinType() && j != JoinType.NATURAL).toArray(JoinType[]::new)); + } + + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + if (newJoinType == JoinType.CROSS) { + join.setOnCondition(null); + } + return increase; + } + + private boolean mutateWhere() { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(gen.generateExpression()); + } + return increase; + } + + private boolean mutateGroupBy() { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + select.clearHavingClause(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + private boolean mutateHaving() { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(gen.generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(gen.generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + private boolean mutateAnd() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression()); + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), gen.generateExpression(), + TiDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + private boolean mutateOr() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression()); + return false; + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), gen.generateExpression(), + TiDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + private boolean mutateLimit() { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(gen.generateConstant(TiDBDataType.INT)); + } + return increase; + } + + private double getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + throws AssertionError, SQLException { + double row = -1; + String explainQuery = "EXPLAIN " + selectStr; + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + double estRows = Double.parseDouble(rs.getString(2)); + if (row == -1) { + row = estRows; + } + String operation = rs.getString(1).split("_")[0]; // Extract operation names for query plans + queryPlanSequences.add(operation); + return estRows; + } + } + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + if (row == -1) { + throw new IgnoreMeException(); + } + return row; + } +} From 5208d661fdf00c3ac3a2a254a0ce05cc1b37ad68 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Fri, 23 Jun 2023 18:50:22 +0800 Subject: [PATCH 0786/1284] mysql: optimize code for CERT (#835) * mysql: remove dead code * mysql: add interfaces for CERT * mysql: code formatting --- src/sqlancer/mysql/MySQLProvider.java | 14 +++++++++ src/sqlancer/mysql/ast/MySQLSelect.java | 4 +++ .../mysql/gen/MySQLExpressionGenerator.java | 31 +------------------ .../mysql/gen/MySQLInsertGenerator.java | 13 +++++--- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 3 -- 5 files changed, 28 insertions(+), 37 deletions(-) diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 0f1704bf3..c1279f281 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -4,6 +4,8 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -18,6 +20,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; import sqlancer.mysql.gen.MySQLDeleteGenerator; import sqlancer.mysql.gen.MySQLDropIndex; @@ -188,4 +191,15 @@ public String getDBMSName() { return "mysql"; } + @Override + public boolean addRowsToAllTables(MySQLGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (MySQLTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = MySQLInsertGenerator.insertRow(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index 7abd1f639..3ed6df405 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -18,6 +18,10 @@ public void setSelectType(SelectType fromOptions) { this.setFromOptions(fromOptions); } + public SelectType getSelectType() { + return fromOptions; + } + public SelectType getFromOptions() { return fromOptions; } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 7911a0f3e..c3d754f58 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -65,10 +65,6 @@ public MySQLExpression generateExpression(int depth) { case UNARY_PREFIX_OPERATION: MySQLExpression subExpr = generateExpression(depth + 1); MySQLUnaryPrefixOperator random = MySQLUnaryPrefixOperator.getRandom(); - if (random == MySQLUnaryPrefixOperator.MINUS) { - // workaround for https://bugs.mysql.com/bug.php?id=99122 - throw new IgnoreMeException(); - } return new MySQLUnaryPrefixOperation(subExpr, random); case UNARY_POSTFIX: return new MySQLUnaryPostfixOperation(generateExpression(depth + 1), @@ -156,34 +152,9 @@ public MySQLExpression generateConstant() { case STRING: /* Replace characters that still trigger open bugs in MySQL */ String string = state.getRandomly().getString().replace("\\", "").replace("\n", ""); - if (string.startsWith("\n")) { - // workaround for https://bugs.mysql.com/bug.php?id=99130 - throw new IgnoreMeException(); - } - if (string.startsWith("-0") || string.startsWith("0.") || string.startsWith(".")) { - // https://bugs.mysql.com/bug.php?id=99145 - throw new IgnoreMeException(); - } - MySQLConstant createStringConstant = MySQLConstant.createStringConstant(string); - // if (Randomly.getBoolean()) { - // return new MySQLCollate(createStringConstant, - // Randomly.fromOptions("ascii_bin", "binary")); - // } - if (string.startsWith("1e")) { - // https://bugs.mysql.com/bug.php?id=99146 - throw new IgnoreMeException(); - } - return createStringConstant; + return MySQLConstant.createStringConstant(string); case DOUBLE: double val = state.getRandomly().getDouble(); - if (Math.abs(val) <= 1 && val != 0) { - // https://bugs.mysql.com/bug.php?id=99145 - throw new IgnoreMeException(); - } - if (Math.abs(val) > 1.0E30) { - // https://bugs.mysql.com/bug.php?id=99146 - throw new IgnoreMeException(); - } return new MySQLDoubleConstant(val); default: throw new AssertionError(); diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 496600905..14994a140 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -19,16 +19,21 @@ public class MySQLInsertGenerator { private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; - public MySQLInsertGenerator(MySQLGlobalState globalState) { + public MySQLInsertGenerator(MySQLGlobalState globalState, MySQLTable table) { this.globalState = globalState; - table = globalState.getSchema().getRandomTable(); + this.table = table; } public static SQLQueryAdapter insertRow(MySQLGlobalState globalState) throws SQLException { + MySQLTable table = globalState.getSchema().getRandomTable(); + return insertRow(globalState, table); + } + + public static SQLQueryAdapter insertRow(MySQLGlobalState globalState, MySQLTable table) throws SQLException { if (Randomly.getBoolean()) { - return new MySQLInsertGenerator(globalState).generateInsert(); + return new MySQLInsertGenerator(globalState, table).generateInsert(); } else { - return new MySQLInsertGenerator(globalState).generateReplace(); + return new MySQLInsertGenerator(globalState, table).generateReplace(); } } diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 00a477714..546962fe6 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -66,9 +66,6 @@ private enum Action { MAX_SP_RECURSION_DEPTH("max_sp_recursion_depth", (r) -> r.getLong(0, 255), Scope.GLOBAL, Scope.SESSION), // MYISAM_DATA_POINTER_SIZE("myisam_data_pointer_size", (r) -> r.getLong(2, 7), Scope.GLOBAL), // MYISAM_MAX_SORT_FILE_SIZE("myisam_max_sort_file_size", (r) -> r.getLong(0, 9223372036854775807L), Scope.GLOBAL), // - // MYISAM_REPAIR_THREADS("myisam_repair_threads", (r) -> r.getLong(1, Long.MAX_VALUE), Scope.GLOBAL, - // Scope.SESSION), // comment out this to avoid java.sql.SQLException: Unknown system variable - // 'myisam_repair_threads' MYISAM_SORT_BUFFER_SIZE("myisam_sort_buffer_size", (r) -> r.getLong(4096, Long.MAX_VALUE), Scope.GLOBAL, Scope.SESSION), // MYISAM_STATS_METHOD("myisam_stats_method", From 1f80de16e38c4a144e8dabbcef00d07cd15470f9 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Fri, 23 Jun 2023 21:50:23 +0800 Subject: [PATCH 0787/1284] Refactor: improve debugging experience (#819) * Update dependabot.yml * refactor: refactor ComparatorHelper.java and improve debugging experience * style: run code format * fix: manually drop a previous useless commit * refactor: keep minial change * refactor: keep minial change * refactor: different logging and assertion message * refactor: different logging and assertion message * naming: change log message --- src/sqlancer/ComparatorHelper.java | 35 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index c2fc009ef..41d2cf996 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -60,7 +60,7 @@ public static List getResultSetFirstColumnAsString(String queryString, E String resultTemp = result.getString(1); if (resultTemp != null) { resultTemp = resultTemp.replaceAll("[\\.]0+$", ""); // Remove the trailing zeros as many DBMS treat - // it as non-bugs + // it as non-bugs } resultSet.add(resultTemp); } @@ -87,13 +87,19 @@ public static List getResultSetFirstColumnAsString(String queryString, E public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, String originalQueryString, List combinedString, SQLGlobalState state) { if (resultSet.size() != secondResultSet.size()) { - String queryFormatString = "-- %s;\n-- cardinality: %d"; + String queryFormatString = "-- %s;" + System.lineSeparator() + "-- cardinality: %d" + + System.lineSeparator(); String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); - String secondQueryString = String.format(queryFormatString, - combinedString.stream().collect(Collectors.joining(";")), secondResultSet.size()); - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", - resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); + String combinedQueryString = String.join(";", combinedString); + String secondQueryString = String.format(queryFormatString, combinedQueryString, secondResultSet.size()); + state.getState().getLocalState() + .log(String.format("%s" + System.lineSeparator() + "%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format( + "The size of the result sets mismatch (%d and %d)!" + System.lineSeparator() + + "First query: \"%s\", whose cardinality is: %d" + System.lineSeparator() + + "Second query:\"%s\", whose cardinality is: %d", + resultSet.size(), secondResultSet.size(), originalQueryString, resultSet.size(), + combinedQueryString, secondResultSet.size()); throw new AssertionError(assertionMessage); } @@ -105,14 +111,17 @@ public static void assumeResultSetsAreEqual(List resultSet, List firstResultSetMisses.removeAll(secondHashSet); Set secondResultSetMisses = new HashSet<>(secondHashSet); secondResultSetMisses.removeAll(firstHashSet); - String queryFormatString = "-- %s;\n-- misses: %s"; + + String queryFormatString = "-- Query: \"%s\"; It misses: \"%s\""; String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); - String secondQueryString = String.format(queryFormatString, - combinedString.stream().collect(Collectors.joining(";")), secondResultSetMisses); + String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), + secondResultSetMisses); // update the SELECT queries to be logged at the bottom of the error log file - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", - firstQueryString, secondQueryString); + state.getState().getLocalState() + .log(String.format("%s" + System.lineSeparator() + "%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format("The content of the result sets mismatch!" + System.lineSeparator() + + "First query : \"%s\"" + System.lineSeparator() + "Second query: \"%s\"", originalQueryString, + secondQueryString); throw new AssertionError(assertionMessage); } } From 3d37edc805dfa41454ee2a3d5e0ccd3effc298a7 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 26 Jun 2023 08:56:25 +0800 Subject: [PATCH 0788/1284] mysql: cert init (#837) * mysql: cert init * CERT oracle base class * mysql: stream API optimization * cert: abstract mutate method * cert: getRandomExcept --- .../common/oracle/CERTOracleBase.java | 88 +++++++ src/sqlancer/mysql/MySQLOptions.java | 14 +- src/sqlancer/mysql/MySQLProvider.java | 21 ++ src/sqlancer/mysql/ast/MySQLSelect.java | 4 - .../mysql/oracle/MySQLCERTOracle.java | 217 ++++++++++++++++++ src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 72 ++---- 6 files changed, 359 insertions(+), 57 deletions(-) create mode 100644 src/sqlancer/common/oracle/CERTOracleBase.java create mode 100644 src/sqlancer/mysql/oracle/MySQLCERTOracle.java diff --git a/src/sqlancer/common/oracle/CERTOracleBase.java b/src/sqlancer/common/oracle/CERTOracleBase.java new file mode 100644 index 000000000..42e8e5833 --- /dev/null +++ b/src/sqlancer/common/oracle/CERTOracleBase.java @@ -0,0 +1,88 @@ +package sqlancer.common.oracle; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; + +public abstract class CERTOracleBase> implements TestOracle { + + protected final S state; + protected final ExpectedErrors errors; + protected List queryPlan1Sequences; + protected List queryPlan2Sequences; + + protected enum Mutator { + JOIN, DISTINCT, WHERE, GROUPBY, HAVING, AND, OR, LIMIT; + + public static Mutator getRandomExcept(Mutator... exclude) { + Mutator[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(Mutator[]::new); + return Randomly.fromOptions(values); + } + } + + protected CERTOracleBase(S state) { + this.state = state; + this.errors = new ExpectedErrors(); + } + + protected boolean mutate(Mutator... exclude) { + Mutator m = Mutator.getRandomExcept(exclude); + switch (m) { + case JOIN: + return mutateJoin(); + case DISTINCT: + return mutateDistinct(); + case WHERE: + return mutateWhere(); + case GROUPBY: + return mutateGroupBy(); + case HAVING: + return mutateHaving(); + case AND: + return mutateAnd(); + case OR: + return mutateOr(); + case LIMIT: + return mutateLimit(); + default: + throw new AssertionError(m); + } + } + + protected boolean mutateJoin() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateDistinct() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateWhere() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateGroupBy() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateHaving() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateAnd() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateOr() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateLimit() { + throw new UnsupportedOperationException(); + } + +} diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index ad8060090..2d9e80bc0 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -11,6 +11,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; +import sqlancer.mysql.oracle.MySQLCERTOracle; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @@ -45,7 +46,18 @@ public boolean requiresAllTablesToContainRows() { return true; } - } + }, + CERT { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }; } @Override diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index c1279f281..0c3a1f2e7 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -18,8 +18,10 @@ import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; import sqlancer.mysql.gen.MySQLDeleteGenerator; @@ -36,6 +38,7 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLChecksum; import sqlancer.mysql.gen.tblmaintenance.MySQLOptimize; import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; +import sqlancer.mysql.oracle.MySQLCERTOracle; @AutoService(DatabaseProvider.class) public class MySQLProvider extends SQLProviderAdapter { @@ -153,6 +156,24 @@ public void generateDatabase(MySQLGlobalState globalState) throws Exception { } }); se.executeStatements(); + + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 + && globalState.getDbmsSpecificOptions().getTestOracleFactory().get(0) + .create(globalState) instanceof MySQLCERTOracle) { + // Enfore statistic collected for all tables + ExpectedErrors errors = new ExpectedErrors(); + MySQLErrors.addExpressionErrors(errors); + for (MySQLTable table : globalState.getSchema().getDatabaseTables()) { + StringBuilder sb = new StringBuilder(); + sb.append("ANALYZE TABLE "); + sb.append(table.getName()); + sb.append(" UPDATE HISTOGRAM ON "); + String columns = table.getColumns().stream().map(MySQLColumn::getName) + .collect(Collectors.joining(", ")); + sb.append(columns + ";"); + globalState.executeStatement(new SQLQueryAdapter(sb.toString(), errors)); + } + } } @Override diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index 3ed6df405..7abd1f639 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -18,10 +18,6 @@ public void setSelectType(SelectType fromOptions) { this.setFromOptions(fromOptions); } - public SelectType getSelectType() { - return fromOptions; - } - public SelectType getFromOptions() { return fromOptions; } diff --git a/src/sqlancer/mysql/oracle/MySQLCERTOracle.java b/src/sqlancer/mysql/oracle/MySQLCERTOracle.java new file mode 100644 index 000000000..2cdf3c7cf --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLCERTOracle.java @@ -0,0 +1,217 @@ +package sqlancer.mysql.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CERTOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + +public class MySQLCERTOracle extends CERTOracleBase implements TestOracle { + private MySQLExpressionGenerator gen; + private MySQLSelect select; + + public MySQLCERTOracle(MySQLGlobalState globalState) { + super(globalState); + MySQLErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + queryPlan1Sequences = new ArrayList<>(); + queryPlan2Sequences = new ArrayList<>(); + + // Randomly generate a query + MySQLTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new MySQLExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList())); + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) + .collect(Collectors.toList()); + + select = new MySQLSelect(); + select.setFetchColumns(fetchColumns); + select.setFromList(tableList); + + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateExpression()); + } + } + + // Set the join. Todo: to make it random + // List joinExpressions = getJoins(tableList, state); + // select.setJoinList(joinExpressions); + + // Get the result of the first query + String queryString1 = MySQLVisitor.asString(select); + int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + boolean increase = mutate(Mutator.JOIN, Mutator.LIMIT); + + // Get the result of the second query + String queryString2 = MySQLVisitor.asString(select); + int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + // Check structural equivalence + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check the results + if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { + throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 + + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + } + } + + @Override + protected boolean mutateDistinct() { + MySQLSelect.SelectType selectType = select.getFromOptions(); + if (selectType != MySQLSelect.SelectType.ALL) { + select.setSelectType(MySQLSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(MySQLSelect.SelectType.DISTINCT); + return false; + } + } + + @Override + protected boolean mutateWhere() { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(gen.generateExpression()); + } + return increase; + } + + @Override + protected boolean mutateGroupBy() { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + @Override + protected boolean mutateHaving() { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(gen.generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(gen.generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + @Override + protected boolean mutateAnd() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression()); + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(), MySQLBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + @Override + protected boolean mutateOr() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression()); + return false; + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(), MySQLBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + // The limit clause only accpets positive integers, which is not supported yet + // private boolean mutateLimit() { + // boolean increase = select.getLimitClause() != null; + // if (increase) { + // select.setLimitClause(null); + // } else { + // select.setLimitClause(gen.generateConstant()); + // } + // return increase; + // } + + private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + throws AssertionError, SQLException { + int row = -1; + String explainQuery = "EXPLAIN " + selectStr; + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + int estRows = rs.getInt(10); + if (row == -1) { + row = estRows; + } + String operation = rs.getString(2); + queryPlanSequences.add(operation); + } + } + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + if (row == -1) { + throw new IgnoreMeException(); + } + return row; + } + +} diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java index d23789b26..1bef4ac96 100644 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -11,8 +11,8 @@ import sqlancer.Randomly; import sqlancer.SQLGlobalState; import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CERTOracleBase; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.tidb.TiDBErrors; @@ -31,24 +31,12 @@ import sqlancer.tidb.ast.TiDBTableReference; import sqlancer.tidb.visitor.TiDBVisitor; -public class TiDBCERTOracle implements TestOracle { - private final TiDBGlobalState state; +public class TiDBCERTOracle extends CERTOracleBase implements TestOracle { private TiDBExpressionGenerator gen; - private final ExpectedErrors errors = new ExpectedErrors(); private TiDBSelect select; - private List queryPlan1Sequences; - private List queryPlan2Sequences; - - public enum Mutator { - JOIN, WHERE, GROUPBY, HAVING, AND, OR, LIMIT; - - public static Mutator getRandom() { - return Randomly.fromOptions(values()); - } - } public TiDBCERTOracle(TiDBGlobalState globalState) { - state = globalState; + super(globalState); TiDBErrors.addExpressionErrors(errors); } @@ -96,34 +84,7 @@ public void check() throws SQLException { double rowCount1 = getRow(state, queryString1, queryPlan1Sequences); // Mutate the query - boolean increase = false; - - Mutator mutation = Mutator.getRandom(); - switch (mutation) { - case JOIN: - increase = mutateJoin(); - break; - case WHERE: - increase = mutateWhere(); - break; - case GROUPBY: - increase = mutateGroupBy(); - break; - case HAVING: - increase = mutateHaving(); - break; - case AND: - increase = mutateAnd(); - break; - case OR: - increase = mutateOr(); - break; - case LIMIT: - increase = mutateLimit(); - break; - default: - throw new AssertionError(); - } + boolean increase = mutate(Mutator.DISTINCT); // Get the result of the second query String queryString2 = TiDBVisitor.asString(select); @@ -142,12 +103,13 @@ public void check() throws SQLException { */ // Check the results if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { - throw new AssertionError("Mutator: " + mutation + ", Inconsistent result for query: EXPLAIN " + queryString1 - + "; --" + rowCount1 + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 + + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); } } - private boolean mutateJoin() { + @Override + protected boolean mutateJoin() { if (select.getJoinList().isEmpty()) { return false; } @@ -183,7 +145,8 @@ private boolean mutateJoin() { return increase; } - private boolean mutateWhere() { + @Override + protected boolean mutateWhere() { boolean increase = select.getWhereClause() != null; if (increase) { select.setWhereClause(null); @@ -193,7 +156,8 @@ private boolean mutateWhere() { return increase; } - private boolean mutateGroupBy() { + @Override + protected boolean mutateGroupBy() { boolean increase = select.getGroupByExpressions().size() > 0; if (increase) { select.clearGroupByExpressions(); @@ -204,7 +168,8 @@ private boolean mutateGroupBy() { return increase; } - private boolean mutateHaving() { + @Override + protected boolean mutateHaving() { if (select.getGroupByExpressions().size() == 0) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(gen.generateExpression()); @@ -220,7 +185,8 @@ private boolean mutateHaving() { } } - private boolean mutateAnd() { + @Override + protected boolean mutateAnd() { if (select.getWhereClause() == null) { select.setWhereClause(gen.generateExpression()); } else { @@ -231,7 +197,8 @@ private boolean mutateAnd() { return false; } - private boolean mutateOr() { + @Override + protected boolean mutateOr() { if (select.getWhereClause() == null) { select.setWhereClause(gen.generateExpression()); return false; @@ -243,7 +210,8 @@ private boolean mutateOr() { } } - private boolean mutateLimit() { + @Override + protected boolean mutateLimit() { boolean increase = select.getLimitClause() != null; if (increase) { select.setLimitClause(null); From 6957c891beb1f5fec38d1f0db18d752ba47d8697 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 26 Jun 2023 08:58:23 +0800 Subject: [PATCH 0789/1284] feat: stonedb follow up implementation (#836) * feat: add init support feat: add support feat: init support of stonedb * feat: add init support * fix: optimize imports * fix: change from MySQL to StoneDB options * fix: connection string * fix: StoneDBSchema.java * fix: StoneDBSchema.java unused enum * feat: add StoneDB table generator * feat: add support for Action and mapActions * feat: add appendType StoneDBTableGenerator.java * fix: fix errors in mvn verify -DskipTests * feat: add expression generator for StoneDB * feat: add alter index insert generator * refactor: refactor according to comments, * fix: naming error and mvn verify error * fix: simplify the BigInteger generation * fix: fix the BigInteger generation * feat: add drop index and table update * style: add comments * feat: let StoneDBInsertGenerator extends AbstractInsertGenerator --- src/check_names.py | 1 + src/sqlancer/Randomly.java | 14 +- src/sqlancer/stonedb/StoneDBBugs.java | 6 + src/sqlancer/stonedb/StoneDBErrors.java | 7 + src/sqlancer/stonedb/StoneDBOptions.java | 46 +++ src/sqlancer/stonedb/StoneDBProvider.java | 122 ++++++ src/sqlancer/stonedb/StoneDBSchema.java | 342 +++++++++++++++++ .../stonedb/StoneDBToStringVisitor.java | 27 ++ src/sqlancer/stonedb/ast/StoneDBConstant.java | 206 ++++++++++ .../stonedb/ast/StoneDBExpression.java | 4 + .../stonedb/gen/StoneDBDeleteGenerator.java | 60 +++ .../gen/StoneDBExpressionGenerator.java | 287 ++++++++++++++ .../gen/StoneDBIndexDropGenerator.java | 47 +++ .../stonedb/gen/StoneDBIndexGenerator.java | 75 ++++ .../stonedb/gen/StoneDBInsertGenerator.java | 99 +++++ .../gen/StoneDBTableAlterGenerator.java | 94 +++++ .../gen/StoneDBTableCreateGenerator.java | 357 ++++++++++++++++++ .../gen/StoneDBTableUpdateGenerator.java | 69 ++++ .../stonedb/oracle/StoneDBNoRECOracle.java | 16 + .../stonedb/oracle/StoneDBTLPOracle.java | 16 + 20 files changed, 1893 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/stonedb/StoneDBBugs.java create mode 100644 src/sqlancer/stonedb/StoneDBErrors.java create mode 100644 src/sqlancer/stonedb/StoneDBOptions.java create mode 100644 src/sqlancer/stonedb/StoneDBProvider.java create mode 100644 src/sqlancer/stonedb/StoneDBSchema.java create mode 100644 src/sqlancer/stonedb/StoneDBToStringVisitor.java create mode 100644 src/sqlancer/stonedb/ast/StoneDBConstant.java create mode 100644 src/sqlancer/stonedb/ast/StoneDBExpression.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java diff --git a/src/check_names.py b/src/check_names.py index 1b110370b..a88c94e89 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -53,4 +53,5 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes name_to_files["Doris"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "doris")) + name_to_files["StoneDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "stonedb")) verify_all_dbs(name_to_files) diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index e1250abb3..2ceacfb30 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -1,6 +1,7 @@ package sqlancer; import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -215,7 +216,6 @@ public String getString(Randomly r) { }, ALPHANUMERIC { - @Override public String getString(Randomly r) { return getStringOfAlphabet(r, ALPHANUMERIC_ALPHABET); @@ -224,7 +224,6 @@ public String getString(Randomly r) { }, ALPHANUMERIC_SPECIALCHAR { - @Override public String getString(Randomly r) { return getStringOfAlphabet(r, ALPHANUMERIC_SPECIALCHAR_ALPHABET); @@ -452,6 +451,17 @@ public long getLong(long left, long right) { return getNextLong(left, right); } + public BigInteger getBigInteger(BigInteger left, BigInteger right) { + if (left.equals(right)) { + return left; + } + BigInteger result = new BigInteger(String.valueOf(getInteger(left.intValue(), right.intValue()))); + if (result.compareTo(left) < 0 && result.compareTo(right) > 0) { + throw new IgnoreMeException(); + } + return result; + } + public BigDecimal getRandomBigDecimal() { return BigDecimal.valueOf(getThreadRandom().get().nextDouble()); } diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java new file mode 100644 index 000000000..7ed539763 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBBugs.java @@ -0,0 +1,6 @@ +package sqlancer.stonedb; + +public final class StoneDBBugs { + private StoneDBBugs() { + } +} diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java new file mode 100644 index 000000000..54fd28ad8 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -0,0 +1,7 @@ +package sqlancer.stonedb; + +public final class StoneDBErrors { + private StoneDBErrors() { + } + +} diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java new file mode 100644 index 000000000..928bcfa39 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBOptions.java @@ -0,0 +1,46 @@ +package sqlancer.stonedb; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.stonedb.StoneDBOptions.StoneDBOracleFactory; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.oracle.StoneDBNoRECOracle; +import sqlancer.stonedb.oracle.StoneDBTLPOracle; + +@Parameters(separators = "=", commandDescription = "StoneDB (default host: " + StoneDBOptions.DEFAULT_HOST + + ", default port: " + StoneDBOptions.DEFAULT_PORT + ")") +public class StoneDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 3306; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(StoneDBOracleFactory.NOREC); + + public enum StoneDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBNoRECOracle(globalState); + } + }, + TLP { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws Exception { + return new StoneDBTLPOracle(globalState); + } + } + } + + @Override + public List getTestOracleFactory() { + return oracles; + } +} diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java new file mode 100644 index 000000000..376235d92 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBProvider.java @@ -0,0 +1,122 @@ +package sqlancer.stonedb; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.stonedb.gen.StoneDBDeleteGenerator; +import sqlancer.stonedb.gen.StoneDBIndexGenerator; +import sqlancer.stonedb.gen.StoneDBInsertGenerator; +import sqlancer.stonedb.gen.StoneDBTableAlterGenerator; +import sqlancer.stonedb.gen.StoneDBTableCreateGenerator; + +@AutoService(DatabaseProvider.class) +public class StoneDBProvider extends SQLProviderAdapter { + + public StoneDBProvider() { + super(StoneDBGlobalState.class, StoneDBOptions.class); + } + + public static class StoneDBGlobalState extends SQLGlobalState { + @Override + protected StoneDBSchema readSchema() throws Exception { + return StoneDBSchema.fromConnection(getConnection(), getDatabaseName()); + } + } + + enum Action implements AbstractAction { + SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), // + ALTER_TABLE(StoneDBTableAlterGenerator::generate), // + DELETE(StoneDBDeleteGenerator::generate), // + INDEX(StoneDBIndexGenerator::generate), // + INSERT(StoneDBInsertGenerator::generate); // + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public Query getQuery(StoneDBGlobalState globalState) throws Exception { + return sqlQueryProvider.getQuery(globalState); + } + } + + private static int mapActions(StoneDBGlobalState globalState, Action a) { + globalState.getRandomly(); + switch (a) { + case SHOW_TABLES: + return 1; + default: + throw new AssertionError(a); + } + } + + @Override + public void generateDatabase(StoneDBGlobalState globalState) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = StoneDBTableCreateGenerator.generate(globalState, tableName); + globalState.executeStatement(createTable); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + StoneDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(StoneDBGlobalState globalState) throws Exception { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = StoneDBOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = StoneDBOptions.DEFAULT_PORT; + } + String databaseName = globalState.getDatabaseName(); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "stonedb"; + } +} diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java new file mode 100644 index 000000000..eccf05c27 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -0,0 +1,342 @@ +package sqlancer.stonedb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.stonedb.ast.StoneDBConstant; +import sqlancer.stonedb.ast.StoneDBExpression; + +public class StoneDBSchema extends AbstractSchema { + + public enum StoneDBDataType { + + NULL, TINYINT, SMALLINT, MEDIUMINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, YEAR, TIME, DATE, DATETIME, TIMESTAMP, + CHAR, VARCHAR, TINYTEXT, TEXT, MEDIUMTEXT, LONGTEXT, BINARY, VARBINARY, TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB; + + public static StoneDBDataType getRandomWithoutNull() { + return Randomly.fromOptions(values()); + } + + public static Node getRandomValue(StoneDBDataType dataType) { + switch (dataType) { + case TINYINT: + break; + case SMALLINT: + break; + case MEDIUMINT: + break; + case INT: + return StoneDBConstant.createIntConstant((int) Randomly.getNonCachedInteger()); + case BIGINT: + break; + case FLOAT: + break; + case DOUBLE: + return StoneDBConstant.createDoubleConstant(Randomly.getUncachedDouble()); + case DECIMAL: + break; + case YEAR: + break; + case TIME: + break; + case DATE: + return StoneDBConstant.createDateConstant(Randomly.getNonCachedInteger()); + case DATETIME: + break; + case TIMESTAMP: + return StoneDBConstant.createTimestampConstant(Randomly.getNonCachedInteger()); + case CHAR: + break; + case VARCHAR: + break; + case TINYTEXT: + break; + case TEXT: + return StoneDBConstant.createTextConstant(new Randomly().getString()); + case MEDIUMTEXT: + break; + case LONGTEXT: + break; + case BINARY: + break; + case VARBINARY: + break; + case TINYBLOB: + break; + case BLOB: + break; + case MEDIUMBLOB: + break; + case LONGBLOB: + break; + default: + throw new AssertionError(); + } + return null; + } + } + + public static class StoneDBTable + extends AbstractRelationalTable { + + public StoneDBTable(String tableName, List columns, List indexes, boolean isView) { + super(tableName, columns, indexes, isView); + } + + public boolean hasPrimaryKey() { + return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); + } + + } + + public static final class StoneDBIndex extends TableIndex { + private StoneDBIndex(String indexName) { + super(indexName); + } + + public static StoneDBIndex create(String indexName) { + return new StoneDBIndex(indexName); + } + + @Override + public String getIndexName() { + if (super.getIndexName().contentEquals("PRIMARY")) { + return "`PRIMARY`"; + } else { + return super.getIndexName(); + } + } + } + + public static StoneDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con, databaseName); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, databaseName, tableName); + List indexes = getIndexes(con, databaseName, tableName); + boolean isView = tableName.startsWith("v"); + StoneDBTable t = new StoneDBTable(tableName, databaseColumns, indexes, isView); + for (StoneDBColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new StoneDBSchema(databaseTables); + } + + private static List getIndexes(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List indexes = new ArrayList<>(); + try (ResultSet rs = con.createStatement() + .executeQuery("SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = '" + + databaseName + "' AND TABLE_NAME='" + tableName + "';")) { + while (rs.next()) { + String indexName = rs.getString("INDEX_NAME"); + indexes.add(StoneDBIndex.create(indexName)); + } + } + return indexes; + } + + private static List getTableColumns(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List columns = new ArrayList<>(); + try (ResultSet rs = con.createStatement() + .executeQuery("select * from information_schema.columns where table_schema = '" + databaseName + + "' AND TABLE_NAME='" + tableName + "';")) { + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + String dataType = rs.getString("DATA_TYPE"); + int precision = rs.getInt("NUMERIC_PRECISION"); + boolean isNullable = !rs.getString("IS_NULLABLE").equals("NO"); + boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); + StoneDBColumn c = new StoneDBColumn(columnName, getColumnCompositeDataType(dataType), isPrimaryKey, + isNullable, precision); + columns.add(c); + } + } + return columns; + } + + private static StoneDBCompositeDataType getColumnCompositeDataType(String typeString) { + if (Arrays.stream(StoneDBDataType.values()).noneMatch(e -> e.name().equals(typeString.toUpperCase()))) { + throw new AssertionError(typeString); + } + return new StoneDBCompositeDataType(StoneDBDataType.valueOf(typeString.toUpperCase())); + } + + public static StoneDBDataType getColumnDataType(String typeString) { + if (Arrays.stream(StoneDBDataType.values()).noneMatch(e -> e.name().equals(typeString.toUpperCase()))) { + throw new AssertionError(typeString); + } + return StoneDBDataType.valueOf(typeString.toUpperCase()); + } + + private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { + List tableNames = new ArrayList<>(); + try (ResultSet rs = con.createStatement() + .executeQuery("select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + + databaseName + "';")) { + while (rs.next()) { + if (rs.getString("ENGINE").equals("TIANMU")) { + tableNames.add(rs.getString("TABLE_NAME")); + } + } + } + return tableNames; + } + + public static class StoneDBColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + private final int precision; + + public StoneDBColumn(String name, StoneDBCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable, + int precision) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + this.precision = precision; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + public int getPrecision() { + return precision; + } + + } + + public StoneDBSchema(List databaseTables) { + super(databaseTables); + } + + public static class StoneDBCompositeDataType { + private final StoneDBDataType dataType; + private final int size; + + public StoneDBCompositeDataType(StoneDBDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public StoneDBCompositeDataType(StoneDBDataType dataType) { + this.dataType = dataType; + int size = -1; + switch (dataType) { + case TINYINT: + size = 1; + break; + case SMALLINT: + size = 2; + break; + case MEDIUMINT: + size = 3; + break; + case INT: + size = 4; + break; + case BIGINT: + size = 8; + break; + case FLOAT: + size = 4; + break; + case DOUBLE: + size = 8; + break; + case DECIMAL: + size = -2; + break; + case YEAR: + size = -2; + break; + case TIME: + size = -2; + break; + case DATE: + size = -2; + break; + case DATETIME: + size = -2; + break; + case TIMESTAMP: + size = -2; + break; + case CHAR: + size = -2; + break; + case VARCHAR: + size = -2; + break; + case TINYTEXT: + size = -2; + break; + case TEXT: + size = -2; + break; + case MEDIUMTEXT: + size = -2; + break; + case LONGTEXT: + size = -2; + break; + case BINARY: + size = -2; + break; + case VARBINARY: + size = -2; + break; + case TINYBLOB: + size = -2; + break; + case BLOB: + size = -2; + break; + case MEDIUMBLOB: + size = -2; + break; + case LONGBLOB: + size = -2; + break; + default: + throw new AssertionError(); + } + this.size = size; + } + + public StoneDBDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static StoneDBCompositeDataType getRandomWithoutNull() { + StoneDBDataType type = StoneDBDataType.getRandomWithoutNull(); + return new StoneDBCompositeDataType(type); + } + } +} diff --git a/src/sqlancer/stonedb/StoneDBToStringVisitor.java b/src/sqlancer/stonedb/StoneDBToStringVisitor.java new file mode 100644 index 000000000..3cebd97f1 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBToStringVisitor.java @@ -0,0 +1,27 @@ +package sqlancer.stonedb; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.stonedb.ast.StoneDBConstant; +import sqlancer.stonedb.ast.StoneDBExpression; + +public class StoneDBToStringVisitor extends NewToStringVisitor { + @Override + public void visitSpecific(Node expr) { + if (expr instanceof StoneDBConstant) { + visit((StoneDBConstant) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(StoneDBConstant constant) { + sb.append(constant.toString()); + } + + public static String asString(Node expr) { + StoneDBToStringVisitor visitor = new StoneDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/stonedb/ast/StoneDBConstant.java b/src/sqlancer/stonedb/ast/StoneDBConstant.java new file mode 100644 index 000000000..f0abae48a --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBConstant.java @@ -0,0 +1,206 @@ +package sqlancer.stonedb.ast; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +import sqlancer.common.ast.newast.Node; + +public class StoneDBConstant implements Node { + + private StoneDBConstant() { + } + + public static class StoneDBNullConstant extends StoneDBConstant { + @Override + public String toString() { + return "NULL"; + } + } + + public static Node createNullConstant() { + return new StoneDBNullConstant(); + } + + public static class StoneDBIntConstant extends StoneDBConstant { + private final Integer value; + + public StoneDBIntConstant(int value) { + this.value = value; + } + + @Override + public String toString() { + if (value.equals(Integer.MIN_VALUE)) { + return "INT_NULL"; + } + return String.valueOf(value); + } + } + + public static Node createIntConstant(int val) { + return new StoneDBIntConstant(val); + } + + public static class StoneDBBigIntConstant extends StoneDBConstant { + private final Long value; + + public StoneDBBigIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + // For INT type: -2147483648 is reserved to indicate INT_NULL in Tianmu engine, Minimum Value Signed start + // from -2147483647 + // refer: https://stonedb.io/docs/SQL-reference/data-types + if (value.equals(Long.MIN_VALUE)) { + return "BIGINT_NULL"; + } + return String.valueOf(value); + } + } + + public static class StoneDBDoubleConstant extends StoneDBConstant { + + private final Double value; + + public StoneDBDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + } + + public static Node createDoubleConstant(double val) { + return new StoneDBDoubleConstant(val); + } + + public static class StoneDBTextConstant extends StoneDBConstant { + + private final String value; + + public StoneDBTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static Node createTextConstant(String text) { + return new StoneDBTextConstant(text); + } + + public static class StoneDBBitConstant extends StoneDBConstant { + + private final String value; + + public StoneDBBitConstant(long value) { + this.value = Long.toBinaryString(value); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "B'" + value + "'"; + } + + } + + public static class StoneDBDateConstant extends StoneDBConstant { + + public String textRepr; + + public StoneDBDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + } + + public static Node createDateConstant(long integer) { + return new StoneDBDateConstant(integer); + } + + public static class StoneDBTimestampConstant extends StoneDBConstant { + + public String textRepr; + + public StoneDBTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + } + + public static Node createTimestampConstant(long integer) { + return new StoneDBTimestampConstant(integer); + } + + public static class StoneDBBooleanConstant extends StoneDBConstant { + + private final boolean value; + + public StoneDBBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static Node createBooleanConstant(boolean val) { + return new StoneDBBooleanConstant(val); + } + +} diff --git a/src/sqlancer/stonedb/ast/StoneDBExpression.java b/src/sqlancer/stonedb/ast/StoneDBExpression.java new file mode 100644 index 000000000..fb8e75124 --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBExpression.java @@ -0,0 +1,4 @@ +package sqlancer.stonedb.ast; + +public interface StoneDBExpression { +} diff --git a/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java new file mode 100644 index 000000000..d52aed526 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java @@ -0,0 +1,60 @@ +package sqlancer.stonedb.gen; + +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public final class StoneDBDeleteGenerator { + private final StoneDBGlobalState globalState; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + private StoneDBDeleteGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBDeleteGenerator(globalState).getQuery(); + } + + public SQLQueryAdapter getQuery() { + StoneDBTable randomTable = globalState.getSchema().getRandomTable(); + sb.append("DELETE"); + if (Randomly.getBoolean()) { + sb.append(" LOW_PRIORITY"); + } + if (Randomly.getBoolean()) { + sb.append(" QUICK"); + } + if (Randomly.getBoolean()) { + sb.append(" IGNORE"); + } + sb.append(" FROM "); + sb.append(randomTable.getName()); + if (Randomly.getBoolean()) { + sb.append(" AS "); + sb.append(globalState.getSchema().getRandomTable().getName()); + } + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) + .setColumns(randomTable.getColumns()).generateExpression())); + } + if (Randomly.getBoolean()) { + sb.append(" ORDER BY "); + sb.append(Randomly.fromOptions( + randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))); + } + if (Randomly.getBoolean()) { + sb.append(" LIMIT "); + sb.append(new Randomly().getInteger()); + } + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java new file mode 100644 index 000000000..cf578ecb0 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -0,0 +1,287 @@ +package sqlancer.stonedb.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewCaseOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.ast.StoneDBConstant; +import sqlancer.stonedb.ast.StoneDBExpression; + +public class StoneDBExpressionGenerator extends UntypedExpressionGenerator, StoneDBColumn> { + + private final StoneDBGlobalState globalState; + + public StoneDBExpressionGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, BINARY_BITWISE, BETWEEN, IN, + CASE + } + + @Override + public Node negatePredicate(Node predicate) { + return null; + } + + @Override + public Node isNull(Node expr) { + return null; + } + + @Override + public Node generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return StoneDBConstant.createNullConstant(); + } + StoneDBDataType type = StoneDBDataType.getRandomWithoutNull(); + return generateConstant(type); + } + + public Node generateConstant(StoneDBDataType dataType) { + switch (dataType) { + case INT: + return StoneDBConstant + .createIntConstant(globalState.getRandomly().getInteger(Integer.MIN_VALUE + 1, Integer.MAX_VALUE)); + case DATE: + return StoneDBConstant.createDateConstant(globalState.getRandomly().getInteger()); + case TIMESTAMP: + return StoneDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case VARCHAR: + return StoneDBConstant.createTextConstant(globalState.getRandomly().getString()); + case DOUBLE: + return StoneDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + default: + throw new IgnoreMeException(); + } + } + + public Node generateConstant(StoneDBDataType dataType, boolean isNullable) { + if (isNullable && Randomly.getBooleanWithSmallProbability()) { + generateConstant(StoneDBDataType.NULL); + } + return generateConstant(dataType); + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBoolean()) { + StoneDBAggregateFunction aggregateFunction = StoneDBAggregateFunction.getRandom(); + allowAggregates = false; + return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs(), depth + 1), + aggregateFunction); + } + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + Operator op; + switch (expr) { + case UNARY_PREFIX: + op = StoneDBUnaryPrefixOperator.getRandom(); + return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), op); + case UNARY_POSTFIX: + op = StoneDBUnaryPostfixOperator.getRandom(); + return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), op); + case BINARY_COMPARISON: + op = StoneDBBinaryComparisonOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = StoneDBBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + op = StoneDBBinaryArithmeticOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_BITWISE: + op = StoneDBBinaryBitwiseOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BETWEEN: + return new NewBetweenOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new NewInOperatorNode<>(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new NewCaseOperatorNode<>(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(); + } + } + + @Override + protected Node generateColumn() { + return null; + } + + public enum StoneDBAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1), FIRST(1), SUM(1); + + private int nrArgs; + + StoneDBAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static StoneDBAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + public enum StoneDBUnaryPrefixOperator implements Operator { + + NOT("NOT"), PLUS("+"), MINUS("-"); + + private final String textRepr; + + StoneDBUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static StoneDBUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum StoneDBUnaryPostfixOperator implements Operator { + + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private final String textRepr; + + StoneDBUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static StoneDBUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + /* + * Comparison operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/comparison-operators + */ + public enum StoneDBBinaryComparisonOperator implements Operator { + EQUAL("="), GREATER(">"), LESS("<"), GREATER_EQUAL(">="), LESS_EQUAL("<="), + NOT_EQUALS(Randomly.fromList(Arrays.asList("!=", "<>"))), NULL_SAFE_EQUAL("<=>"), IN("IN"), NOT_IN("NOT_IN"), + LIKE("LIKE"), IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private final String textRepr; + + StoneDBBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + /* + * Logical operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/logical-operators + */ + public enum StoneDBBinaryLogicalOperator implements Operator { + + NOT("NOT"), AND("AND"), OR("OR"), XOR("XOR"); + + private final String textRepr; + + StoneDBBinaryLogicalOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + /* + * Arithmetic operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/arithmetic-operators + */ + public enum StoneDBBinaryArithmeticOperator implements Operator { + ADDITION("+"), MINUS("-"), MULTIPLICATION("*"), DIVISION(Randomly.fromList(Arrays.asList("/", " div "))), + MODULO(Randomly.fromList(Arrays.asList("%", " mod "))); + + private final String textRepr; + + StoneDBBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + /* + * Bitwise operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/bitwise-operators + */ + public enum StoneDBBinaryBitwiseOperator implements Operator { + AND("&"), OR("|"), XOR("^"), INVERSION("!"), LEFTSHIFT("<<"), RIGHTSHIFT(">>"); + + private final String textRepr; + + StoneDBBinaryBitwiseOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java new file mode 100644 index 000000000..dda72c735 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java @@ -0,0 +1,47 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; + +public class StoneDBIndexDropGenerator { + // which table to drop index + StoneDBSchema.StoneDBTable table; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBIndexDropGenerator(StoneDBGlobalState globalState) { + table = globalState.getSchema().getRandomTable(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBIndexDropGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + if (!table.hasIndexes()) { + return null; + } + sb.append("DROP INDEX "); + sb.append(table.getRandomIndex().getIndexName()); + sb.append(" ON "); + sb.append(table.getName()); + appendAlgoOrLockOption(); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private void appendAlgoOrLockOption() { + if (Randomly.getBoolean()) { + return; + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); + sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); + } else { + sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); + sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java new file mode 100644 index 000000000..9a86cf7e4 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java @@ -0,0 +1,75 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; + +public class StoneDBIndexGenerator { + private final StoneDBGlobalState globalState; + // which table to add index + StoneDBTable table; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBIndexGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + table = globalState.getSchema().getRandomTable(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBIndexGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("CREATE "); + sb.append(Randomly.fromOptions("UNIQUE", "FULLTEXT", "SPATIAL")); + sb.append(" INDEX"); + sb.append(globalState.getSchema().getFreeIndexName()); + appendIndexType(); + sb.append(" ON "); + sb.append(table.getName()); + appendKeyPart(); + appendIndexOption(); + appendAlgoOrLockOption(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void appendIndexType() { + if (Randomly.getBoolean()) { + return; + } + sb.append(" USING "); + sb.append(Randomly.fromOptions("BTREE", "HASH")); + } + + private void appendKeyPart() { + sb.append("("); + sb.append(table.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + sb.append(")"); + } + + private void appendIndexOption() { + if (Randomly.getBoolean()) { + return; + } + sb.append(Randomly.fromOptions(" VISIBLE", " INVISIBLE")); + } + + private void appendAlgoOrLockOption() { + if (Randomly.getBoolean()) { + return; + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); + sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); + } else { + sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); + sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java new file mode 100644 index 000000000..05a2a24e9 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java @@ -0,0 +1,99 @@ +package sqlancer.stonedb.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBInsertGenerator extends AbstractInsertGenerator { + private final StoneDBGlobalState globalState; + // which table to insert into + private final StoneDBTable table; + // which subset columns of the table to add values + private final List columns; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBInsertGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + table = globalState.getSchema().getRandomTable(); + columns = table.getRandomNonEmptyColumnSubset(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBInsertGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("INSERT"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("LOW_PRIORITY", "DELAYED", "HIGH_PRIORITY")); + } + if (Randomly.getBoolean()) { + sb.append(" IGNORE"); + } + sb.append(" INTO "); + sb.append(table.getName()); + appendPartition(); + appendColumnsAndValues(columns); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private void appendPartition() { + + } + + private void appendColumnsAndValues(List columns) { + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(Randomly.fromOptions(" VALUES ", " VALUE ")); + appendValues(); + } + + private void appendValues() { + int nrRows; + if (Randomly.getBoolean()) { + nrRows = 1; + } else { + nrRows = 1 + Randomly.smallNumber(); + } + for (int row = 0; row < nrRows; row++) { + appendOneValue(row); + } + } + + private void appendOneValue(int nrRow) { + if (nrRow != 0) { + sb.append(", "); + } + sb.append("("); + for (int c = 0; c < columns.size(); c++) { + if (c != 0) { + sb.append(", "); + } + sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState).generateConstant())); + + } + sb.append(")"); + } + + @Override + protected void insertValue(StoneDBColumn column) { + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append("DEFAULT"); + } else { + sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java new file mode 100644 index 000000000..55ebd0715 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -0,0 +1,94 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; + +public class StoneDBTableAlterGenerator { + private final StoneDBGlobalState globalState; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN, CHANGE_COLUMN, RENAME_COLUMN + } + + public StoneDBTableAlterGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBTableAlterGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("ALTER TABLE "); + StoneDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) + .setColumns(table.getColumns()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(StoneDBCompositeDataType.getRandomWithoutNull()); + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" FIRST"); + } else { + sb.append(" AFTER "); + sb.append(table.getRandomColumn().getName()); + } + } + break; + case DROP_COLUMN: + sb.append(Randomly.fromOptions("DROP COLUMN ", "DROP ")); + sb.append(table.getRandomColumn().getName()); + break; + case ALTER_COLUMN: + sb.append(Randomly.fromOptions("ALTER COLUMN ", "ALTER ")); + sb.append(table.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(" SET DEFAULT ").append(generator.generateExpression()); + } else { + sb.append(" DROP DEFAULT"); + } + if (Randomly.getBoolean()) { + sb.append(" SET ").append(Randomly.fromOptions("VISIBLE", "INVISIBLE")); + } + break; + case CHANGE_COLUMN: + sb.append(Randomly.fromOptions("CHANGE COLUMN ", "CHANGE ")); + String oldColumnName = table.getRandomColumn().getName(); + String newColumnName = table.getFreeColumnName(); + sb.append(oldColumnName).append(" ").append(newColumnName); + sb.append(" "); + sb.append(StoneDBCompositeDataType.getRandomWithoutNull()); + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" FIRST"); + } else { + sb.append(" AFTER "); + sb.append(table.getRandomColumn().getName()); + } + } + break; + case RENAME_COLUMN: + sb.append("RENAME COLUMN"); + sb.append(table.getRandomColumn().getName()); + sb.append(" TO "); + sb.append(table.getFreeColumnName()); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java new file mode 100644 index 000000000..134e0edfe --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java @@ -0,0 +1,357 @@ +package sqlancer.stonedb.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; + +public class StoneDBTableCreateGenerator { + // the name of the table to create + private final String tableName; + private final StoneDBSchema schema; + // the name of the columns in the table + private final List columns = new ArrayList<>(); + private final boolean allowPrimaryKey; + private boolean setPrimaryKey; + private final StringBuilder sb = new StringBuilder(); + private final Randomly r; + + public StoneDBTableCreateGenerator(StoneDBGlobalState globalState, String tableName) { + this.tableName = tableName; + this.schema = globalState.getSchema(); + allowPrimaryKey = Randomly.getBoolean(); + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String tableName) { + return new StoneDBTableCreateGenerator(globalState, tableName).getQuery(); + } + + public SQLQueryAdapter getQuery() { + ExpectedErrors errors = new ExpectedErrors(); + sb.append(Randomly.fromOptions("CREATE TABLE", "CREATE TEMPORARY TABLE")); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS "); + } + sb.append(tableName); + // ues link statement + if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { + sb.append(" LIKE "); + sb.append(schema.getRandomTable().getName()); + return new SQLQueryAdapter(sb.toString(), true); + } else { + appendColumns(); + sb.append(" "); + appendTableOptions(); + addCommonErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + } + + private enum TableOptions { + AUTO_INCREMENT, AVG_ROW_LENGTH, CHECKSUM, COMPRESSION, DELAY_KEY_WRITE, /* ENCRYPTION, */ INSERT_METHOD, + KEY_BLOCK_SIZE, MAX_ROWS, MIN_ROWS, PACK_KEYS, STATS_AUTO_RECALC, STATS_PERSISTENT, STATS_SAMPLE_PAGES; + + public static List getRandomTableOptions() { + List options; + // try to ensure that usually, only a few of these options are generated + if (Randomly.getBooleanWithSmallProbability()) { + options = Randomly.subset(TableOptions.values()); + } else { + if (Randomly.getBoolean()) { + options = Collections.emptyList(); + } else { + options = Randomly.nonEmptySubset(Arrays.asList(TableOptions.values()), Randomly.smallNumber()); + } + } + return options; + } + } + + private void appendTableOptions() { + List tableOptions = TableOptions.getRandomTableOptions(); + int i = 0; + for (TableOptions o : tableOptions) { + if (i++ != 0) { + sb.append(", "); + } + switch (o) { + case AUTO_INCREMENT: + sb.append("AUTO_INCREMENT = "); + sb.append(r.getPositiveInteger()); + break; + // The valid range for avg_row_length is [0,4294967295] + case AVG_ROW_LENGTH: + sb.append("AVG_ROW_LENGTH = "); + sb.append(r.getLong(0, 4294967295L + 1)); + break; + case CHECKSUM: + sb.append("CHECKSUM = 1"); + break; + case COMPRESSION: + sb.append("COMPRESSION = '"); + sb.append(Randomly.fromOptions("ZLIB", "LZ4", "NONE")); + sb.append("'"); + break; + case DELAY_KEY_WRITE: + sb.append("DELAY_KEY_WRITE = "); + sb.append(Randomly.fromOptions(0, 1)); + break; + case INSERT_METHOD: + sb.append("INSERT_METHOD = "); + sb.append(Randomly.fromOptions("NO", "FIRST", "LAST")); + break; + // The valid range for key_block_size is [0,65535] + case KEY_BLOCK_SIZE: + sb.append("KEY_BLOCK_SIZE = "); + sb.append(r.getInteger(0, 65535 + 1)); + break; + case MAX_ROWS: + sb.append("MAX_ROWS = "); + sb.append(r.getLong(0, Long.MAX_VALUE)); + break; + case MIN_ROWS: + sb.append("MIN_ROWS = "); + sb.append(r.getLong(1, Long.MAX_VALUE)); + break; + case PACK_KEYS: + sb.append("PACK_KEYS = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); + break; + case STATS_AUTO_RECALC: + sb.append("STATS_AUTO_RECALC = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); + break; + case STATS_PERSISTENT: + sb.append("STATS_PERSISTENT = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); + break; + case STATS_SAMPLE_PAGES: + sb.append("STATS_SAMPLE_PAGES = "); + sb.append(r.getInteger(1, Short.MAX_VALUE)); + break; + default: + throw new AssertionError(o); + } + } + } + + private void addCommonErrors(ExpectedErrors list) { + list.add("The storage engine for the table doesn't support"); + list.add("doesn't have this option"); + list.add("must include all columns"); + list.add("not allowed type for this type of partitioning"); + list.add("doesn't support BLOB/TEXT columns"); + list.add("A BLOB field is not allowed in partition function"); + list.add("Too many keys specified; max 1 keys allowed"); + list.add("The total length of the partitioning fields is too large"); + list.add("Got error -1 - 'Unknown error -1' from storage engine"); + } + + private void appendColumns() { + sb.append("("); + for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(i); + } + sb.append(")"); + } + + private void appendColumn(int columnId) { + String columnName = DBMSCommon.createColumnName(columnId); + columns.add(columnName); + sb.append(columnName); + appendColumnDefinition(); + } + + private void appendColumnDefinition() { + sb.append(" "); + StoneDBDataType randomType = StoneDBDataType.getRandomWithoutNull(); + appendType(randomType); + sb.append(" "); + appendColumnOption(randomType); + } + + private enum ColumnOptions { + NULL_OR_NOT_NULL, UNIQUE, COMMENT, COLUMN_FORMAT, STORAGE, PRIMARY_KEY + } + + private void appendColumnOption(StoneDBDataType type) { + boolean isTextType = type == StoneDBDataType.VARCHAR; + boolean isNull = false; + boolean columnHasPrimaryKey = false; + List columnOptions = Randomly.subset(ColumnOptions.values()); + // boolean tableHasNullableColumn; + // if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { + // tableHasNullableColumn = true; + // } + if (isTextType) { + // TODO: restriction due to the limited key length + columnOptions.remove(ColumnOptions.PRIMARY_KEY); + columnOptions.remove(ColumnOptions.UNIQUE); + } + for (ColumnOptions o : columnOptions) { + sb.append(" "); + switch (o) { + case NULL_OR_NOT_NULL: + // PRIMARY KEYs cannot be NULL + if (!columnHasPrimaryKey) { + if (Randomly.getBoolean()) { + sb.append("NULL"); + } + // tableHasNullableColumn = true; + isNull = true; + } else { + sb.append("NOT NULL"); + } + break; + case UNIQUE: + sb.append("UNIQUE"); + if (Randomly.getBoolean()) { + sb.append(" KEY"); + } + break; + case COMMENT: + // TODO: generate randomly + sb.append(String.format("COMMENT '%s' ", "asdf")); + break; + case COLUMN_FORMAT: + sb.append("COLUMN_FORMAT "); + sb.append(Randomly.fromOptions("FIXED", "DYNAMIC", "DEFAULT")); + break; + case STORAGE: + sb.append("STORAGE "); + sb.append(Randomly.fromOptions("DISK", "MEMORY")); + break; + case PRIMARY_KEY: + // PRIMARY KEYs cannot be NULL + if (allowPrimaryKey && !setPrimaryKey && !isNull) { + sb.append("PRIMARY KEY"); + setPrimaryKey = true; + columnHasPrimaryKey = true; + } + break; + default: + throw new AssertionError(); + } + } + } + + private void appendType(StoneDBDataType randomType) { + switch (randomType) { + case TINYINT: + sb.append("TINYINT"); + // sb.append(r.getInteger(-128, 127)); + break; + case SMALLINT: + sb.append("SMALLINT"); + // sb.append(r.getInteger(-32768, 32767)); + break; + case MEDIUMINT: + sb.append("MEDIUMINT"); + // sb.append(r.getInteger(-8388608, 8388607)); + break; + case INT: + sb.append("INT"); + // sb.append(r.getInteger(-2147483647, 2147483647)); + break; + case BIGINT: + sb.append("BIGINT"); + // sb.append(r.getBigInteger(new BigInteger("-9223372036854775806"), new + // BigInteger("9223372036854775807"))); + break; + case FLOAT: + sb.append("FLOAT"); + optionallyAddPrecisionAndScale(sb); + break; + case DOUBLE: + sb.append("DOUBLE"); + optionallyAddPrecisionAndScale(sb); + break; + case DECIMAL: + sb.append("DECIMAL"); // The default value is P(10,0); + break; + case YEAR: + sb.append("YEAR"); + break; + case TIME: + sb.append("TIME"); + break; + case DATE: + sb.append("DATE"); + break; + case DATETIME: + sb.append("DATETIME"); + break; + case TIMESTAMP: + sb.append("TIMESTAMP"); + break; + case CHAR: + sb.append("CHAR"); + break; + case VARCHAR: + sb.append("VARCHAR("); + sb.append(r.getInteger(0, 65535)); + sb.append(")"); + break; + case TINYTEXT: + sb.append("TINYTEXT"); + break; + case TEXT: + sb.append("TEXT"); + break; + case MEDIUMTEXT: + sb.append("MEDIUMTEXT"); + break; + case LONGTEXT: + sb.append("LONGTEXT"); + break; + case BINARY: + sb.append("BINARY"); + break; + case VARBINARY: + sb.append("VARBINARY"); + break; + case TINYBLOB: + sb.append("TINYBLOB"); + break; + case BLOB: + sb.append("BLOB"); + break; + case MEDIUMBLOB: + sb.append("MEDIUMBLOB"); + break; + case LONGBLOB: + sb.append("LONGBLOB"); + break; + default: + throw new AssertionError(); + } + } + + public static void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean()) { + sb.append("("); + // The maximum number of digits (M) for DECIMAL is 65 + long m = Randomly.getNotCachedInteger(1, 65); + sb.append(m); + sb.append(", "); + // The maximum number of supported decimals (D) is 30 + long nCandidate = Randomly.getNotCachedInteger(1, 30); + // For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'c0'). + long n = Math.min(nCandidate, m); + sb.append(n); + sb.append(")"); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java new file mode 100644 index 000000000..bff098a64 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java @@ -0,0 +1,69 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBTableUpdateGenerator { + private final StoneDBGlobalState globalState; + // which table to insert into + private final StoneDBTable table; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBTableUpdateGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + table = globalState.getSchema().getRandomTable(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBTableUpdateGenerator(globalState).getQuery(); + } + + public SQLQueryAdapter getQuery() { + sb.append("UPDATE"); + sb.append(Randomly.fromOptions(" ", " LOW_PRIORITY ", " LOW_PRIORITY IGNORE ", " IGNORE ")); + sb.append(table.getName()); + sb.append("SET "); + appendAssignmentList(); + if (Randomly.getBoolean()) { + appendWhereCondition(); + } + if (Randomly.getBoolean()) { + appendOrderBy(); + } + if (Randomly.getBoolean()) { + appendLimit(); + } + return new SQLQueryAdapter(sb.toString(), errors); + } + + public void appendAssignmentList() { + for (int i = 0; i < new Randomly().getInteger(1, table.getColumns().size()); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(table.getColumns().get(i)); + sb.append(" = "); + sb.append(StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType + .getRandomValue(table.getColumns().get(i).getType().getPrimitiveDataType()))); + } + } + + private void appendWhereCondition() { + } + + private void appendOrderBy() { + sb.append(" ORDER BY "); + sb.append(table.getRandomColumn().getName()); + } + + private void appendLimit() { + sb.append(" LIMIT "); + sb.append(new Randomly().getInteger(0, (int) table.getNrRows(globalState))); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java new file mode 100644 index 000000000..ac998b180 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -0,0 +1,16 @@ +package sqlancer.stonedb.oracle; + +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; + +public class StoneDBNoRECOracle extends NoRECBase implements TestOracle { + public StoneDBNoRECOracle(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java b/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java new file mode 100644 index 000000000..96deda89e --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java @@ -0,0 +1,16 @@ +package sqlancer.stonedb.oracle; + +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; + +public class StoneDBTLPOracle extends NoRECBase implements TestOracle { + public StoneDBTLPOracle(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + + } +} From 37c33981c7fdc3039fe7ab6db15e7ce6c577228f Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 26 Jun 2023 20:54:28 +0800 Subject: [PATCH 0790/1284] CockroachDB: optimize code for CERT (#839) * cockroachdb: fill all tables * cockroachdb: flat outer join * cockroachdb: add utility functions * cockroachdb: support inserting to a specific table * cockorachdb: optimize generators * randomly: assert --- src/sqlancer/Randomly.java | 6 ++ .../cockroachdb/CockroachDBErrors.java | 3 +- .../cockroachdb/CockroachDBProvider.java | 12 ++++ .../cockroachdb/CockroachDBSchema.java | 4 ++ .../CockroachDBToStringVisitor.java | 61 +++++++++++-------- .../cockroachdb/ast/CockroachDBJoin.java | 52 ++++++---------- .../gen/CockroachDBInsertGenerator.java | 6 +- ...CockroachDBSetClusterSettingGenerator.java | 3 +- .../gen/CockroachDBShowGenerator.java | 2 +- .../oracle/CockroachDBNoRECOracle.java | 21 +------ 10 files changed, 88 insertions(+), 82 deletions(-) diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 2ceacfb30..8494c189a 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -135,6 +135,12 @@ public static List nonEmptySubset(List columns, int nr) { return extractNrRandomColumns(columns, nr); } + public static List nonEmptySubsetLeast(List columns, int min) { + int nr = getNextInt(min, columns.size() + 1); + assert nr <= columns.size(); + return extractNrRandomColumns(columns, nr); + } + public static List nonEmptySubsetPotentialDuplicates(List columns) { List arr = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index e282981dc..7cb738ee8 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -19,7 +19,8 @@ public static void addExpressionErrors(ExpectedErrors errors) { } errors.add("exceeds supported timestamp bounds"); - + errors.add("expected STORED COMPUTED COLUMN expression to have type bytes"); + errors.add("volatile functions are not allowed in STORED COMPUTED COLUMN"); errors.add("cannot cast negative integer to bit varying with unbounded width"); errors.add("negative value for LIMIT"); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index c9e0d24a9..c2497e5c8 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -7,6 +7,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -340,4 +341,15 @@ protected void executeMutator(int index, CockroachDBGlobalState globalState) thr globalState.executeStatement(queryMutateTable); } + @Override + public boolean addRowsToAllTables(CockroachDBGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (CockroachDBTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = CockroachDBInsertGenerator.insert(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index c245f4cce..708afef92 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -208,6 +208,10 @@ public CockroachDBTables getRandomTableNonEmptyTables() { return new CockroachDBTables(Randomly.nonEmptySubset(getDatabaseTables())); } + public CockroachDBTables getRandomTableNonEmptyTables(int nr) { + return new CockroachDBTables(Randomly.nonEmptySubsetLeast(getDatabaseTables(), nr)); + } + private static CockroachDBCompositeDataType getColumnType(String typeString) { if (typeString.endsWith("[]")) { String substring = typeString.substring(0, typeString.length() - 2); diff --git a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java index 17a09a401..1c6fe0f36 100644 --- a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java @@ -142,54 +142,65 @@ public void visit(CockroachDBJoin join) { switch (join.getJoinType()) { case INNER: sb.append(" INNER "); - potentiallyAddHint(); + potentiallyAddHint(false); sb.append("JOIN "); visit(join.getRightTable()); sb.append(" ON "); visit(join.getOnCondition()); break; - case NATURAL: - sb.append(" NATURAL "); - // potentiallyAddHint(); + case LEFT: + sb.append(" LEFT"); + sb.append(" OUTER "); + potentiallyAddHint(true); sb.append("JOIN "); visit(join.getRightTable()); + sb.append(" ON "); + visit(join.getOnCondition()); break; - case CROSS: - sb.append(" CROSS "); - potentiallyAddHint(); + case RIGHT: + sb.append(" RIGHT"); + sb.append(" OUTER "); + potentiallyAddHint(true); sb.append("JOIN "); visit(join.getRightTable()); + sb.append(" ON "); + visit(join.getOnCondition()); break; - case OUTER: - sb.append(" "); - switch (join.getOuterType()) { - case FULL: - sb.append("FULL"); - break; - case LEFT: - sb.append("LEFT"); - break; - case RIGHT: - sb.append("RIGHT"); - break; - default: - throw new AssertionError(); - } + case FULL: + sb.append(" FULL"); sb.append(" OUTER "); - potentiallyAddHint(); + potentiallyAddHint(true); sb.append("JOIN "); visit(join.getRightTable()); sb.append(" ON "); visit(join.getOnCondition()); break; + case CROSS: + sb.append(" CROSS "); + potentiallyAddHint(false); + sb.append("JOIN "); + visit(join.getRightTable()); + break; + case NATURAL: + sb.append(" NATURAL "); + // potentiallyAddHint(false); + sb.append("JOIN "); + visit(join.getRightTable()); + break; default: throw new AssertionError(); } } - private void potentiallyAddHint() { + private void potentiallyAddHint(boolean isOuter) { if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions("HASH", "MERGE", "LOOKUP")); + return; + } else { + if (isOuter) { + sb.append(Randomly.fromOptions("HASH", "MERGE", "LOOKUP")); + } else { + sb.append(Randomly.fromOptions("HASH", "MERGE")); + } sb.append(" "); } } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java index 740a7901c..7259eed22 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java @@ -6,26 +6,17 @@ public class CockroachDBJoin implements CockroachDBExpression { private final CockroachDBExpression leftTable; private final CockroachDBExpression rightTable; - private final JoinType joinType; - private final CockroachDBExpression onCondition; - private OuterType outerType; + private JoinType joinType; + private CockroachDBExpression onCondition; public enum JoinType { - INNER, NATURAL, CROSS, OUTER; + INNER, LEFT, RIGHT, FULL, CROSS, NATURAL; public static JoinType getRandom() { return Randomly.fromOptions(values()); } } - public enum OuterType { - FULL, LEFT, RIGHT; - - public static OuterType getRandom() { - return Randomly.fromOptions(values()); - } - } - public CockroachDBJoin(CockroachDBExpression leftTable, CockroachDBExpression rightTable, JoinType joinType, CockroachDBExpression whereCondition) { this.leftTable = leftTable; @@ -42,35 +33,28 @@ public CockroachDBExpression getRightTable() { return rightTable; } - public JoinType getJoinType() { - return joinType; - } - - public CockroachDBExpression getOnCondition() { - return onCondition; - } - - public static CockroachDBJoin createNaturalJoin(CockroachDBExpression left, CockroachDBExpression right) { - return new CockroachDBJoin(left, right, JoinType.NATURAL, null); + public void setJoinType(JoinType joinType) { + this.joinType = joinType; } - public static CockroachDBJoin createCrossJoin(CockroachDBExpression left, CockroachDBExpression right) { - return new CockroachDBJoin(left, right, JoinType.CROSS, null); + public JoinType getJoinType() { + return joinType; } - public static CockroachDBJoin createOuterJoin(CockroachDBExpression left, CockroachDBExpression right, - OuterType type, CockroachDBExpression onClause) { - CockroachDBJoin join = new CockroachDBJoin(left, right, JoinType.OUTER, onClause); - join.setOuterType(type); - return join; + public void setOnCondition(CockroachDBExpression onCondition) { + this.onCondition = onCondition; } - private void setOuterType(OuterType outerType) { - this.outerType = outerType; + public CockroachDBExpression getOnCondition() { + return onCondition; } - public OuterType getOuterType() { - return outerType; + public static CockroachDBJoin createJoin(CockroachDBExpression left, CockroachDBExpression right, JoinType type, + CockroachDBExpression onClause) { + if (type.compareTo(JoinType.CROSS) >= 0) { + return new CockroachDBJoin(left, right, type, null); + } else { + return new CockroachDBJoin(left, right, type, onClause); + } } - } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java index 47de622d8..decbe927f 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java @@ -18,6 +18,11 @@ private CockroachDBInsertGenerator() { } public static SQLQueryAdapter insert(CockroachDBGlobalState globalState) { + CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + return insert(globalState, table); + } + + public static SQLQueryAdapter insert(CockroachDBGlobalState globalState, CockroachDBTable table) { ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addExpressionErrors(errors); // e.g., caused by computed columns @@ -32,7 +37,6 @@ public static SQLQueryAdapter insert(CockroachDBGlobalState globalState) { errors.add("foreign key violation"); errors.add("multi-part foreign key"); StringBuilder sb = new StringBuilder(); - CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean isUpsert = Randomly.getBoolean(); if (!isUpsert) { sb.append("INSERT INTO "); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 42740506d..cfc89562a 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -20,7 +20,8 @@ private enum CockroachDBClusterSetting { RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), SQL_STATS_HISTOGRAM_COLLECTION_ENABLED("sql.stats.histogram_collection.enabled", - CockroachDBSetSessionGenerator::onOff); + CockroachDBSetSessionGenerator::onOff), + HISTOGRAM_COLLECT("sql.stats.histogram_collection.enabled", CockroachDBSetSessionGenerator::onOff); private Function f; private String name; diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java index 09abbfbd0..9b84d9c94 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java @@ -23,7 +23,7 @@ public static SQLQueryAdapter show(CockroachDBGlobalState globalState) { case EXPERIMENTAL_FINGERPRINTS: sb.append("SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE "); sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); - errors.add("as type bytes: bytea encoded value ends with incomplete escape sequence"); + errors.add("bytea encoded value ends with incomplete escape sequence"); errors.add("invalid bytea escape sequence"); break; case DATABASES: diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 6ab133b1c..c627b9151 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -19,7 +19,6 @@ import sqlancer.cockroachdb.ast.CockroachDBColumnReference; import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBJoin.OuterType; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; @@ -75,24 +74,8 @@ public static List getJoins(List t columns.addAll(rightTable.getTable().getColumns()); CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) .setColumns(columns); - switch (CockroachDBJoin.JoinType.getRandom()) { - case INNER: - joinExpressions.add(new CockroachDBJoin(leftTable, rightTable, CockroachDBJoin.JoinType.INNER, - joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); - break; - case NATURAL: - joinExpressions.add(CockroachDBJoin.createNaturalJoin(leftTable, rightTable)); - break; - case CROSS: - joinExpressions.add(CockroachDBJoin.createCrossJoin(leftTable, rightTable)); - break; - case OUTER: - joinExpressions.add(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.getRandom(), - joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); - break; - default: - throw new AssertionError(); - } + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); } return joinExpressions; } From 5a69cad46238a075cb91715938defb55d138ec30 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 26 Jun 2023 20:55:39 +0800 Subject: [PATCH 0791/1284] Refactor: rename column variable names in InsertGenerators (#838) --- src/sqlancer/common/gen/AbstractInsertGenerator.java | 2 +- src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java | 2 +- src/sqlancer/h2/H2InsertGenerator.java | 2 +- src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/common/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java index 61357fd10..1a0b2a997 100644 --- a/src/sqlancer/common/gen/AbstractInsertGenerator.java +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -24,6 +24,6 @@ protected void insertColumns(List columns) { } } - protected abstract void insertValue(C tiDBColumn); + protected abstract void insertValue(C column); } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 1e5eb60b9..4fd3f390c 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -41,7 +41,7 @@ private SQLQueryAdapter generate() { } @Override - protected void insertValue(DuckDBColumn tiDBColumn) { + protected void insertValue(DuckDBColumn columnDuckDB) { // TODO: select a more meaningful value if (Randomly.getBooleanWithRatherLowProbability()) { sb.append("DEFAULT"); diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index cfb17846a..c4e559b0e 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -56,7 +56,7 @@ private SQLQueryAdapter generate() { } @Override - protected void insertValue(H2Column tiDBColumn) { + protected void insertValue(H2Column columnH2) { sb.append(H2ToStringVisitor.asString(gen.generateConstant())); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java index a27ec9c92..a1159d310 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -57,7 +57,7 @@ protected void insertColumns(List columns) { } @Override - protected void insertValue(YCQLColumn tiDBColumn) { + protected void insertValue(YCQLColumn columnYCQL) { // TODO: select a more meaningful value sb.append(YCQLToStringVisitor.asString(new YCQLExpressionGenerator(globalState).generateConstant())); } From 03ddf95b1ec9ee016c4298d913a0da2f2d236b1c Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 26 Jun 2023 23:32:07 +0800 Subject: [PATCH 0792/1284] cockroachdb: cert init (#842) * cockroachdb: cert init * tidb: getRandomExcept Join --- .../cockroachdb/CockroachDBOptions.java | 12 + .../cockroachdb/CockroachDBProvider.java | 19 +- .../cockroachdb/ast/CockroachDBJoin.java | 8 + .../oracle/CockroachDBCERTOracle.java | 275 ++++++++++++++++++ src/sqlancer/tidb/ast/TiDBJoin.java | 7 + src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 10 +- 6 files changed, 320 insertions(+), 11 deletions(-) create mode 100644 src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index c41b018ee..e50a30c51 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -12,6 +12,7 @@ import sqlancer.OracleFactory; import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; @@ -88,6 +89,17 @@ public TestOracle create(CockroachDBGlobalState globalSt oracles.add(new CockroachDBTLPDistinctOracle(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + CERT { + @Override + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } }; } diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index c2497e5c8..7ecd4de9c 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -36,6 +36,7 @@ import sqlancer.cockroachdb.gen.CockroachDBTruncateGenerator; import sqlancer.cockroachdb.gen.CockroachDBUpdateGenerator; import sqlancer.cockroachdb.gen.CockroachDBViewGenerator; +import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; @@ -130,13 +131,13 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio QueryManager manager = globalState.getManager(); MainOptions options = globalState.getOptions(); List standardSettings = new ArrayList<>(); - standardSettings.add("--Don't send automatic bug reports\n" - + "SET CLUSTER SETTING debug.panic_on_failed_assertions = true;"); + standardSettings.add("--Don't send automatic bug reports"); + standardSettings.add("SET CLUSTER SETTING debug.panic_on_failed_assertions = true;"); standardSettings.add("SET CLUSTER SETTING diagnostics.reporting.enabled = false;"); standardSettings.add("SET CLUSTER SETTING diagnostics.reporting.send_crash_reports = false;"); - standardSettings.add("-- Disable the collection of metrics and hope that it helps performance\n" - + "SET CLUSTER SETTING sql.metrics.statement_details.enabled = 'off'"); + standardSettings.add("-- Disable the collection of metrics and hope that it helps performance"); + standardSettings.add("SET CLUSTER SETTING sql.metrics.statement_details.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING sql.metrics.statement_details.plan_collection.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING sql.stats.automatic_collection.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING timeseries.storage.enabled = 'off'"); @@ -250,6 +251,16 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio } total--; } + + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 && globalState + .getDbmsSpecificOptions().getTestOracleFactory().get(0).create(globalState) instanceof CockroachDBCERTOracle) { + // Enfore statistic collected for all tables + ExpectedErrors errors = new ExpectedErrors(); + CockroachDBErrors.addExpressionErrors(errors); + for (CockroachDBTable table : globalState.getSchema().getDatabaseTables()) { + globalState.executeStatement(new SQLQueryAdapter("ANALYZE " + table.getName() + ";", errors)); + } + } } @Override diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java index 7259eed22..b53c4f681 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java @@ -1,5 +1,7 @@ package sqlancer.cockroachdb.ast; +import java.util.Arrays; + import sqlancer.Randomly; public class CockroachDBJoin implements CockroachDBExpression { @@ -15,6 +17,12 @@ public enum JoinType { public static JoinType getRandom() { return Randomly.fromOptions(values()); } + + public static JoinType getRandomExcept(JoinType... exclude) { + JoinType[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(JoinType[]::new); + return Randomly.fromOptions(values); + } } public CockroachDBJoin(CockroachDBExpression leftTable, CockroachDBExpression rightTable, JoinType joinType, diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java new file mode 100644 index 000000000..010188431 --- /dev/null +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -0,0 +1,275 @@ +package sqlancer.cockroachdb.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.cockroachdb.CockroachDBCommon; +import sqlancer.cockroachdb.CockroachDBErrors; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; +import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation; +import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation.CockroachDBBinaryLogicalOperator; +import sqlancer.cockroachdb.ast.CockroachDBColumnReference; +import sqlancer.cockroachdb.ast.CockroachDBExpression; +import sqlancer.cockroachdb.ast.CockroachDBJoin; +import sqlancer.cockroachdb.ast.CockroachDBJoin.JoinType; +import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.ast.CockroachDBTableReference; +import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CERTOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; + +public class CockroachDBCERTOracle extends CERTOracleBase + implements TestOracle { + private CockroachDBExpressionGenerator gen; + private CockroachDBSelect select; + + public CockroachDBCERTOracle(CockroachDBGlobalState globalState) { + super(globalState); + CockroachDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + queryPlan1Sequences = new ArrayList<>(); + queryPlan2Sequences = new ArrayList<>(); + + // Randomly generate a query + CockroachDBTables tables = state.getSchema().getRandomTableNonEmptyTables(2); + List tableList = CockroachDBCommon.getTableReferences( + tables.getTables().stream().map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList())); + gen = new CockroachDBExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new CockroachDBColumnReference(c)).collect(Collectors.toList())); + select = new CockroachDBSelect(); + select.setFetchColumns(fetchColumns); + select.setFromList(tableList); + select.setDistinct(Randomly.getBoolean()); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } + } + + // Set the join. + List joinExpressions = getJoins(tableList, state); + select.setJoinList(joinExpressions); + + // Get the result of the first query + String queryString1 = CockroachDBVisitor.asString(select); + int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + // Mutate the query + // Disable limit due to its false positive + boolean increase = mutate(); + + // Get the result of the second query + String queryString2 = CockroachDBVisitor.asString(select); + int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + // Check structural equivalence + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check the results + if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { + throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 + + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + } + } + + private List getJoins(List tableList, + CockroachDBGlobalState globalState) throws AssertionError { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getPercentage() < 0.8) { + CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); + CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, + CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + return joinExpressions; + } + + @Override + protected boolean mutateJoin() { + if (select.getJoinList().isEmpty()) { + return false; + } + + CockroachDBJoin join = (CockroachDBJoin) Randomly.fromList(select.getJoinList()); + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((CockroachDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((CockroachDBTableReference) join.getRightTable()).getTable().getColumns()); + CockroachDBExpressionGenerator joinGen2 = new CockroachDBExpressionGenerator(state).setColumns(columns); + join.setOnCondition(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); + } + + JoinType newJoinType = CockroachDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation + // between LEFT and RIGHT + // join + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); + } else { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); + } + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + return increase; + } + + @Override + protected boolean mutateDistinct() { + boolean increase = select.isDistinct(); + select.setDistinct(!select.isDistinct()); + return increase; + } + + @Override + protected boolean mutateWhere() { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } + return increase; + } + + @Override + protected boolean mutateGroupBy() { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + @Override + protected boolean mutateHaving() { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + @Override + protected boolean mutateAnd() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + @Override + protected boolean mutateOr() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + @Override + protected boolean mutateLimit() { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(gen.generateConstant(CockroachDBDataType.INT.get())); + } + return increase; + } + + private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + throws AssertionError, SQLException { + int row = -1; + String explainQuery = "EXPLAIN (VERBOSE) " + selectStr; + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String content = rs.getString(1); + if (content.contains("count:")) { + try { + int number = Integer.parseInt(content.split("count: ")[1].split(" ")[0].replace(",", "")); + if (row == -1) { + row = number; + } + } catch (Exception e) { // To avoid the situation that no number is found + } + } + if (content.contains("• ")) { + String operation = content.split("• ")[1].split(" ")[0]; + queryPlanSequences.add(operation); + } + } + } + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + if (row == -1) { + throw new IgnoreMeException(); + } + return row; + } + +} diff --git a/src/sqlancer/tidb/ast/TiDBJoin.java b/src/sqlancer/tidb/ast/TiDBJoin.java index e48c7a6f7..f4496a736 100644 --- a/src/sqlancer/tidb/ast/TiDBJoin.java +++ b/src/sqlancer/tidb/ast/TiDBJoin.java @@ -1,6 +1,7 @@ package sqlancer.tidb.ast; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import sqlancer.Randomly; @@ -22,6 +23,12 @@ public enum JoinType { public static JoinType getRandom() { return Randomly.fromOptions(values()); } + + public static JoinType getRandomExcept(JoinType... exclude) { + JoinType[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(JoinType[]::new); + return Randomly.fromOptions(values); + } } public enum NaturalJoinType { diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java index 1bef4ac96..4e56dd5fa 100644 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -129,14 +128,11 @@ protected boolean mutateJoin() { if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation // between LEFT and RIGHT // join - newJoinType = Randomly.fromOptions(Arrays.stream(JoinType.values()) - .filter(j -> j != JoinType.LEFT && j != JoinType.RIGHT && j != JoinType.NATURAL) - .toArray(JoinType[]::new)); + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); } else { - newJoinType = Randomly.fromOptions(Arrays.stream(JoinType.values()) - .filter(j -> j != join.getJoinType() && j != JoinType.NATURAL).toArray(JoinType[]::new)); + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); } - + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); join.setJoinType(newJoinType); if (newJoinType == JoinType.CROSS) { From 1aeb0f6447af76d72bfada610c2cb334f847ab42 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Wed, 28 Jun 2023 15:36:15 +0800 Subject: [PATCH 0793/1284] doc: add CERT (#843) * cockroachdb: formatting * doc: add CERT * cert: collect every tables for any cert oracle --- README.md | 6 ++++-- src/sqlancer/cockroachdb/CockroachDBProvider.java | 6 +++--- src/sqlancer/mysql/MySQLProvider.java | 7 +++---- src/sqlancer/tidb/TiDBProvider.java | 6 +++--- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f22b00f4d..5c18c887d 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Man SQLancer operates in the following two phases: -1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. **New: we support query plan guided generation now. See Generation Approaches below.** -2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. +1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. **News: we support Query Plan Guidance (QPG) now. See Generation Approaches below.** +2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. **News: we support Cardinality Estimation Restriction Testing (CERT) oracle now. See Testing Approaches below.** # Getting Started @@ -40,6 +40,8 @@ If you launch SQLancer without parameters, available options and commands are di | Pivoted Query Synthesis (PQS) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described [here](https://arxiv.org/abs/2001.04174). PQS is the most powerful technique, but also requires more implementation effort than the other two techniques. It is currently unmaintained. | | Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | | Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | +| Cardinality Estimation Restriction Testing (CERT) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. It is described [here](https://arxiv.org/abs/2306.00355). It derives a query to a more restrict query, whose estimated cardinality should be no more than that for the original query. An violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. | + # Generation Approaches | Approach | Description | |----------|-------------| diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 7ecd4de9c..494c63a5b 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -19,6 +19,7 @@ import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; +import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.gen.CockroachDBCommentOnGenerator; @@ -36,7 +37,6 @@ import sqlancer.cockroachdb.gen.CockroachDBTruncateGenerator; import sqlancer.cockroachdb.gen.CockroachDBUpdateGenerator; import sqlancer.cockroachdb.gen.CockroachDBViewGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; @@ -252,8 +252,8 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio total--; } - if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 && globalState - .getDbmsSpecificOptions().getTestOracleFactory().get(0).create(globalState) instanceof CockroachDBCERTOracle) { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == CockroachDBOracleFactory.CERT)) { // Enfore statistic collected for all tables ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 0c3a1f2e7..201e0c799 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -21,6 +21,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; @@ -38,7 +39,6 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLChecksum; import sqlancer.mysql.gen.tblmaintenance.MySQLOptimize; import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; -import sqlancer.mysql.oracle.MySQLCERTOracle; @AutoService(DatabaseProvider.class) public class MySQLProvider extends SQLProviderAdapter { @@ -157,9 +157,8 @@ public void generateDatabase(MySQLGlobalState globalState) throws Exception { }); se.executeStatements(); - if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 - && globalState.getDbmsSpecificOptions().getTestOracleFactory().get(0) - .create(globalState) instanceof MySQLCERTOracle) { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == MySQLOracleFactory.CERT)) { // Enfore statistic collected for all tables ExpectedErrors errors = new ExpectedErrors(); MySQLErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 3701a4006..3e4133761 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -23,6 +23,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.gen.TiDBAlterTableGenerator; @@ -36,7 +37,6 @@ import sqlancer.tidb.gen.TiDBTableGenerator; import sqlancer.tidb.gen.TiDBUpdateGenerator; import sqlancer.tidb.gen.TiDBViewGenerator; -import sqlancer.tidb.oracle.TiDBCERTOracle; @AutoService(DatabaseProvider.class) public class TiDBProvider extends SQLProviderAdapter { @@ -139,8 +139,8 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { } } - if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 && globalState - .getDbmsSpecificOptions().getTestOracleFactory().get(0).create(globalState) instanceof TiDBCERTOracle) { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == TiDBOracleFactory.CERT)) { // Disable strict Group By constraints for ROW oracle globalState.executeStatement(new SQLQueryAdapter( "SET @@sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';")); From a90884f8199c0b7e0e9284ed9f07aeeec0d21084 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:57:35 +0800 Subject: [PATCH 0794/1284] Feat: add support for StoneDB NoREC oracle (#845) * feat: add StoneDBNoRECOracle support * refactor: change \n to System.lineSeparator() * refactor: improve code quality --- src/sqlancer/stonedb/StoneDBSchema.java | 11 ++ src/sqlancer/stonedb/ast/StoneDBJoin.java | 126 ++++++++++++++++++ src/sqlancer/stonedb/ast/StoneDBSelect.java | 17 +++ .../gen/StoneDBExpressionGenerator.java | 15 +++ .../stonedb/oracle/StoneDBNoRECOracle.java | 110 ++++++++++++++- 5 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/stonedb/ast/StoneDBJoin.java create mode 100644 src/sqlancer/stonedb/ast/StoneDBSelect.java diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java index eccf05c27..58170ce73 100644 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -12,6 +12,7 @@ import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -99,6 +100,12 @@ public boolean hasPrimaryKey() { } + public static class StoneDBTables extends AbstractTables { + public StoneDBTables(List tables) { + super(tables); + } + } + public static final class StoneDBIndex extends TableIndex { private StoneDBIndex(String indexName) { super(indexName); @@ -339,4 +346,8 @@ public static StoneDBCompositeDataType getRandomWithoutNull() { return new StoneDBCompositeDataType(type); } } + + public StoneDBTables getRandomTableNonEmptyTables() { + return new StoneDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } } diff --git a/src/sqlancer/stonedb/ast/StoneDBJoin.java b/src/sqlancer/stonedb/ast/StoneDBJoin.java new file mode 100644 index 000000000..d32d49f5b --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBJoin.java @@ -0,0 +1,126 @@ +package sqlancer.stonedb.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator; + +public class StoneDBJoin implements Node { + + public enum JoinType { + INNER, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum NaturalJoinType { + FULL, LEFT, RIGHT; + + public static NaturalJoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + private NaturalJoinType naturalJoinType; + + public StoneDBJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node onCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = onCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + public NaturalJoinType getNaturalJoinType() { + return naturalJoinType; + } + + public void setNaturalJoinType(NaturalJoinType naturalJoinType) { + this.naturalJoinType = naturalJoinType; + } + + public static List> getJoins( + List> tableList, StoneDBGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + // get two tables to join + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + // store all columns in the above two tables + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + // create a join generator + StoneDBExpressionGenerator joinGen = new StoneDBExpressionGenerator(globalState).setColumns(columns); + switch (StoneDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(StoneDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + joinExpressions.add(StoneDBJoin.createNaturalJoin(leftTable, rightTable, NaturalJoinType.getRandom())); + break; + case LEFT: + joinExpressions + .add(StoneDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions + .add(StoneDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static StoneDBJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node onClause) { + return new StoneDBJoin(left, right, JoinType.RIGHT, onClause); + } + + public static StoneDBJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node onClause) { + return new StoneDBJoin(left, right, JoinType.LEFT, onClause); + } + + public static StoneDBJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node onClause) { + return new StoneDBJoin(left, right, JoinType.INNER, onClause); + } + + public static Node createNaturalJoin(TableReferenceNode left, + TableReferenceNode right, NaturalJoinType naturalJoinType) { + StoneDBJoin join = new StoneDBJoin(left, right, JoinType.NATURAL, null); + join.setNaturalJoinType(naturalJoinType); + return join; + } +} diff --git a/src/sqlancer/stonedb/ast/StoneDBSelect.java b/src/sqlancer/stonedb/ast/StoneDBSelect.java new file mode 100644 index 000000000..081106260 --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBSelect.java @@ -0,0 +1,17 @@ +package sqlancer.stonedb.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class StoneDBSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index cf578ecb0..59fb7e052 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -18,6 +18,7 @@ import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -35,6 +36,20 @@ private enum Expression { CASE } + public static class StoneDBCastOperation extends NewUnaryPostfixOperatorNode { + + public StoneDBCastOperation(Node expr, StoneDBCompositeDataType type) { + super(expr, new Operator() { + + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + } + + } + @Override public Node negatePredicate(Node predicate) { return null; diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index ac998b180..74ed75a91 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -1,16 +1,122 @@ package sqlancer.stonedb.oracle; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBSchema.StoneDBTables; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; public class StoneDBNoRECOracle extends NoRECBase implements TestOracle { - public StoneDBNoRECOracle(StoneDBGlobalState state) { - super(state); + + private final StoneDBSchema schema; + + public StoneDBNoRECOracle(StoneDBGlobalState globalState) { + super(globalState); + this.schema = globalState.getSchema(); } @Override public void check() throws Exception { + StoneDBTables randomTables = schema.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(state).setColumns(columns); + Node randomWhereCondition = gen.generateExpression(); + List tables = randomTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = StoneDBJoin.getJoins(tableList, state); + // get and check count + int secondCount = getUnoptimizedQueryCount(new ArrayList<>(tableList), randomWhereCondition, joins); + int firstCount = getOptimizedQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError(optimizedQueryString + "; -- " + firstCount + System.lineSeparator() + + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getUnoptimizedQueryCount(List> tableList, + Node randomWhereCondition, List> joins) throws SQLException { + StoneDBSelect select = new StoneDBSelect(); + Node asText = new NewPostfixTextNode<>(new StoneDBCastOperation( + new NewPostfixTextNode<>(randomWhereCondition, + " IS NOT NULL AND " + StoneDBToStringVisitor.asString(randomWhereCondition)), + new StoneDBCompositeDataType(StoneDBDataType.INT, 8)), "as count"); + select.setFetchColumns(List.of(asText)); + select.setFromList(tableList); + select.setJoinList(joins); + unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res"; + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError("error occurred when executing: \"" + unoptimizedQueryString + "\"", e); + } + if (rs == null) { + return -1; + } + int secondCount = 0; + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + private int getOptimizedQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, + List> joins) { + StoneDBSelect select = new StoneDBSelect(); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new StoneDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + select.setJoinList(joins); + int firstCount; + try (Statement stat = con.createStatement()) { + optimizedQueryString = StoneDBToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + firstCount = rs.getFetchSize(); + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; } } From bff57106f3e103124f0fc010933c0235c46bd277 Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:51:08 +0800 Subject: [PATCH 0795/1284] feat: Implement an AST based reducer prototype (#847) * feat: Implement an AST based reducer prototype + The reduer can run recursively visiting the ast-tree node and transforming the structure of the query + Provide some test cases at `test/reducer/TestASTBasedReducer.java` demonstrating the effect of the reducer. + Currently the reducer focus on only Select statements. Could be extended later. * fix format issues * feat: Add some support and do some modification according to the code review + Add support for String value shortening. + Add support for derived sub select. + Add some comments. + restore changes of settings files. * fix format issues * fix format issues * fix format issues * remove 1-element constaint when reducing list of expressions --- pom.xml | 5 + src/sqlancer/ASTBasedReducer.java | 530 ++++++++++++++++++ .../sqlancer/reducer/TestASTBasedReducer.java | 129 +++++ test/sqlancer/reducer/TestEnvironment.java | 23 +- .../reducer/TestStatementReducer.java | 6 +- 5 files changed, 688 insertions(+), 5 deletions(-) create mode 100644 src/sqlancer/ASTBasedReducer.java create mode 100644 test/sqlancer/reducer/TestASTBasedReducer.java diff --git a/pom.xml b/pom.xml index 46211aac2..4944369fa 100644 --- a/pom.xml +++ b/pom.xml @@ -363,6 +363,11 @@ commons-csv 1.9.0 + + com.github.jsqlparser + jsqlparser + 4.6 + diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java new file mode 100644 index 000000000..705d9d307 --- /dev/null +++ b/src/sqlancer/ASTBasedReducer.java @@ -0,0 +1,530 @@ +package sqlancer; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; + +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.CaseExpression; +import net.sf.jsqlparser.expression.DoubleValue; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.NullValue; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.WhenClause; +import net.sf.jsqlparser.expression.operators.relational.Between; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.expression.operators.relational.InExpression; +import net.sf.jsqlparser.expression.operators.relational.ItemsList; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.StatementVisitorAdapter; +import net.sf.jsqlparser.statement.select.GroupByElement; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectBody; +import net.sf.jsqlparser.statement.select.SelectExpressionItem; +import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; +import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.select.SubSelect; +import net.sf.jsqlparser.statement.select.WithItem; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; + +final class ExpressionTransformer { + + private static List flattenChildren(BinaryExpression expr) { + List candidates = new ArrayList<>(); + Expression lhs = expr.getLeftExpression(); + Expression rhs = expr.getRightExpression(); + candidates.add(lhs); + candidates.add(rhs); + return candidates; + } + + private static List flattenChildren(Between expr) { + Expression lhs = expr.getBetweenExpressionStart(); + Expression rhs = expr.getBetweenExpressionEnd(); + return List.of(lhs, rhs); + } + + public static List candidateExpressions(Expression expr) { + if (expr instanceof Parenthesis) { + // try removing a pair of brackets. + Parenthesis paren = (Parenthesis) expr; + return List.of(paren.getExpression()); + } else if (expr instanceof BinaryExpression) { + return flattenChildren((BinaryExpression) expr); + } else if (expr instanceof Between) { + return flattenChildren((Between) expr); + } else if (expr instanceof LongValue) { + LongValue longValue = (LongValue) expr; + if (String.valueOf(longValue).length() >= 4) { + return List.of(new NullValue(), new LongValue(10), new LongValue(0), new LongValue(1)); + } + return new ArrayList<>(); + } else if (expr instanceof DoubleValue) { + DoubleValue doubleValue = (DoubleValue) expr; + double literal = doubleValue.getValue(); + if (String.valueOf(literal).length() <= 4) { + return new ArrayList<>(); + } + double roundedValue = Math.round(literal * 10.0) / 10.0; + return List.of(new NullValue(), new DoubleValue(String.valueOf(roundedValue))); + } else if (expr instanceof StringValue) { + StringValue sv = (StringValue) expr; + String str = sv.getValue(); + if (str.length() > 4) { + return List.of(new NullValue(), new StringValue(" ")); + } + return new ArrayList<>(); + } else if (expr instanceof CaseExpression) { + CaseExpression caseExpression = (CaseExpression) expr; + return List.of(caseExpression.getSwitchExpression(), caseExpression.getElseExpression()); + } else { + return new ArrayList<>(); + } + } + + private ExpressionTransformer() throws Exception { + throw new AssertionError("Do not initialize the util class"); + } +} + +@SuppressWarnings("unchecked") +public class ASTBasedReducer, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> + implements Reducer { + + private final DatabaseProvider provider; + + @SuppressWarnings("unused") + private G state; + private G newGlobalState; + private Reproducer reproducer; + private int reduceTargetIndex; + private Statement targetStatement; + + // statement after reduction. + private List> reducedStatements; + + public ASTBasedReducer(DatabaseProvider provider) { + this.provider = provider; + } + + private void updateStatements() { + String queryString = targetStatement.toString(); + boolean couldAffectSchema = queryString.contains("CREATE TABLE") || queryString.contains("EXPLAIN"); + reducedStatements.set(reduceTargetIndex, (Query) new SQLQueryAdapter(queryString, couldAffectSchema)); + } + + public

void expressionReduce(P parent, Expression subExpr, // NOPMD + BiConsumer setter) { + boolean observeChange; + do { + observeChange = false; + List candidates = ExpressionTransformer.candidateExpressions(subExpr); + for (Expression candidate : candidates) { + try { + setter.accept(parent, candidate); + if (bugStillTriggers()) { + subExpr = candidate; + observeChange = true; + } + } catch (Exception ignoredException) { + } + } + setter.accept(parent, subExpr); + } while (observeChange); + } + + public void listElementRemovingReduce(P parent, List elms, // NOPMD + BiConsumer> setter) { + // TODO: For AST-Reducer, is delta-debugging needed ? Or just use the naive approach ? + boolean observeChange; + do { + observeChange = false; + for (int i = elms.size() - 1; i >= 0; i--) { + List reducedElms = new ArrayList<>(elms); + reducedElms.subList(i, i + 1).clear(); + setter.accept(parent, reducedElms); + try { + if (bugStillTriggers()) { + elms = reducedElms; + observeChange = true; + } + } catch (Exception e) { + System.out.println("An error occurred when trying executing reduced statements"); + e.printStackTrace(); + } + } + setter.accept(parent, elms); + } while (observeChange); + + } + + ExpressionVisitorAdapter expressionReducerVisitor = new ExpressionVisitorAdapter() { + + @Override + public void visit(InExpression expr) { + Expression rhs = expr.getRightExpression(); + + if (rhs instanceof SubSelect) { + SubSelect subSelect = (SubSelect) rhs; + subSelect.getSelectBody().accept(selectReducerVisitor); + } else { + ItemsList itemslist = expr.getRightItemsList(); + itemslist.accept(this); + } + } + + @Override + protected void visitBinaryExpression(BinaryExpression expr) { + Expression lhs = expr.getLeftExpression(); + Expression rhs = expr.getRightExpression(); + expressionReduce(expr, lhs, (expression, candidate) -> { + expression.setLeftExpression(candidate); + updateStatements(); + }); + expressionReduce(expr, rhs, (expression, candidate) -> { + expression.setRightExpression(candidate); + updateStatements(); + }); + lhs.accept(this); + rhs.accept(this); + } + + // @Override + // public void visit(DateValue value) { + // super.visit(value); + // } + // + // @Override + // public void visit(TimeValue value) { + // super.visit(value); + // } + // + // @Override + // public void visit(LikeExpression expr) { + // super.visit(expr); + // } + + @Override + public void visit(CaseExpression expr) { + Expression switchExpr = expr.getSwitchExpression(); + Expression elseExpr = expr.getElseExpression(); + + expressionReduce(expr, switchExpr, (parent, sw) -> { + parent.setSwitchExpression(sw); + updateStatements(); + }); + expressionReduce(expr, elseExpr, (parent, els) -> { + parent.setElseExpression(els); + updateStatements(); + }); + + super.visit(expr); + } + + @Override + public void visit(WhenClause whenClause) { + Expression when = whenClause.getWhenExpression(); + Expression then = whenClause.getThenExpression(); + + expressionReduce(whenClause, when, (wc, w) -> { + wc.setWhenExpression(w); + updateStatements(); + }); + + expressionReduce(whenClause, then, (wc, t) -> { + wc.setThenExpression(t); + updateStatements(); + }); + + super.visit(whenClause); + } + + @Override + public void visit(Parenthesis parenthesis) { + Expression closedExpr = parenthesis.getExpression(); + expressionReduce(parenthesis, closedExpr, (p, s) -> { + p.setExpression(s); + updateStatements(); + }); + closedExpr.accept(this); + } + + // @Override + // public void visit(Function function) { + // super.visit(function); + // } + + @Override + public void visit(ExpressionList expressionList) { + List expressions = expressionList.getExpressions(); + listElementRemovingReduce(expressionList, expressions, (l, es) -> { + l.setExpressions(es); + updateStatements(); + }); + expressions = expressionList.getExpressions(); + for (int i = 0; i < expressions.size(); i++) { + Expression expr = expressions.get(i); + int index = i; + expressionReduce(expressions, expr, (l, e) -> { + l.set(index, e); + updateStatements(); + }); + } + super.visit(expressionList); + } + + @Override + public void visit(SelectExpressionItem selectExpressionItem) { + Expression expr = selectExpressionItem.getExpression(); + expressionReduce(selectExpressionItem, expr, (item, e) -> { + item.setExpression(e); + updateStatements(); + }); + super.visit(selectExpressionItem); + } + + @Override + public void visit(SubSelect subSelect) { + subSelect.getSelectBody().accept(selectReducerVisitor); + } + + }; + + SelectVisitorAdapter selectReducerVisitor = new SelectVisitorAdapter() { + + // Clauses that would be tried removing. + // examples: + // Remove when: select * from table when 1 -> select * from table + // Remove limit: select * from table limit 1 -> select * from table + private final String[] removeList = { "Limit", "Offset", "Where", "Having", "GroupBy", "Distinct", + "OrderByElements", "Joins" }; + + // Clauses that would be tried transforming. + // examples: + // select * from t where a + b < c + d + // where clause might become one of the statement below after transformation: + // -> select * from table where c + d + // -> select * from table where a + b + + private final String[] transformList = { "Where", "Having", "FromItem", "SelectItems", "GroupBy", "Joins" }; + + // Clauses that would be visited for further reduction. + // example: + // select * from t where a + b < c + d + // Assuming that the coexistence of a and c would trigger the bug. + // The where clause : a + b < c + d would be visited and a + b, c + d would be reduced respectively. + // a + b < c + d might become a + c + private final String[] descendList = { "Where", "Having", "FromItem", "SelectItems", "GroupBy" }; + + private String getterName(String astNodeName) { + return "get" + astNodeName; + } + + private String setterName(String astNodeName) { + if (astNodeName.equals("GroupBy")) { + return "set" + astNodeName + "Element"; + } else { + return "set" + astNodeName; + } + } + + @Override + public void visit(WithItem withItem) { + // withItem.getItemsList(); + } + + @Override + public void visit(PlainSelect plainSelect) { + // transform section. Lists defined above would be iterated to get the corresponding clause name. Reflection + // is used to avoid repetitive code. e.g. The current astNodeName is When `getWhen`, `setWhen` would be + // called. + for (String astNodeName : removeList) { + try { + Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); + Object astNode = nodeGetter.invoke(plainSelect); + if (astNode == null) { + continue; + } + Method nodeSetter = plainSelect.getClass().getMethod(setterName(astNodeName), + nodeGetter.getReturnType()); + nodeSetter.invoke(plainSelect, new Object[] { null }); + updateStatements(); + if (!bugStillTriggers()) { + nodeSetter.invoke(plainSelect, astNode); + updateStatements(); + } + } catch (Exception e) { + throw new AssertionError(e); + } + } + + // Pull Up Section + for (String astNodeName : transformList) { + try { + Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); + Object astNode = nodeGetter.invoke(plainSelect); + if (astNode == null) { + continue; + } + Method nodeSetter = plainSelect.getClass().getMethod(setterName(astNodeName), + nodeGetter.getReturnType()); + + if (astNode instanceof Expression) { + Expression expr = (Expression) astNode; + expressionReduce(plainSelect, expr, (select, expression) -> { + try { + nodeSetter.invoke(select, expression); + updateStatements(); + } catch (IllegalAccessException | InvocationTargetException e) { + e.printStackTrace(); + } + }); + } else if (astNode instanceof List) { + List elms = (List) astNode; + if (elms.size() <= 1) { + continue; + } + listElementRemovingReduce(plainSelect, elms, (select, items) -> { + try { + nodeSetter.invoke(select, items); + updateStatements(); + } catch (IllegalAccessException | InvocationTargetException e) { + e.printStackTrace(); + } + }); + } else if (astNode instanceof GroupByElement) { + GroupByElement groupByElement = (GroupByElement) astNode; + ExpressionList expressionList = groupByElement.getGroupByExpressionList(); + if (expressionList == null) { + groupByElement.getGroupingSets(); + // TODO: TO BE IMPLEMENTED. + } else { + List elms = expressionList.getExpressions(); + if (elms.size() <= 1) { + continue; + } + listElementRemovingReduce(groupByElement, elms, (select, items) -> { + groupByElement.setGroupByExpressionList(new ExpressionList(items)); + updateStatements(); + }); + } + } + } catch (Exception e) { + throw new AssertionError(e); + } + } + + for (String astNodeName : descendList) { + try { + Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); + Object astNode = nodeGetter.invoke(plainSelect); + if (astNode == null) { + continue; + } + if (astNode instanceof Expression) { + ((Expression) astNode).accept(expressionReducerVisitor); + } else if (astNode instanceof List) { + // Really hacky... Some other ways to simplify it ? + List elms = (List) astNode; + for (Object obj : elms) { + if (obj instanceof SelectExpressionItem) { + ((SelectExpressionItem) obj).accept(expressionReducerVisitor); + } else if (obj instanceof Expression) { + ((Expression) obj).accept(expressionReducerVisitor); + } + } + } else if (astNode instanceof GroupByElement) { + GroupByElement groupByElement = (GroupByElement) astNode; + ExpressionList expressionList = groupByElement.getGroupByExpressionList(); + if (expressionList != null) { + expressionList.accept(expressionReducerVisitor); + } + // TODO: groupByElement.getGroupingSets() TO BE IMPLEMENTED + + } + } catch (Exception e) { + throw new AssertionError(e); + } + } + } + + @Override + public void visit(SetOperationList setOpList) { + List selectBodies = setOpList.getSelects(); + listElementRemovingReduce(setOpList, selectBodies, (optionList, selects) -> { + optionList.setSelects(selects); + updateStatements(); + }); + for (SelectBody selectBody : selectBodies) { + if (selectBody instanceof PlainSelect) { + visit((PlainSelect) selectBody); + } + } + } + + }; + + StatementVisitorAdapter statementReducerVisitor = new StatementVisitorAdapter() { + @Override + public void visit(Select select) { + SelectBody selectBody = select.getSelectBody(); + if (selectBody != null) { + selectBody.accept(selectReducerVisitor); + } + } + }; + + @Override + public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + this.state = state; + this.newGlobalState = newGlobalState; + this.reproducer = reproducer; + + List> initialBugInducingStatements = state.getState().getStatements(); + reducedStatements = new ArrayList<>(); + for (Query query : initialBugInducingStatements) { + reducedStatements.add((Query) query); + } + + for (int i = 0; i < reducedStatements.size(); i++) { + reduceTargetIndex = i; + Query query = reducedStatements.get(reduceTargetIndex); + targetStatement = CCJSqlParserUtil.parse(query.getQueryString()); + targetStatement.accept(statementReducerVisitor); + } + + newGlobalState.getState().setStatements(new ArrayList<>(reducedStatements)); + } + + private boolean bugStillTriggers() throws Exception { + try (C con2 = provider.createDatabase(newGlobalState)) { + newGlobalState.setConnection(con2); + List> candidateStatements = new ArrayList<>(reducedStatements); + newGlobalState.getState().setStatements(new ArrayList<>(candidateStatements)); + + for (Query s : candidateStatements) { + try { + s.execute(newGlobalState); + } catch (Throwable ignoredException) { + // ignore + } + } + try { + if (reproducer.bugStillTriggers(newGlobalState)) { + return true; + } + } catch (Throwable ignoredException) { + + } + } + return false; + } +} diff --git a/test/sqlancer/reducer/TestASTBasedReducer.java b/test/sqlancer/reducer/TestASTBasedReducer.java new file mode 100644 index 000000000..b65ceef21 --- /dev/null +++ b/test/sqlancer/reducer/TestASTBasedReducer.java @@ -0,0 +1,129 @@ +package sqlancer.reducer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.query.Query; + +import java.util.Arrays; +import java.util.List; + +public class TestASTBasedReducer { + @Test + void testLongStatement() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStr = { + "SELECT DISTINCT * FROM v0 WHERE ((v0.rowid || ( (v0.c + v0.d) < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)" }; + env.setInitialStatementsFromStrings(List.of(queriesStr)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("&&"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStr)); + System.out.println(reducedResult); + } + + @Test + void testReducingMultipleTokensToOne() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStr = { + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)" }; + env.setInitialStatementsFromStrings(List.of(queriesStr)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("||"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStr)); + System.out.println(reducedResult); + } + + @Test + void testMultipleStatements() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)", + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)", + "SELECT * FROM table_3;" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.toUpperCase().contains("UNION"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } + + @Test + void testJoin() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { "SELECT * FROM t0, t1, t2, t3, t4 Where t2.val = t1.val" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("t1") && queriesString.contains("WHERE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } + + @Test + void testCase() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT STRING_AGG(v0.c2) FROM t0, v0 WHERE (CASE true WHEN (ABS(true) BETWEEN (v0.c0 LIKE NULL ESCAPE v0.c2) AND (DATE '1970-01-23' NOT IN (v0.c2))) THEN (0.07914839711718646 NOT BETWEEN '' AND ((v0.c0)OR(v0.c2))) WHEN v0.c1 THEN ((v0.c1)-(v0.c0)) WHEN t0.c1 THEN (TIMESTAMP '1969-12-29 20:22:33' IN (PI(), v0.c2, (v0.c1 BETWEEN '' AND v0.rowid))) WHEN v0.c1 THEN TIMESTAMP '1969-12-16 17:24:43' WHEN ((((v0.c1)-(t0.c0)))||(t0.c0)) THEN true ELSE ((0.279978719843174)/(((v0.c1)>(DATE '1969-12-19')))) END ) GROUP BY ((DATE '1970-01-24') IS NULL), t0.c1, (CASE (v0.c1 LIKE ((0.9833120083624495)SIMILAR TO(t0.rowid)) ESCAPE CEIL(TIMESTAMP '1970-01-11 16:38:26')) WHEN t0.rowid THEN 0.27742217994251717 ELSE ((v0.c0) IS NOT NULL) END );" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.toUpperCase().contains("CASE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } + + @Test + void testConstantVar() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT STRING_AGG(v0.c2) FROM t0 GROUP BY ( (CASE (t0.rowid) WHEN t0.rowid THEN 0.27742217994251717 ELSE ((v0.c0) IS NOT NULL) END) )" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.toUpperCase().contains("CASE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } + + @Test + void testFunction() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT DATE '1970-01-11', false, t1.c1, t1.c1, (t1.c1 NOT IN (((('' LIKE t1.c2 ESCAPE t1.c1)) IS NOT NULL))) FROM t1 WHERE t1.c0 GROUP BY (((('Zlb)' IN (t1.c0)) LIKE t1.c1 ESCAPE (0.6419925594156123 BETWEEN t1.c1 AND ')-'))) ::BOOL) HAVING ((LAST_DAY(1630554083))&(AVG((CASE t1.c0 WHEN t1.c2 THEN DATE '1970-01-09' ELSE '' END )))) LIMIT 714775291;" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("AVG"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } +} diff --git a/test/sqlancer/reducer/TestEnvironment.java b/test/sqlancer/reducer/TestEnvironment.java index 3329597aa..3ba1fc11e 100644 --- a/test/sqlancer/reducer/TestEnvironment.java +++ b/test/sqlancer/reducer/TestEnvironment.java @@ -21,8 +21,27 @@ public class TestEnvironment { private VirtualDBProvider provider = null; private VirtualDBGlobalState state, newGlobalState; - public TestEnvironment() throws Exception { + private Reducer reducer = null; + + enum ReducerType { + USING_STATEMENT_REDUCER, USING_AST_BASED_REDUCER + }; + + private TestEnvironment(ReducerType type) throws Exception { setUpTestingEnvironment(); + if (type == ReducerType.USING_STATEMENT_REDUCER) { + reducer = new StatementReducer<>(provider); + } else if (type == ReducerType.USING_AST_BASED_REDUCER) { + reducer = new ASTBasedReducer<>(provider); + } + } + + public static TestEnvironment getStatementReducerEnv() throws Exception { + return new TestEnvironment(ReducerType.USING_STATEMENT_REDUCER); + } + + public static TestEnvironment getASTBasedReducerEnv() throws Exception { + return new TestEnvironment(ReducerType.USING_AST_BASED_REDUCER); } /** @@ -99,7 +118,7 @@ public void setBugInducingCondition(Function>, Boolean> bugInducin } public void runReduce() throws Exception { - Reducer reducer = new StatementReducer<>(provider); + Reproducer reproducer = provider.generateAndTestDatabase(newGlobalState); reducer.reduce(state, reproducer, newGlobalState); } diff --git a/test/sqlancer/reducer/TestStatementReducer.java b/test/sqlancer/reducer/TestStatementReducer.java index 83c7de94e..66a9a3c4c 100644 --- a/test/sqlancer/reducer/TestStatementReducer.java +++ b/test/sqlancer/reducer/TestStatementReducer.java @@ -13,7 +13,7 @@ public class TestStatementReducer { @Test void testSimple() throws Exception { - TestEnvironment env = new TestEnvironment(); + TestEnvironment env = TestEnvironment.getStatementReducerEnv(); String[] queriesStr = { "CREATE TABLE FAKE_TABLE;", "SELECT * FROM FAKE_TABLE;", "EXIT", }; env.setInitialStatementsFromStrings(List.of(queriesStr)); @@ -30,7 +30,7 @@ void testSimple() throws Exception { @Test void testDeltaDebugging() throws Exception { - TestEnvironment env = new TestEnvironment(); + TestEnvironment env = TestEnvironment.getStatementReducerEnv(); List fakeStatements = new ArrayList<>(); for (int i = 0; i < 10000; i++) { String statement = "Statement_" + i + ";"; @@ -51,7 +51,7 @@ void testDeltaDebugging() throws Exception { @Test void testDeltaDebuggingWithStatementsCombination() throws Exception { - TestEnvironment env = new TestEnvironment(); + TestEnvironment env = TestEnvironment.getStatementReducerEnv(); List fakeStatements = new ArrayList<>(); String pattern = "(.*\\n)*(Statement_2;)\\n(.*\\n)*(Statement_318);\\n(.*\\n)*(Statement_990;)(.*\\n)*.*"; From 5bcd9ce69898f2866094535610ab0b4c04778843 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Wed, 5 Jul 2023 20:47:03 +0800 Subject: [PATCH 0796/1284] Feat: add TestStoneDB.java in test/sqlancer/dbms (#844) * feat: set up CI for StoneDB * feat: set up mvn -Dtest for StoneDB * fix: typo * feat: add CI entry for mvn test StoneDB * fix: change to StoneDB binary in CI * fix: change to StoneDB binary in CI * drop: manually drop CI commit * feat: add stonedb 5.7 CI * fix: rename and free space * fix: free up space again * fix: delete useless df . -H * fix: add space check and fix * fix: change to v1.0.3 * fix: typo * refactor: delete useless space check --- .github/workflows/main.yml | 29 ++++++++++++++++++++++++++++- test/sqlancer/dbms/TestStoneDB.java | 22 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 test/sqlancer/dbms/TestStoneDB.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1bafaf3d1..990543c0d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -375,6 +375,33 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test + stonedb: + name: DBMS Tests (StoneDB) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up StoneDB + run: | + docker pull stoneatom/stonedb:v1.0.3 + docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb:v1.0.3 + docker ps + ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb:v1.0.3 --format "{{.ID}}") + docker exec $ContainerID bash + sleep 180s + docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + - name: Set up JDK 11 + uses: actions/setup-java@v1.4.4 + with: + java-version: 11 + - name: Build + run: mvn -B package -DskipTests=true + - name: StoneDB Tests + run: | + STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDB + tidb: name: DBMS Tests (TiDB) runs-on: ubuntu-latest @@ -550,4 +577,4 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - mvn -Dtest=TestStatementReducer test + mvn -Dtest=TestStatementReducer test \ No newline at end of file diff --git a/test/sqlancer/dbms/TestStoneDB.java b/test/sqlancer/dbms/TestStoneDB.java new file mode 100644 index 000000000..14de121be --- /dev/null +++ b/test/sqlancer/dbms/TestStoneDB.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestStoneDB { + + String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); + boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); + + @Test + public void testStoneDB() { + assumeTrue(stoneDBIsAvailable); + assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", + "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "NoREC")); + } + +} From 7f99a129ad4d6ef60816bfc62c2e38d350a4a337 Mon Sep 17 00:00:00 2001 From: Nathee Jaywaree <73115539+eltfshr@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:08:50 +0800 Subject: [PATCH 0797/1284] feat: add MySQLUpdateGenerator class (#849) * Add MySQLUpdateGenerator class * Extract common update insert errors in mysql --- src/sqlancer/mysql/MySQLErrors.java | 11 ++++ src/sqlancer/mysql/MySQLProvider.java | 5 ++ .../mysql/gen/MySQLInsertGenerator.java | 10 +--- .../mysql/gen/MySQLUpdateGenerator.java | 58 +++++++++++++++++++ 4 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/mysql/gen/MySQLUpdateGenerator.java diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 540a5c5db..9a8f076ce 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -16,4 +16,15 @@ public static void addExpressionErrors(ExpectedErrors errors) { } } + public static void addInsertUpdateErrors(ExpectedErrors errors) { + errors.add("doesn't have a default value"); + errors.add("Data truncation"); + errors.add("Incorrect integer value"); + errors.add("Duplicate entry"); + errors.add("Data truncated for column"); + errors.add("Data truncated for functional index"); + errors.add("cannot be null"); + errors.add("Incorrect decimal value"); + } + } diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 201e0c799..9392550cc 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -31,6 +31,7 @@ import sqlancer.mysql.gen.MySQLSetGenerator; import sqlancer.mysql.gen.MySQLTableGenerator; import sqlancer.mysql.gen.MySQLTruncateTableGenerator; +import sqlancer.mysql.gen.MySQLUpdateGenerator; import sqlancer.mysql.gen.admin.MySQLFlush; import sqlancer.mysql.gen.admin.MySQLReset; import sqlancer.mysql.gen.datadef.MySQLIndexGenerator; @@ -67,6 +68,7 @@ enum Action implements AbstractAction { String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); return MySQLTableGenerator.generate(g, tableName); }), // + UPDATE(MySQLUpdateGenerator::create), // DELETE(MySQLDeleteGenerator::delete), // DROP_INDEX(MySQLDropIndex::generate); @@ -132,6 +134,9 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { case SELECT_INFO: nrPerformed = r.getInteger(0, 10); break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; case DELETE: nrPerformed = r.getInteger(0, 10); break; diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 14994a140..86083fd2d 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -7,6 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -88,14 +89,7 @@ private SQLQueryAdapter generateInto() { } sb.append(")"); } - errors.add("doesn't have a default value"); - errors.add("Data truncation"); - errors.add("Incorrect integer value"); - errors.add("Duplicate entry"); - errors.add("Data truncated for functional index"); - errors.add("Data truncated for column"); - errors.add("cannot be null"); - errors.add("Incorrect decimal value"); + MySQLErrors.addInsertUpdateErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java new file mode 100644 index 000000000..55ba3dd45 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java @@ -0,0 +1,58 @@ +package sqlancer.mysql.gen; + +import java.sql.SQLException; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; + +public class MySQLUpdateGenerator extends AbstractUpdateGenerator { + + private final MySQLGlobalState globalState; + private MySQLExpressionGenerator gen; + + public MySQLUpdateGenerator(MySQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter create(MySQLGlobalState globalState) throws SQLException { + return new MySQLUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() throws SQLException { + MySQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new MySQLExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + MySQLErrors.addExpressionErrors(errors); + sb.append(MySQLVisitor.asString(gen.generateExpression())); + } + MySQLErrors.addInsertUpdateErrors(errors); + errors.add("doesn't have this option"); + + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void updateValue(MySQLColumn column) { + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant()); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append(MySQLVisitor.asString(gen.generateExpression())); + } + } + +} From 26de30200a03b6ed13b16f4c548d9783aa78cd5c Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Fri, 7 Jul 2023 20:32:50 +0800 Subject: [PATCH 0798/1284] fix: SQL syntax errors and add expected errors in StoneDB implementation Part 1 (#846) * fix: add space when creating table * fix: add expected errors and key length specification * feat: generate comments randomly * fix: add expected errors and key restriction * feat: add support for more map actions * fix: add space between INDEX and indexName * fix: add space and {} * fix: remove pre-added errors before testing * refactor: rename delete useless as when deleting from table * fix: add missing space * fix: add missing ( * fix: add missing length for varbinary type * fix: string generation strategy in comments * fix: add provider * fix: use global randomly * fix: insert one row * feat: add expected errors * feat: no length * feat: no length * style: format * fix: Tianmu engine does not support fulltext index. * fix: fix alter table and getwithoutnull * fix: add expected errors * feat: support StringVisitor of StoneSelect * fix: delete useless import * fix: syntax error in index create * style: run formate * fix: syntax errors * fix: delete AS in DELETE * feat: add expectedErrors for all generator * refactor: change cast operation to lambda and fix dataType toString error * feat: add expected errors * refactor: remove redundant identifier * refactor: remove unused expected errors * refactor: use parent StringBuilder * refactor: add more expected errors * refactor: change action perform times * refactor: rename and change action perform times * fix: delete unexcepted errors * refactor: delete useless code --- src/sqlancer/Main.java | 2 + src/sqlancer/stonedb/StoneDBProvider.java | 44 ++++++-- src/sqlancer/stonedb/StoneDBSchema.java | 7 ++ .../stonedb/StoneDBToStringVisitor.java | 62 ++++++++++ .../gen/StoneDBExpressionGenerator.java | 11 +- .../gen/StoneDBIndexCreateGenerator.java | 106 ++++++++++++++++++ .../stonedb/gen/StoneDBIndexGenerator.java | 75 ------------- .../gen/StoneDBTableAlterGenerator.java | 19 +++- .../gen/StoneDBTableCreateGenerator.java | 64 ++++++----- ....java => StoneDBTableDeleteGenerator.java} | 19 ++-- ....java => StoneDBTableInsertGenerator.java} | 25 ++++- .../stonedb/oracle/StoneDBNoRECOracle.java | 3 +- 12 files changed, 290 insertions(+), 147 deletions(-) create mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java rename src/sqlancer/stonedb/gen/{StoneDBDeleteGenerator.java => StoneDBTableDeleteGenerator.java} (77%) rename src/sqlancer/stonedb/gen/{StoneDBInsertGenerator.java => StoneDBTableInsertGenerator.java} (73%) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index dfdc949d6..76d7a36f4 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -45,6 +45,7 @@ import sqlancer.postgres.PostgresProvider; import sqlancer.questdb.QuestDBProvider; import sqlancer.sqlite3.SQLite3Provider; +import sqlancer.stonedb.StoneDBProvider; import sqlancer.tidb.TiDBProvider; import sqlancer.timescaledb.TimescaleDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; @@ -660,6 +661,7 @@ private static void checkForIssue799(List> providers) providers.add(new PostgresProvider()); providers.add(new QuestDBProvider()); providers.add(new SQLite3Provider()); + providers.add(new StoneDBProvider()); providers.add(new TiDBProvider()); providers.add(new TimescaleDBProvider()); providers.add(new YCQLProvider()); diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java index 376235d92..e0c317c53 100644 --- a/src/sqlancer/stonedb/StoneDBProvider.java +++ b/src/sqlancer/stonedb/StoneDBProvider.java @@ -19,11 +19,13 @@ import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; -import sqlancer.stonedb.gen.StoneDBDeleteGenerator; -import sqlancer.stonedb.gen.StoneDBIndexGenerator; -import sqlancer.stonedb.gen.StoneDBInsertGenerator; +import sqlancer.stonedb.gen.StoneDBIndexCreateGenerator; +import sqlancer.stonedb.gen.StoneDBIndexDropGenerator; import sqlancer.stonedb.gen.StoneDBTableAlterGenerator; import sqlancer.stonedb.gen.StoneDBTableCreateGenerator; +import sqlancer.stonedb.gen.StoneDBTableDeleteGenerator; +import sqlancer.stonedb.gen.StoneDBTableInsertGenerator; +import sqlancer.stonedb.gen.StoneDBTableUpdateGenerator; @AutoService(DatabaseProvider.class) public class StoneDBProvider extends SQLProviderAdapter { @@ -40,11 +42,17 @@ protected StoneDBSchema readSchema() throws Exception { } enum Action implements AbstractAction { - SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), // - ALTER_TABLE(StoneDBTableAlterGenerator::generate), // - DELETE(StoneDBDeleteGenerator::generate), // - INDEX(StoneDBIndexGenerator::generate), // - INSERT(StoneDBInsertGenerator::generate); // + TABLES_SHOW((g) -> new SQLQueryAdapter("SHOW TABLES")), // + TABLE_ALTER(StoneDBTableAlterGenerator::generate), // + TABLE_CREATE((g) -> { + String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); + return StoneDBTableCreateGenerator.generate(g, tableName); + }), // + TABLE_DELETE(StoneDBTableDeleteGenerator::generate), // + TABLE_UPDATE(StoneDBTableUpdateGenerator::generate), // + INDEX_CREATE(StoneDBIndexCreateGenerator::generate), // + INDEX_DROP(StoneDBIndexDropGenerator::generate), // + TABLE_INSERT(StoneDBTableInsertGenerator::generate); // private final SQLQueryProvider sqlQueryProvider; @@ -59,10 +67,24 @@ public Query getQuery(StoneDBGlobalState globalState) throws Exception { } private static int mapActions(StoneDBGlobalState globalState, Action a) { - globalState.getRandomly(); + Randomly r = globalState.getRandomly(); switch (a) { - case SHOW_TABLES: - return 1; + case TABLES_SHOW: + return r.getInteger(0, 1); + case TABLE_ALTER: + return r.getInteger(0, 5); + case TABLE_CREATE: + return r.getInteger(0, 1); + case TABLE_DELETE: + return r.getInteger(0, 10); + case TABLE_INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case TABLE_UPDATE: + return r.getInteger(0, 1); + case INDEX_CREATE: + return r.getInteger(0, 1); + case INDEX_DROP: + return r.getInteger(0, 1); default: throw new AssertionError(a); } diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java index 58170ce73..c24b67793 100644 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -5,6 +5,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -25,6 +27,11 @@ public enum StoneDBDataType { CHAR, VARCHAR, TINYTEXT, TEXT, MEDIUMTEXT, LONGTEXT, BINARY, VARBINARY, TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB; public static StoneDBDataType getRandomWithoutNull() { + List collect = Stream.of(values()).filter(c -> c != NULL).collect(Collectors.toList()); + return Randomly.fromList(collect); + } + + public static StoneDBDataType getRandom() { return Randomly.fromOptions(values()); } diff --git a/src/sqlancer/stonedb/StoneDBToStringVisitor.java b/src/sqlancer/stonedb/StoneDBToStringVisitor.java index 3cebd97f1..9d7711873 100644 --- a/src/sqlancer/stonedb/StoneDBToStringVisitor.java +++ b/src/sqlancer/stonedb/StoneDBToStringVisitor.java @@ -4,12 +4,18 @@ import sqlancer.common.ast.newast.Node; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; public class StoneDBToStringVisitor extends NewToStringVisitor { @Override public void visitSpecific(Node expr) { if (expr instanceof StoneDBConstant) { visit((StoneDBConstant) expr); + } else if (expr instanceof StoneDBSelect) { + visit((StoneDBSelect) expr); + } else if (expr instanceof StoneDBJoin) { + visit((StoneDBJoin) expr); } else { throw new AssertionError(expr.getClass()); } @@ -19,6 +25,62 @@ private void visit(StoneDBConstant constant) { sb.append(constant.toString()); } + private void visit(StoneDBSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + private void visit(StoneDBJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getNaturalJoinType() != null) { + sb.append(join.getNaturalJoinType()); + } + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + public static String asString(Node expr) { StoneDBToStringVisitor visitor = new StoneDBToStringVisitor(); visitor.visit(expr); diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 59fb7e052..385154815 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -18,7 +18,6 @@ import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -38,14 +37,8 @@ private enum Expression { public static class StoneDBCastOperation extends NewUnaryPostfixOperatorNode { - public StoneDBCastOperation(Node expr, StoneDBCompositeDataType type) { - super(expr, new Operator() { - - @Override - public String getTextRepresentation() { - return "::" + type.toString(); - } - }); + public StoneDBCastOperation(Node expr, StoneDBDataType type) { + super(expr, () -> "::" + type.toString()); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java new file mode 100644 index 000000000..3b7e63422 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java @@ -0,0 +1,106 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.Randomly.StringGenerationStrategy; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; + +public class StoneDBIndexCreateGenerator { + private final StoneDBGlobalState globalState; + private final Randomly r; + // which table to add index + StoneDBTable table; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBIndexCreateGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + r = globalState.getRandomly(); + table = globalState.getSchema().getRandomTable(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBIndexCreateGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("CREATE "); + sb.append(Randomly.fromOptions("UNIQUE", "FULLTEXT", "SPATIAL")); + sb.append(" INDEX "); + sb.append(globalState.getSchema().getFreeIndexName()); + appendIndexType(); + sb.append(" ON "); + sb.append(table.getName()); + appendKeyPart(); + appendIndexOption(); + appendAlgoOrLockOption(); + addExpectedErrors(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void addExpectedErrors() { + // java.sql.SQLException: Tianmu engine does not support unique index. + errors.add("Tianmu engine does not support unique index"); + // java.sql.SQLException: Tianmu engine does not support fulltext index. + errors.add("Tianmu engine does not support fulltext index"); + // java.sql.SQLException: The used table type doesn't support SPATIAL indexes + errors.add("The used table type doesn't support SPATIAL indexes"); + // java.sql.SQLException: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY. + errors.add("ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY."); + // java.sql.SQLSyntaxErrorException: Key column 'c0' doesn't exist in table + errors.add("doesn't exist in table"); + // java.sql.SQLSyntaxErrorException: A SPATIAL index may only contain a geometrical type column + errors.add("A SPATIAL index may only contain a geometrical type column"); + } + + private void appendIndexType() { + // The index_type clause cannot be used for FULLTEXT INDEX or SPATIAL INDEX specifications. + if (sb.toString().contains("FULLTEXT") || sb.toString().contains("SPATIAL")) { + return; + } + if (Randomly.getBoolean()) { + return; + } + sb.append(" USING "); + sb.append(Randomly.fromOptions("BTREE", "HASH")); + } + + private void appendKeyPart() { + sb.append("("); + sb.append(table.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" ASC", " DESC")); + } + sb.append(")"); + } + + private void appendIndexOption() { + if (Randomly.getBoolean()) { + return; + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions("KEY_BLOCK_SIZE ", "KEY_BLOCK_SIZE = ")); + sb.append(r.getInteger(1, Math.max(1, Randomly.smallNumber()))); + sb.append(" "); + } + if (Randomly.getBoolean()) { + StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; + sb.append(String.format("COMMENT '%s' ", strategy.getString(r))); + } + } + + private void appendAlgoOrLockOption() { + if (Randomly.getBoolean()) { + return; + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); + sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); + } else { + sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); + sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java deleted file mode 100644 index 9a86cf7e4..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java +++ /dev/null @@ -1,75 +0,0 @@ -package sqlancer.stonedb.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; - -public class StoneDBIndexGenerator { - private final StoneDBGlobalState globalState; - // which table to add index - StoneDBTable table; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBIndexGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - table = globalState.getSchema().getRandomTable(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBIndexGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("CREATE "); - sb.append(Randomly.fromOptions("UNIQUE", "FULLTEXT", "SPATIAL")); - sb.append(" INDEX"); - sb.append(globalState.getSchema().getFreeIndexName()); - appendIndexType(); - sb.append(" ON "); - sb.append(table.getName()); - appendKeyPart(); - appendIndexOption(); - appendAlgoOrLockOption(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void appendIndexType() { - if (Randomly.getBoolean()) { - return; - } - sb.append(" USING "); - sb.append(Randomly.fromOptions("BTREE", "HASH")); - } - - private void appendKeyPart() { - sb.append("("); - sb.append(table.getRandomColumn().getName()); - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions("ASC", "DESC")); - } - sb.append(")"); - } - - private void appendIndexOption() { - if (Randomly.getBoolean()) { - return; - } - sb.append(Randomly.fromOptions(" VISIBLE", " INVISIBLE")); - } - - private void appendAlgoOrLockOption() { - if (Randomly.getBoolean()) { - return; - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); - sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); - } else { - sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); - sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index 55ebd0715..f25174948 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -4,6 +4,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; @@ -38,7 +39,7 @@ private SQLQueryAdapter getQuery() { String columnName = table.getFreeColumnName(); sb.append(columnName); sb.append(" "); - sb.append(StoneDBCompositeDataType.getRandomWithoutNull()); + sb.append(StoneDBCompositeDataType.getRandomWithoutNull().getPrimitiveDataType().toString()); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); @@ -54,9 +55,11 @@ private SQLQueryAdapter getQuery() { break; case ALTER_COLUMN: sb.append(Randomly.fromOptions("ALTER COLUMN ", "ALTER ")); - sb.append(table.getRandomColumn().getName()); + StoneDBColumn randomColumn = table.getRandomColumn(); + sb.append(randomColumn.getName()); if (Randomly.getBoolean()) { - sb.append(" SET DEFAULT ").append(generator.generateExpression()); + sb.append(" SET DEFAULT ").append(generator + .generateConstant(randomColumn.getType().getPrimitiveDataType(), Randomly.getBoolean())); } else { sb.append(" DROP DEFAULT"); } @@ -70,7 +73,7 @@ private SQLQueryAdapter getQuery() { String newColumnName = table.getFreeColumnName(); sb.append(oldColumnName).append(" ").append(newColumnName); sb.append(" "); - sb.append(StoneDBCompositeDataType.getRandomWithoutNull()); + sb.append(StoneDBCompositeDataType.getRandomWithoutNull().getPrimitiveDataType().toString()); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); @@ -81,7 +84,7 @@ private SQLQueryAdapter getQuery() { } break; case RENAME_COLUMN: - sb.append("RENAME COLUMN"); + sb.append("RENAME COLUMN "); sb.append(table.getRandomColumn().getName()); sb.append(" TO "); sb.append(table.getFreeColumnName()); @@ -89,6 +92,12 @@ private SQLQueryAdapter getQuery() { default: throw new AssertionError(action); } + addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors, true); } + + private void addExpectedErrors() { + // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead + errors.add("You can't delete all columns with ALTER TABLE; use DROP TABLE instead"); + } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java index 134e0edfe..622c86c3c 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java @@ -1,11 +1,15 @@ package sqlancer.stonedb.gen; +import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.PRIMARY_KEY; +import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.UNIQUE; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import sqlancer.Randomly; +import sqlancer.Randomly.StringGenerationStrategy; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -22,6 +26,7 @@ public class StoneDBTableCreateGenerator { private final boolean allowPrimaryKey; private boolean setPrimaryKey; private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); private final Randomly r; public StoneDBTableCreateGenerator(StoneDBGlobalState globalState, String tableName) { @@ -36,24 +41,36 @@ public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String ta } public SQLQueryAdapter getQuery() { - ExpectedErrors errors = new ExpectedErrors(); - sb.append(Randomly.fromOptions("CREATE TABLE", "CREATE TEMPORARY TABLE")); + sb.append(Randomly.fromOptions("CREATE TABLE ", "CREATE TEMPORARY TABLE ")); if (Randomly.getBoolean()) { - sb.append(" IF NOT EXISTS "); + sb.append("IF NOT EXISTS "); } sb.append(tableName); // ues link statement if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { sb.append(" LIKE "); sb.append(schema.getRandomTable().getName()); - return new SQLQueryAdapter(sb.toString(), true); } else { appendColumns(); sb.append(" "); appendTableOptions(); - addCommonErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true); } + addExpectedErrors(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void addExpectedErrors() { + // java.sql.SQLSyntaxErrorException: BLOB/TEXT column 'c0' used in key specification without a key length + errors.add("used in key specification without a key length"); + // java.sql.SQLException: Tianmu engine does not support unique index. + errors.add("Tianmu engine does not support unique index"); + // java.sql.SQLException: BLOB column 'c0' can't be used in key specification with the used table type + errors.add("can't be used in key specification with the used table type"); + // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes + errors.add("Specified key was too long; max key length is 3072 bytes"); + // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT + // instead + errors.add("Column length too big for column"); } private enum TableOptions { @@ -144,18 +161,6 @@ private void appendTableOptions() { } } - private void addCommonErrors(ExpectedErrors list) { - list.add("The storage engine for the table doesn't support"); - list.add("doesn't have this option"); - list.add("must include all columns"); - list.add("not allowed type for this type of partitioning"); - list.add("doesn't support BLOB/TEXT columns"); - list.add("A BLOB field is not allowed in partition function"); - list.add("Too many keys specified; max 1 keys allowed"); - list.add("The total length of the partitioning fields is too large"); - list.add("Got error -1 - 'Unknown error -1' from storage engine"); - } - private void appendColumns() { sb.append("("); for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { @@ -178,11 +183,10 @@ private void appendColumnDefinition() { sb.append(" "); StoneDBDataType randomType = StoneDBDataType.getRandomWithoutNull(); appendType(randomType); - sb.append(" "); appendColumnOption(randomType); } - private enum ColumnOptions { + protected enum ColumnOptions { NULL_OR_NOT_NULL, UNIQUE, COMMENT, COLUMN_FORMAT, STORAGE, PRIMARY_KEY } @@ -195,10 +199,14 @@ private void appendColumnOption(StoneDBDataType type) { // if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { // tableHasNullableColumn = true; // } + // only use one key, unique key or primary key, but not both + if (columnOptions.contains(PRIMARY_KEY) && columnOptions.contains(UNIQUE)) { + columnOptions.remove(Randomly.fromOptions(PRIMARY_KEY, UNIQUE)); + } if (isTextType) { // TODO: restriction due to the limited key length - columnOptions.remove(ColumnOptions.PRIMARY_KEY); - columnOptions.remove(ColumnOptions.UNIQUE); + columnOptions.remove(PRIMARY_KEY); + columnOptions.remove(UNIQUE); } for (ColumnOptions o : columnOptions) { sb.append(" "); @@ -222,8 +230,8 @@ private void appendColumnOption(StoneDBDataType type) { } break; case COMMENT: - // TODO: generate randomly - sb.append(String.format("COMMENT '%s' ", "asdf")); + StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; + sb.append(String.format("COMMENT '%s' ", strategy.getString(r))); break; case COLUMN_FORMAT: sb.append("COLUMN_FORMAT "); @@ -297,12 +305,10 @@ private void appendType(StoneDBDataType randomType) { sb.append("TIMESTAMP"); break; case CHAR: - sb.append("CHAR"); + sb.append("CHAR").append(Randomly.fromOptions("", "(" + r.getInteger(0, 255) + ")")); break; case VARCHAR: - sb.append("VARCHAR("); - sb.append(r.getInteger(0, 65535)); - sb.append(")"); + sb.append("VARCHAR").append("(").append(r.getInteger(0, 65535)).append(")"); break; case TINYTEXT: sb.append("TINYTEXT"); @@ -320,7 +326,7 @@ private void appendType(StoneDBDataType randomType) { sb.append("BINARY"); break; case VARBINARY: - sb.append("VARBINARY"); + sb.append("VARBINARY").append("(").append(r.getInteger(0, 65535)).append(")"); break; case TINYBLOB: sb.append("TINYBLOB"); diff --git a/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java similarity index 77% rename from src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java rename to src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index d52aed526..8e6c89dee 100644 --- a/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -10,17 +10,19 @@ import sqlancer.stonedb.StoneDBSchema.StoneDBTable; import sqlancer.stonedb.StoneDBToStringVisitor; -public final class StoneDBDeleteGenerator { +public final class StoneDBTableDeleteGenerator { private final StoneDBGlobalState globalState; private final StringBuilder sb = new StringBuilder(); ExpectedErrors errors = new ExpectedErrors(); + Randomly r; - private StoneDBDeleteGenerator(StoneDBGlobalState globalState) { + private StoneDBTableDeleteGenerator(StoneDBGlobalState globalState) { this.globalState = globalState; + r = globalState.getRandomly(); } public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBDeleteGenerator(globalState).getQuery(); + return new StoneDBTableDeleteGenerator(globalState).getQuery(); } public SQLQueryAdapter getQuery() { @@ -37,10 +39,6 @@ public SQLQueryAdapter getQuery() { } sb.append(" FROM "); sb.append(randomTable.getName()); - if (Randomly.getBoolean()) { - sb.append(" AS "); - sb.append(globalState.getSchema().getRandomTable().getName()); - } if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) @@ -48,12 +46,13 @@ public SQLQueryAdapter getQuery() { } if (Randomly.getBoolean()) { sb.append(" ORDER BY "); - sb.append(Randomly.fromOptions( - randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))); + sb.append(String.join(", ", Randomly.fromOptions( + randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))) + .replace('[', '(').replace(']', ')')); } if (Randomly.getBoolean()) { sb.append(" LIMIT "); - sb.append(new Randomly().getInteger()); + sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); } return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java similarity index 73% rename from src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java rename to src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index 05a2a24e9..7a0e3847f 100644 --- a/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -13,23 +13,22 @@ import sqlancer.stonedb.StoneDBSchema.StoneDBTable; import sqlancer.stonedb.StoneDBToStringVisitor; -public class StoneDBInsertGenerator extends AbstractInsertGenerator { +public class StoneDBTableInsertGenerator extends AbstractInsertGenerator { private final StoneDBGlobalState globalState; // which table to insert into private final StoneDBTable table; // which subset columns of the table to add values private final List columns; - private final StringBuilder sb = new StringBuilder(); ExpectedErrors errors = new ExpectedErrors(); - public StoneDBInsertGenerator(StoneDBGlobalState globalState) { + public StoneDBTableInsertGenerator(StoneDBGlobalState globalState) { this.globalState = globalState; table = globalState.getSchema().getRandomTable(); columns = table.getRandomNonEmptyColumnSubset(); } public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBInsertGenerator(globalState).getQuery(); + return new StoneDBTableInsertGenerator(globalState).getQuery(); } private SQLQueryAdapter getQuery() { @@ -45,9 +44,21 @@ private SQLQueryAdapter getQuery() { sb.append(table.getName()); appendPartition(); appendColumnsAndValues(columns); + addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } + private void addExpectedErrors() { + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row + errors.add("Data truncation: Out of range value for column "); + // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 'field list' + errors.add("Unknown column "); + // java.sql.SQLException: Insert duplicate key on row: 4, pk: 138609795916627968 + errors.add("Insert duplicate key on row: "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: + errors.add("Data truncation: Incorrect datetime value: "); + } + private void appendPartition() { } @@ -60,6 +71,7 @@ private void appendColumnsAndValues(List columns) { appendValues(); } + // append nrRows rows private void appendValues() { int nrRows; if (Randomly.getBoolean()) { @@ -72,6 +84,7 @@ private void appendValues() { } } + // append all columns of one row private void appendOneValue(int nrRow) { if (nrRow != 0) { sb.append(", "); @@ -81,12 +94,12 @@ private void appendOneValue(int nrRow) { if (c != 0) { sb.append(", "); } - sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState).generateConstant())); - + insertValue(columns.get(c)); } sb.append(")"); } + // append one column of one row @Override protected void insertValue(StoneDBColumn column) { if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 74ed75a91..1d3dac312 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -21,7 +21,6 @@ import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; import sqlancer.stonedb.StoneDBSchema.StoneDBTables; @@ -69,7 +68,7 @@ private int getUnoptimizedQueryCount(List> tableList, Node asText = new NewPostfixTextNode<>(new StoneDBCastOperation( new NewPostfixTextNode<>(randomWhereCondition, " IS NOT NULL AND " + StoneDBToStringVisitor.asString(randomWhereCondition)), - new StoneDBCompositeDataType(StoneDBDataType.INT, 8)), "as count"); + StoneDBDataType.INT), "as count"); select.setFetchColumns(List.of(asText)); select.setFromList(tableList); select.setJoinList(joins); From e3fbde0df29b386f100c371b1fcf11ff7005c7c6 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 10 Jul 2023 15:52:20 +0800 Subject: [PATCH 0799/1284] feat: add expected errors (#852) --- src/sqlancer/stonedb/StoneDBBugs.java | 3 +++ src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java index 7ed539763..01040c429 100644 --- a/src/sqlancer/stonedb/StoneDBBugs.java +++ b/src/sqlancer/stonedb/StoneDBBugs.java @@ -1,6 +1,9 @@ package sqlancer.stonedb; public final class StoneDBBugs { + // https://github.com/stoneatom/stonedb/issues/1933 + public static boolean bug1933 = true; + private StoneDBBugs() { } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index 8e6c89dee..3ff809740 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -6,6 +6,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.stonedb.StoneDBBugs; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; import sqlancer.stonedb.StoneDBToStringVisitor; @@ -54,6 +55,13 @@ public SQLQueryAdapter getQuery() { sb.append(" LIMIT "); sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); } + addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } + + private void addExpectedErrors() { + if (StoneDBBugs.bug1933) { + errors.add("assert failed on i < m_idx.size() at tianmu_attr.h:387, msg: [bad dpn index 0/0]"); + } + } } From 517e39474b609d9c582c9f8266e2a3f8568f3070 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Wed, 12 Jul 2023 22:13:22 +0800 Subject: [PATCH 0800/1284] fix & refactor: fix StoneDB implementation bugs and refactor code, Part 2 (#854) * fix: add space when creating table * fix: add expected errors and key length specification * feat: generate comments randomly * fix: add expected errors and key restriction * feat: add support for more map actions * fix: add space between INDEX and indexName * fix: add space and {} * fix: remove pre-added errors before testing * refactor: rename delete useless as when deleting from table * fix: add missing space * fix: add missing ( * fix: add missing length for varbinary type * fix: string generation strategy in comments * fix: add provider * fix: use global randomly * fix: insert one row * feat: add expected errors * feat: no length * feat: no length * style: format * fix: Tianmu engine does not support fulltext index. * fix: fix alter table and getwithoutnull * fix: add expected errors * feat: support StringVisitor of StoneSelect * fix: delete useless import * fix: syntax error in index create * style: run formate * fix: syntax errors * fix: delete AS in DELETE * feat: add expectedErrors for all generator * refactor: change cast operation to lambda and fix dataType toString error * feat: add expected errors * refactor: remove redundant identifier * refactor: remove unused expected errors * refactor: use parent StringBuilder * refactor: add more expected errors * feat: add some options according to docs * refactor: change action perform times * refactor: rename and change action perform times * fix: delete unexcepted errors * feat: add one expected error * feat: add another option when assigning assignment list * refactor: refactor append type, reused function * refactor: refactor append type argus * fix: fix null pointer exception * refactor: delete useless code * fix: fix cast operation bug in StoneDB * feat: add some options according to docs * feat: add one expected error * feat: add another option when assigning assignment list * fix: fix null pointer exception * fix: fix cast operation bug in StoneDB * fix: fix cast operation bug in StoneDB * fix: Tianmu engine does not support fulltext index. * feat: add expectedErrors for all generator * refactor: remove unused expected errors * refactor: delete useless code * feat: add some options according to docs * feat: add one expected error * feat: add another option when assigning assignment list * fix: fix null pointer exception * fix: fix cast operation bug in StoneDB * fix: fix cast operation bug in StoneDB * fix: delete rename column * feat: add expected errors * refactor: reuse code * refactor: refactor append type argus * refactor: reuse code, rename, fix * fix: syntax error * fix: change / alter column error * fix: change / alter column error * fix: change / alter column error * fix: duplicate data type when add column in alter table * fix: duplicate data type when add column in alter table * fix: no space error * fix: fix as count in NoREC Oracle * fix: fix as count in NoREC Oracle * fix: fix as count in NoREC Oracle * fix: fix bitwise operation * fix: fix binary operation * fix: fix in operation * feat: add expected errors * feat: add expected errors * feat: fix errors in fetch size * feat: add expected errors * fix: regex match string * feat: add expected errors * git: merge main * feat: disable delete order by * feat: disable delete order by * feat: disable delete order by --- src/sqlancer/stonedb/StoneDBSchema.java | 81 +++++++++++ .../stonedb/StoneDBToStringVisitor.java | 12 ++ .../gen/StoneDBExpressionGenerator.java | 53 ++++++-- .../gen/StoneDBIndexCreateGenerator.java | 20 ++- .../gen/StoneDBTableAlterGenerator.java | 85 ++++++++---- .../gen/StoneDBTableCreateGenerator.java | 127 +++--------------- .../gen/StoneDBTableDeleteGenerator.java | 9 +- .../gen/StoneDBTableInsertGenerator.java | 5 + .../gen/StoneDBTableUpdateGenerator.java | 7 +- .../stonedb/oracle/StoneDBNoRECOracle.java | 20 ++- 10 files changed, 245 insertions(+), 174 deletions(-) diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java index c24b67793..2150cbba1 100644 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -92,6 +92,87 @@ public static Node getRandomValue(StoneDBDataType dataType) { } return null; } + + public static String getTypeAndValue(StoneDBDataType dataType) { + StringBuilder sb = new StringBuilder(); + switch (dataType) { + case TINYINT: + return "TINYINT"; + case SMALLINT: + return "SMALLINT"; + case MEDIUMINT: + return "MEDIUMINT"; + case INT: + return "INT"; + case BIGINT: + return "BIGINT"; + case FLOAT: + sb.append("FLOAT"); + optionallyAddPrecisionAndScale(sb); + return sb.toString(); + case DOUBLE: + sb.append("DOUBLE"); + optionallyAddPrecisionAndScale(sb); + return sb.toString(); + case DECIMAL: + return "DECIMAL"; // The default value is P(10,0); + case YEAR: + return "YEAR"; + case TIME: + return "TIME"; + case DATE: + return "DATE"; + case DATETIME: + return "DATETIME"; + case TIMESTAMP: + return "TIMESTAMP"; + case CHAR: + sb.append("CHAR").append(Randomly.fromOptions("", "(" + new Randomly().getInteger(0, 255) + ")")); + return sb.toString(); + case VARCHAR: + sb.append("VARCHAR").append("(").append(new Randomly().getInteger(0, 65535)).append(")"); + return sb.toString(); + case TINYTEXT: + return "TINYTEXT"; + case TEXT: + return "TEXT"; + case MEDIUMTEXT: + return "MEDIUMTEXT"; + case LONGTEXT: + return "LONGTEXT"; + case BINARY: + return "BINARY"; + case VARBINARY: + sb.append("VARBINARY").append("(").append(new Randomly().getInteger(0, 65535)).append(")"); + return sb.toString(); + case TINYBLOB: + return "TINYBLOB"; + case BLOB: + return "BLOB"; + case MEDIUMBLOB: + return "MEDIUMBLOB"; + case LONGBLOB: + return "LONGBLOB"; + default: + throw new AssertionError(); + } + } + + private static void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean()) { + sb.append("("); + // The maximum number of digits (M) for DECIMAL is 65 + long m = Randomly.getNotCachedInteger(1, 65); + sb.append(m); + sb.append(", "); + // The maximum number of supported decimals (D) is 30 + long nCandidate = Randomly.getNotCachedInteger(1, 30); + // For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'c0'). + long n = Math.min(nCandidate, m); + sb.append(n); + sb.append(")"); + } + } } public static class StoneDBTable diff --git a/src/sqlancer/stonedb/StoneDBToStringVisitor.java b/src/sqlancer/stonedb/StoneDBToStringVisitor.java index 9d7711873..45b050b50 100644 --- a/src/sqlancer/stonedb/StoneDBToStringVisitor.java +++ b/src/sqlancer/stonedb/StoneDBToStringVisitor.java @@ -2,10 +2,12 @@ import sqlancer.common.ast.newast.NewToStringVisitor; import sqlancer.common.ast.newast.Node; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; import sqlancer.stonedb.ast.StoneDBJoin; import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; public class StoneDBToStringVisitor extends NewToStringVisitor { @Override @@ -16,6 +18,8 @@ public void visitSpecific(Node expr) { visit((StoneDBSelect) expr); } else if (expr instanceof StoneDBJoin) { visit((StoneDBJoin) expr); + } else if (expr instanceof StoneDBCastOperation) { + visit((StoneDBCastOperation) expr); } else { throw new AssertionError(expr.getClass()); } @@ -81,6 +85,14 @@ private void visit(StoneDBJoin join) { } } + private void visit(StoneDBCastOperation cast) { + sb.append("CAST("); + visit(cast.getExpr()); + sb.append(" AS "); + sb.append(cast.getType() == StoneDBDataType.INT ? "UNSIGNED" : cast.getType().toString()); + sb.append(") "); + } + public static String asString(Node expr) { StoneDBToStringVisitor visitor = new StoneDBToStringVisitor(); visitor.visit(expr); diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 385154815..87876c5cc 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -2,11 +2,14 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.NewCaseOperatorNode; @@ -32,25 +35,35 @@ public StoneDBExpressionGenerator(StoneDBGlobalState globalState) { private enum Expression { UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, BINARY_BITWISE, BETWEEN, IN, - CASE + NOT_IN, CASE } - public static class StoneDBCastOperation extends NewUnaryPostfixOperatorNode { + public static class StoneDBCastOperation implements Node { + Node expr; + StoneDBDataType type; public StoneDBCastOperation(Node expr, StoneDBDataType type) { - super(expr, () -> "::" + type.toString()); + this.expr = expr; + this.type = type; } + public Node getExpr() { + return expr; + } + + public StoneDBDataType getType() { + return type; + } } @Override public Node negatePredicate(Node predicate) { - return null; + return new NewUnaryPrefixOperatorNode<>(predicate, StoneDBUnaryPrefixOperator.NOT); } @Override public Node isNull(Node expr) { - return null; + return new NewUnaryPostfixOperatorNode<>(expr, StoneDBUnaryPostfixOperator.IS_NULL); } @Override @@ -111,6 +124,12 @@ protected Node generateExpression(int depth) { case BINARY_COMPARISON: op = StoneDBBinaryComparisonOperator.getRandom(); return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case IN: + return new NewInOperatorNode<>(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), false); + case NOT_IN: + return new NewInOperatorNode<>(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), true); case BINARY_LOGICAL: op = StoneDBBinaryLogicalOperator.getRandom(); return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); @@ -123,9 +142,6 @@ protected Node generateExpression(int depth) { case BETWEEN: return new NewBetweenOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); - case IN: - return new NewInOperatorNode<>(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; return new NewCaseOperatorNode<>(generateExpression(depth + 1), generateExpressions(nr, depth + 1), @@ -137,7 +153,17 @@ protected Node generateExpression(int depth) { @Override protected Node generateColumn() { - return null; + StoneDBColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode<>(column); + } + + protected List> generateColumns() { + int size = globalState.getRandomly().getInteger(1, columns.size()); + Set> set = new HashSet<>(); + while (set.size() < size) { + set.add(generateColumn()); + } + return new ArrayList<>(set); } public enum StoneDBAggregateFunction { @@ -160,7 +186,7 @@ public int getNrArgs() { public enum StoneDBUnaryPrefixOperator implements Operator { - NOT("NOT"), PLUS("+"), MINUS("-"); + NOT("NOT"), PLUS("+"), MINUS("-"), INVERSION("!"); private final String textRepr; @@ -203,8 +229,7 @@ public static StoneDBUnaryPostfixOperator getRandom() { */ public enum StoneDBBinaryComparisonOperator implements Operator { EQUAL("="), GREATER(">"), LESS("<"), GREATER_EQUAL(">="), LESS_EQUAL("<="), - NOT_EQUALS(Randomly.fromList(Arrays.asList("!=", "<>"))), NULL_SAFE_EQUAL("<=>"), IN("IN"), NOT_IN("NOT_IN"), - LIKE("LIKE"), IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + NOT_EQUALS(Randomly.fromList(Arrays.asList("!=", "<>"))), NULL_SAFE_EQUAL("<=>"), LIKE("LIKE"); private final String textRepr; @@ -227,7 +252,7 @@ public String getTextRepresentation() { */ public enum StoneDBBinaryLogicalOperator implements Operator { - NOT("NOT"), AND("AND"), OR("OR"), XOR("XOR"); + AND("AND"), OR("OR"), XOR("XOR"); private final String textRepr; @@ -274,7 +299,7 @@ public String getTextRepresentation() { * Bitwise operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/bitwise-operators */ public enum StoneDBBinaryBitwiseOperator implements Operator { - AND("&"), OR("|"), XOR("^"), INVERSION("!"), LEFTSHIFT("<<"), RIGHTSHIFT(">>"); + AND("&"), OR("|"), XOR("^"), LEFTSHIFT("<<"), RIGHTSHIFT(">>"); private final String textRepr; diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java index 3b7e63422..b50f07a6a 100644 --- a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java @@ -5,6 +5,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; public class StoneDBIndexCreateGenerator { @@ -30,10 +31,12 @@ private SQLQueryAdapter getQuery() { sb.append(Randomly.fromOptions("UNIQUE", "FULLTEXT", "SPATIAL")); sb.append(" INDEX "); sb.append(globalState.getSchema().getFreeIndexName()); - appendIndexType(); + if (Randomly.getBoolean()) { + appendIndexType(); + } sb.append(" ON "); sb.append(table.getName()); - appendKeyPart(); + appendKeyParts(); appendIndexOption(); appendAlgoOrLockOption(); addExpectedErrors(); @@ -67,9 +70,20 @@ private void appendIndexType() { sb.append(Randomly.fromOptions("BTREE", "HASH")); } + private void appendKeyParts() { + int numberOfKeyParts = Randomly.fromOptions(1, 1, 1, 1, table.getColumns().size()); + for (int i = 0; i < numberOfKeyParts; i++) { + appendKeyPart(); + } + } + private void appendKeyPart() { sb.append("("); - sb.append(table.getRandomColumn().getName()); + StoneDBColumn randomColumn = table.getRandomColumn(); + sb.append(randomColumn.getName()); + if (Randomly.getBoolean()) { + sb.append(" (").append(Randomly.smallNumber()).append(")"); + } if (Randomly.getBoolean()) { sb.append(Randomly.fromOptions(" ASC", " DESC")); } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index f25174948..b1e4872aa 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -1,24 +1,29 @@ package sqlancer.stonedb.gen; +import java.util.List; +import java.util.regex.Pattern; + import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; public class StoneDBTableAlterGenerator { private final StoneDBGlobalState globalState; private final StringBuilder sb = new StringBuilder(); + private final StoneDBTable table; ExpectedErrors errors = new ExpectedErrors(); enum Action { - ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN, CHANGE_COLUMN, RENAME_COLUMN + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN, CHANGE_COLUMN } public StoneDBTableAlterGenerator(StoneDBGlobalState globalState) { this.globalState = globalState; + table = globalState.getSchema().getRandomTable(t -> !t.isView()); } public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { @@ -27,19 +32,56 @@ public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { private SQLQueryAdapter getQuery() { sb.append("ALTER TABLE "); - StoneDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) - .setColumns(table.getColumns()); sb.append(table.getName()); sb.append(" "); - Action action = Randomly.fromOptions(Action.values()); + appendAlterOptions(); + addExpectedErrors(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void addExpectedErrors() { + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Data too long for column 'c0' at row 2 + errors.addRegex(Pattern.compile("Data truncation: Data too long for column 'c\\d{1,3}' at row \\d{1,3}")); + // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes + errors.add("Specified key was too long; max key length is 3072 bytes"); + // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead + errors.add("You can't delete all columns with ALTER TABLE; use DROP TABLE instead"); + // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 't1' + errors.addRegex(Pattern.compile("Unknown column 'c\\d{1,3}' in 't\\d{1,3}'")); + // java.sql.SQLSyntaxErrorException: BLOB, TEXT, GEOMETRY or JSON column 'c0' can't have a default value + errors.addRegex(Pattern.compile("BLOB, TEXT, GEOMETRY or JSON column 'c\\d{1,3}' can't have a default value")); + // java.sql.SQLSyntaxErrorException: Column length too big for column 'c91' (max = 16383); use BLOB or TEXT + // instead + errors.addRegex(Pattern + .compile("Column length too big for column 'c\\d{1,3}' \\(max = 16383\\); use BLOB or TEXT instead")); + } + + private void appendAlterOptions() { + List actions; + if (Randomly.getBooleanWithSmallProbability()) { + actions = Randomly.subset(Action.values()); + } else { + actions = List.of(Randomly.fromOptions(Action.values())); + } + for (Action action : actions) { + appendAlterOption(action); + sb.append(" "); + } + } + + private void appendAlterOption(Action action) { + StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) + .setColumns(table.getColumns()); switch (action) { case ADD_COLUMN: sb.append("ADD COLUMN "); String columnName = table.getFreeColumnName(); - sb.append(columnName); - sb.append(" "); - sb.append(StoneDBCompositeDataType.getRandomWithoutNull().getPrimitiveDataType().toString()); + sb.append(" ").append(columnName).append(" "); + sb.append(StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull())); + // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT + // instead + errors.addRegex(Pattern + .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); @@ -63,17 +105,17 @@ private SQLQueryAdapter getQuery() { } else { sb.append(" DROP DEFAULT"); } - if (Randomly.getBoolean()) { - sb.append(" SET ").append(Randomly.fromOptions("VISIBLE", "INVISIBLE")); - } break; case CHANGE_COLUMN: sb.append(Randomly.fromOptions("CHANGE COLUMN ", "CHANGE ")); String oldColumnName = table.getRandomColumn().getName(); String newColumnName = table.getFreeColumnName(); - sb.append(oldColumnName).append(" ").append(newColumnName); - sb.append(" "); - sb.append(StoneDBCompositeDataType.getRandomWithoutNull().getPrimitiveDataType().toString()); + sb.append(oldColumnName).append(" ").append(newColumnName).append(" "); + sb.append(StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull())); + // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT + // instead + errors.addRegex(Pattern + .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); @@ -83,21 +125,8 @@ private SQLQueryAdapter getQuery() { } } break; - case RENAME_COLUMN: - sb.append("RENAME COLUMN "); - sb.append(table.getRandomColumn().getName()); - sb.append(" TO "); - sb.append(table.getFreeColumnName()); - break; default: throw new AssertionError(action); } - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void addExpectedErrors() { - // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead - errors.add("You can't delete all columns with ALTER TABLE; use DROP TABLE instead"); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java index 622c86c3c..3b87e99d5 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java @@ -1,12 +1,13 @@ package sqlancer.stonedb.gen; import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.PRIMARY_KEY; -import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.UNIQUE; +import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.UNIQUE_KEY; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.regex.Pattern; import sqlancer.Randomly; import sqlancer.Randomly.StringGenerationStrategy; @@ -41,7 +42,7 @@ public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String ta } public SQLQueryAdapter getQuery() { - sb.append(Randomly.fromOptions("CREATE TABLE ", "CREATE TEMPORARY TABLE ")); + sb.append(Randomly.fromOptions("CREATE TABLE "/* , "CREATE TEMPORARY TABLE " */)); if (Randomly.getBoolean()) { sb.append("IF NOT EXISTS "); } @@ -52,8 +53,10 @@ public SQLQueryAdapter getQuery() { sb.append(schema.getRandomTable().getName()); } else { appendColumns(); - sb.append(" "); - appendTableOptions(); + if (Randomly.getBoolean()) { + sb.append(" "); + appendTableOptions(); + } } addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors, true); @@ -71,6 +74,8 @@ private void addExpectedErrors() { // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT // instead errors.add("Column length too big for column"); + // BLOB/TEXT column 'c0' used in key specification without a key length + errors.addRegex(Pattern.compile("BLOB/TEXT column 'c.*' used in key specification without a key length")); } private enum TableOptions { @@ -187,7 +192,7 @@ private void appendColumnDefinition() { } protected enum ColumnOptions { - NULL_OR_NOT_NULL, UNIQUE, COMMENT, COLUMN_FORMAT, STORAGE, PRIMARY_KEY + NULL_OR_NOT_NULL, PRIMARY_KEY, UNIQUE_KEY, COMMENT, COLUMN_FORMAT, STORAGE } private void appendColumnOption(StoneDBDataType type) { @@ -200,13 +205,13 @@ private void appendColumnOption(StoneDBDataType type) { // tableHasNullableColumn = true; // } // only use one key, unique key or primary key, but not both - if (columnOptions.contains(PRIMARY_KEY) && columnOptions.contains(UNIQUE)) { - columnOptions.remove(Randomly.fromOptions(PRIMARY_KEY, UNIQUE)); + if (columnOptions.contains(PRIMARY_KEY) && columnOptions.contains(UNIQUE_KEY)) { + columnOptions.remove(Randomly.fromOptions(PRIMARY_KEY, UNIQUE_KEY)); } if (isTextType) { // TODO: restriction due to the limited key length columnOptions.remove(PRIMARY_KEY); - columnOptions.remove(UNIQUE); + columnOptions.remove(UNIQUE_KEY); } for (ColumnOptions o : columnOptions) { sb.append(" "); @@ -223,7 +228,7 @@ private void appendColumnOption(StoneDBDataType type) { sb.append("NOT NULL"); } break; - case UNIQUE: + case UNIQUE_KEY: sb.append("UNIQUE"); if (Randomly.getBoolean()) { sb.append(" KEY"); @@ -256,108 +261,6 @@ private void appendColumnOption(StoneDBDataType type) { } private void appendType(StoneDBDataType randomType) { - switch (randomType) { - case TINYINT: - sb.append("TINYINT"); - // sb.append(r.getInteger(-128, 127)); - break; - case SMALLINT: - sb.append("SMALLINT"); - // sb.append(r.getInteger(-32768, 32767)); - break; - case MEDIUMINT: - sb.append("MEDIUMINT"); - // sb.append(r.getInteger(-8388608, 8388607)); - break; - case INT: - sb.append("INT"); - // sb.append(r.getInteger(-2147483647, 2147483647)); - break; - case BIGINT: - sb.append("BIGINT"); - // sb.append(r.getBigInteger(new BigInteger("-9223372036854775806"), new - // BigInteger("9223372036854775807"))); - break; - case FLOAT: - sb.append("FLOAT"); - optionallyAddPrecisionAndScale(sb); - break; - case DOUBLE: - sb.append("DOUBLE"); - optionallyAddPrecisionAndScale(sb); - break; - case DECIMAL: - sb.append("DECIMAL"); // The default value is P(10,0); - break; - case YEAR: - sb.append("YEAR"); - break; - case TIME: - sb.append("TIME"); - break; - case DATE: - sb.append("DATE"); - break; - case DATETIME: - sb.append("DATETIME"); - break; - case TIMESTAMP: - sb.append("TIMESTAMP"); - break; - case CHAR: - sb.append("CHAR").append(Randomly.fromOptions("", "(" + r.getInteger(0, 255) + ")")); - break; - case VARCHAR: - sb.append("VARCHAR").append("(").append(r.getInteger(0, 65535)).append(")"); - break; - case TINYTEXT: - sb.append("TINYTEXT"); - break; - case TEXT: - sb.append("TEXT"); - break; - case MEDIUMTEXT: - sb.append("MEDIUMTEXT"); - break; - case LONGTEXT: - sb.append("LONGTEXT"); - break; - case BINARY: - sb.append("BINARY"); - break; - case VARBINARY: - sb.append("VARBINARY").append("(").append(r.getInteger(0, 65535)).append(")"); - break; - case TINYBLOB: - sb.append("TINYBLOB"); - break; - case BLOB: - sb.append("BLOB"); - break; - case MEDIUMBLOB: - sb.append("MEDIUMBLOB"); - break; - case LONGBLOB: - sb.append("LONGBLOB"); - break; - default: - throw new AssertionError(); - } - } - - public static void optionallyAddPrecisionAndScale(StringBuilder sb) { - if (Randomly.getBoolean()) { - sb.append("("); - // The maximum number of digits (M) for DECIMAL is 65 - long m = Randomly.getNotCachedInteger(1, 65); - sb.append(m); - sb.append(", "); - // The maximum number of supported decimals (D) is 30 - long nCandidate = Randomly.getNotCachedInteger(1, 30); - // For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'c0'). - long n = Math.min(nCandidate, m); - sb.append(n); - sb.append(")"); - } + sb.append(StoneDBDataType.getTypeAndValue(randomType)); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index 3ff809740..ab0828d30 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -45,7 +45,7 @@ public SQLQueryAdapter getQuery() { sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) .setColumns(randomTable.getColumns()).generateExpression())); } - if (Randomly.getBoolean()) { + if (!StoneDBBugs.bug1933 && Randomly.getBoolean()) { sb.append(" ORDER BY "); sb.append(String.join(", ", Randomly.fromOptions( randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))) @@ -55,13 +55,6 @@ public SQLQueryAdapter getQuery() { sb.append(" LIMIT "); sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); } - addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } - - private void addExpectedErrors() { - if (StoneDBBugs.bug1933) { - errors.add("assert failed on i < m_idx.size() at tianmu_attr.h:387, msg: [bad dpn index 0/0]"); - } - } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index 7a0e3847f..2fdb7b6fa 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -1,6 +1,7 @@ package sqlancer.stonedb.gen; import java.util.List; +import java.util.regex.Pattern; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -57,6 +58,10 @@ private void addExpectedErrors() { errors.add("Insert duplicate key on row: "); // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: errors.add("Data truncation: Incorrect datetime value: "); + // java.sql.SQLException: Field 'c0' doesn't have a default value + errors.add("doesn't have a default value"); + // java.sql.SQLException: Data truncated for column 'c0' at row 1 + errors.addRegex(Pattern.compile("Data truncated for column 'c.*' at row .*")); } private void appendPartition() { diff --git a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java index bff098a64..4812a6ee3 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java @@ -49,8 +49,11 @@ public void appendAssignmentList() { } sb.append(table.getColumns().get(i)); sb.append(" = "); - sb.append(StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType - .getRandomValue(table.getColumns().get(i).getType().getPrimitiveDataType()))); + sb.append( + Randomly.fromOptions( + StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType + .getRandomValue(table.getColumns().get(i).getType().getPrimitiveDataType())), + "DEFAULT")); } } diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 1d3dac312..2625bbb7c 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -11,6 +11,7 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.NewPostfixTextNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; @@ -29,6 +30,7 @@ import sqlancer.stonedb.ast.StoneDBJoin; import sqlancer.stonedb.ast.StoneDBSelect; import sqlancer.stonedb.gen.StoneDBExpressionGenerator; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBBinaryLogicalOperator; import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; public class StoneDBNoRECOracle extends NoRECBase implements TestOracle { @@ -65,14 +67,16 @@ public void check() throws Exception { private int getUnoptimizedQueryCount(List> tableList, Node randomWhereCondition, List> joins) throws SQLException { StoneDBSelect select = new StoneDBSelect(); - Node asText = new NewPostfixTextNode<>(new StoneDBCastOperation( - new NewPostfixTextNode<>(randomWhereCondition, - " IS NOT NULL AND " + StoneDBToStringVisitor.asString(randomWhereCondition)), - StoneDBDataType.INT), "as count"); + Node asText = new NewPostfixTextNode<>( + new StoneDBCastOperation( + new NewBinaryOperatorNode<>(new NewPostfixTextNode<>(randomWhereCondition, " IS NOT NULL "), + randomWhereCondition, StoneDBBinaryLogicalOperator.AND), + StoneDBDataType.INT), + " as count"); select.setFetchColumns(List.of(asText)); select.setFromList(tableList); select.setJoinList(joins); - unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res"; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { @@ -104,14 +108,16 @@ private int getOptimizedQueryCount(SQLConnection con, List Date: Sun, 23 Jul 2023 23:15:40 +0800 Subject: [PATCH 0801/1284] feat: add TLP support of StoneDB (#857) * fix: Tianmu engine does not support fulltext index. * feat: add expectedErrors for all generator * refactor: remove unused expected errors * feat: add expectedErrors for all generator * refactor: remove unused expected errors * feat: init support of TLP for StoneDB * fix: mvn test errors * feat: add CI * feat: add CI * feat: add CI * fix: remove unsupported operator * fix: change string generator strategy * feat: add expected errors * fix: binary type should have a max length * fix: mvn format * fix: MySQL 5.7 does not support double, change to decimal * fix: natural join type * fix: only one alter option one time * feat: add expected errors * git: manually revert commits * git: manually revert commits * fix: use latest version and set tianmu_insert_delayed=0 * rename: rename oracle --- .github/workflows/main.yml | 41 ++++- src/sqlancer/stonedb/StoneDBErrors.java | 1 - src/sqlancer/stonedb/StoneDBOptions.java | 55 +++++- .../gen/StoneDBExpressionGenerator.java | 11 ++ .../oracle/StoneDBAggregateOracle.java | 168 ++++++++++++++++++ .../oracle/StoneDBQueryPartitioningBase.java | 71 ++++++++ ...toneDBQueryPartitioningDistinctTester.java | 42 +++++ ...StoneDBQueryPartitioningGroupByTester.java | 50 ++++++ .../StoneDBQueryPartitioningHavingTester.java | 46 +++++ .../StoneDBQueryPartitioningWhereTester.java | 42 +++++ .../stonedb/oracle/StoneDBTLPOracle.java | 16 -- ...TestStoneDB.java => TestStoneDBNoRec.java} | 2 +- test/sqlancer/dbms/TestStoneDBTLP.java | 21 +++ 13 files changed, 534 insertions(+), 32 deletions(-) create mode 100644 src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java rename test/sqlancer/dbms/{TestStoneDB.java => TestStoneDBNoRec.java} (95%) create mode 100644 test/sqlancer/dbms/TestStoneDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 990543c0d..9aaed3c70 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -375,8 +375,8 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test - stonedb: - name: DBMS Tests (StoneDB) + stonedb-norec: + name: DBMS Tests (StoneDB NoRec) runs-on: ubuntu-latest steps: @@ -385,12 +385,11 @@ jobs: fetch-depth: 0 - name: Set up StoneDB run: | - docker pull stoneatom/stonedb:v1.0.3 - docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb:v1.0.3 - docker ps - ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb:v1.0.3 --format "{{.ID}}") + docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb + ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb --format "{{.ID}}") docker exec $ContainerID bash - sleep 180s + sleep 60s + docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Set up JDK 11 uses: actions/setup-java@v1.4.4 @@ -400,7 +399,33 @@ jobs: run: mvn -B package -DskipTests=true - name: StoneDB Tests run: | - STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDB + STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBNoRec + + stonedb-tlp: + name: DBMS Tests (StoneDB TLP) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up StoneDB + run: | + docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb + ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb --format "{{.ID}}") + docker exec $ContainerID bash + sleep 60s + docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf + docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + - name: Set up JDK 11 + uses: actions/setup-java@v1.4.4 + with: + java-version: 11 + - name: Build + run: mvn -B package -DskipTests=true + - name: StoneDB Tests + run: | + STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBTLP tidb: name: DBMS Tests (TiDB) diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java index 54fd28ad8..bdbd6a447 100644 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -3,5 +3,4 @@ public final class StoneDBErrors { private StoneDBErrors() { } - } diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java index 928bcfa39..7e6a2f09b 100644 --- a/src/sqlancer/stonedb/StoneDBOptions.java +++ b/src/sqlancer/stonedb/StoneDBOptions.java @@ -1,7 +1,7 @@ package sqlancer.stonedb; import java.sql.SQLException; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import com.beust.jcommander.Parameter; @@ -9,11 +9,16 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.stonedb.StoneDBOptions.StoneDBOracleFactory; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.oracle.StoneDBAggregateOracle; import sqlancer.stonedb.oracle.StoneDBNoRECOracle; -import sqlancer.stonedb.oracle.StoneDBTLPOracle; +import sqlancer.stonedb.oracle.StoneDBQueryPartitioningDistinctTester; +import sqlancer.stonedb.oracle.StoneDBQueryPartitioningGroupByTester; +import sqlancer.stonedb.oracle.StoneDBQueryPartitioningHavingTester; +import sqlancer.stonedb.oracle.StoneDBQueryPartitioningWhereTester; @Parameters(separators = "=", commandDescription = "StoneDB (default host: " + StoneDBOptions.DEFAULT_HOST + ", default port: " + StoneDBOptions.DEFAULT_PORT + ")") @@ -22,7 +27,7 @@ public class StoneDBOptions implements DBMSSpecificOptions public static final int DEFAULT_PORT = 3306; @Parameter(names = "--oracle") - public List oracles = Arrays.asList(StoneDBOracleFactory.NOREC); + public List oracles = List.of(StoneDBOracleFactory.NOREC); public enum StoneDBOracleFactory implements OracleFactory { NOREC { @@ -31,10 +36,48 @@ public TestOracle create(StoneDBGlobalState globalState) thr return new StoneDBNoRECOracle(globalState); } }, - TLP { + QUERY_PARTITIONING { @Override - public TestOracle create(StoneDBGlobalState globalState) throws Exception { - return new StoneDBTLPOracle(globalState); + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); + oracles.add(new StoneDBQueryPartitioningWhereTester(globalState)); + oracles.add(new StoneDBQueryPartitioningHavingTester(globalState)); + oracles.add(new StoneDBAggregateOracle(globalState)); + oracles.add(new StoneDBQueryPartitioningDistinctTester(globalState)); + oracles.add(new StoneDBQueryPartitioningGroupByTester(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + }, + HAVING { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBAggregateOracle(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBQueryPartitioningDistinctTester(globalState); } } } diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 87876c5cc..54d2c4113 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -166,6 +166,7 @@ protected List> generateColumns() { return new ArrayList<>(set); } + // https://stonedb.io/docs/SQL-reference/functions/aggregate-functions/ public enum StoneDBAggregateFunction { MAX(1), MIN(1), AVG(1), COUNT(1), FIRST(1), SUM(1); @@ -317,4 +318,14 @@ public String getTextRepresentation() { } } + + public NewFunctionNode generateAggregateAndArgs( + StoneDBAggregateFunction aggregateFunction) { + return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + } + + public Node generateAggregate() { + StoneDBAggregateFunction aggrFunc = StoneDBAggregateFunction.getRandom(); + return generateAggregateAndArgs(aggrFunc); + } } diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java new file mode 100644 index 000000000..df09e8961 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -0,0 +1,168 @@ +package sqlancer.stonedb.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBAggregateFunction; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPostfixOperator; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPrefixOperator; + +public class StoneDBAggregateOracle extends StoneDBQueryPartitioningBase { + + public StoneDBAggregateOracle(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + + StoneDBAggregateFunction aggregateFunction = Randomly.fromOptions(StoneDBAggregateFunction.values()); + NewFunctionNode aggregate = gen + .generateAggregateAndArgs(aggregateFunction); + + List> fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + select.setFetchColumns(fetchColumns); + + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + + String originalQuery = StoneDBToStringVisitor.asString(select); + String originalResult = getAggregateResult(originalQuery); + + String metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + String metamorphicResult = getAggregateResult(metamorphicQuery); + + String line1 = "--" + originalQuery + ";"; + String line2 = "--" + originalResult + ";"; + String line3 = "--" + originalResult + ";"; + String line4 = "--" + metamorphicResult + ";"; + String output = String.join(System.lineSeparator(), line1, line2, line3, line4); + state.getState().getLocalState().log(output); + + if (originalResult == null && metamorphicResult != null + || originalResult != null && (!originalResult.contentEquals(metamorphicResult) + && !ComparatorHelper.isEqualDouble(originalResult, metamorphicResult))) { + throw new AssertionError("aggregate result mismatch!" + System.lineSeparator() + output); + } + } + + private String createMetamorphicUnionQuery(StoneDBSelect select, + NewFunctionNode aggregate, + List> from) { + String metamorphicQuery; + Node whereClause = gen.generateExpression(); + + Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + StoneDBUnaryPrefixOperator.NOT); + Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + StoneDBUnaryPostfixOperator.IS_NULL); + List> mappedAggregate = mapped(aggregate); + StoneDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + StoneDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + StoneDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += StoneDBToStringVisitor.asString(leftSelect) + " UNION ALL " + + StoneDBToStringVisitor.asString(middleSelect) + " UNION ALL " + + StoneDBToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as result"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List> mapped( + NewFunctionNode aggregate) { + + StoneDBCastOperation count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(List.of(aggregate)); + case AVG: + NewFunctionNode sum = new NewFunctionNode<>( + aggregate.getArgs(), StoneDBAggregateFunction.SUM); + count = new StoneDBCastOperation(new NewFunctionNode<>(aggregate.getArgs(), StoneDBAggregateFunction.COUNT), + new StoneDBCompositeDataType(StoneDBDataType.DECIMAL).getPrimitiveDataType()); + return aliasArgs(Arrays.asList(sum, count)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List> aliasArgs(List> originalAggregateArgs) { + List> args = new ArrayList<>(); + int i = 0; + for (Node expr : originalAggregateArgs) { + args.add(new NewAliasNode(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(NewFunctionNode aggregate) { + switch (aggregate.getFunc()) { + case AVG: + return "SUM(agg0)/SUM(agg1)"; + case COUNT: + return StoneDBAggregateFunction.SUM + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private StoneDBSelect getSelect(List> aggregates, List> fromList, + Node whereClause, List> joinList) { + StoneDBSelect select = new StoneDBSelect(); + select.setFetchColumns(aggregates); + select.setFromList(fromList); + select.setWhereClause(whereClause); + select.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return select; + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java new file mode 100644 index 000000000..84d9f794f --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java @@ -0,0 +1,71 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBSchema.StoneDBTables; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator; + +public class StoneDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, StoneDBGlobalState> + implements TestOracle { + + StoneDBSchema schema; + StoneDBTables targetTables; + StoneDBExpressionGenerator gen; + StoneDBSelect select; + + public StoneDBQueryPartitioningBase(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + schema = state.getSchema(); + targetTables = schema.getRandomTableNonEmptyTables(); + gen = new StoneDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + + select = new StoneDBSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableReferenceNodeList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = StoneDBJoin.getJoins(tableReferenceNodeList, state); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableReferenceNodeList)); + select.setWhereClause(null); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..cba8290b4 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java @@ -0,0 +1,42 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBQueryPartitioningDistinctTester extends StoneDBQueryPartitioningBase { + public StoneDBQueryPartitioningDistinctTester(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = StoneDBToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + if (Randomly.getBoolean()) { + select.setDistinct(false); + } + + select.setWhereClause(predicate); + String firstQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = StoneDBToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..d7674a71c --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java @@ -0,0 +1,50 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; + +public class StoneDBQueryPartitioningGroupByTester extends StoneDBQueryPartitioningBase { + public StoneDBQueryPartitioningGroupByTester(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + // common to both original and combined query string + select.setGroupByExpressions(select.getFetchColumns()); + // specific to original query string + select.setWhereClause(null); + String originalQueryString = StoneDBToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + // specific to combined query string, will produce the same result as original query string in logic + select.setWhereClause(predicate); + String firstQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = StoneDBToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + // compare the result + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } + + @Override + List> generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java new file mode 100644 index 000000000..ea611241b --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java @@ -0,0 +1,46 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBQueryPartitioningHavingTester extends StoneDBQueryPartitioningBase { + public StoneDBQueryPartitioningHavingTester(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + // common to both original and combined query string + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + // specific to original query string + select.setHavingClause(null); + String originalQueryString = StoneDBToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + // specific to combined query string, will produce the same result as original query string in logic + select.setHavingClause(predicate); + String firstQueryString = StoneDBToStringVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = StoneDBToStringVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = StoneDBToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + // compare the result + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java new file mode 100644 index 000000000..2a8f2b4d4 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java @@ -0,0 +1,42 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBQueryPartitioningWhereTester extends StoneDBQueryPartitioningBase { + public StoneDBQueryPartitioningWhereTester(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + + select.setWhereClause(null); + String originalQueryString = StoneDBToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setWhereClause(predicate); + String firstQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = StoneDBToStringVisitor.asString(select); + + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + // compare the result + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java b/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java deleted file mode 100644 index 96deda89e..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.stonedb.oracle; - -import sqlancer.common.oracle.NoRECBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; - -public class StoneDBTLPOracle extends NoRECBase implements TestOracle { - public StoneDBTLPOracle(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - - } -} diff --git a/test/sqlancer/dbms/TestStoneDB.java b/test/sqlancer/dbms/TestStoneDBNoRec.java similarity index 95% rename from test/sqlancer/dbms/TestStoneDB.java rename to test/sqlancer/dbms/TestStoneDBNoRec.java index 14de121be..4b158f372 100644 --- a/test/sqlancer/dbms/TestStoneDB.java +++ b/test/sqlancer/dbms/TestStoneDBNoRec.java @@ -7,7 +7,7 @@ import sqlancer.Main; -public class TestStoneDB { +public class TestStoneDBNoRec { String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); diff --git a/test/sqlancer/dbms/TestStoneDBTLP.java b/test/sqlancer/dbms/TestStoneDBTLP.java new file mode 100644 index 000000000..d5db5b9ab --- /dev/null +++ b/test/sqlancer/dbms/TestStoneDBTLP.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestStoneDBTLP { + + String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); + boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); + + @Test + public void testStoneDB() { + assumeTrue(stoneDBIsAvailable); + assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", + "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "QUERY_PARTITIONING")); + } + +} From 742ce6f48460f88a9dc9dafbf6213ddfbefcadc6 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:21:11 +0800 Subject: [PATCH 0802/1284] feat: add bugs found (#868) --- src/sqlancer/stonedb/StoneDBBugs.java | 36 +++++++++++++++++++ src/sqlancer/stonedb/StoneDBProvider.java | 5 +++ src/sqlancer/stonedb/ast/StoneDBConstant.java | 5 +++ .../gen/StoneDBExpressionGenerator.java | 18 ++++++++-- .../gen/StoneDBTableInsertGenerator.java | 7 ++-- .../stonedb/oracle/StoneDBNoRECOracle.java | 5 +++ 6 files changed, 72 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java index 01040c429..11a144dad 100644 --- a/src/sqlancer/stonedb/StoneDBBugs.java +++ b/src/sqlancer/stonedb/StoneDBBugs.java @@ -3,6 +3,42 @@ public final class StoneDBBugs { // https://github.com/stoneatom/stonedb/issues/1933 public static boolean bug1933 = true; + // https://github.com/stoneatom/stonedb/issues/1942 + public static boolean bug1942 = true; + // https://github.com/stoneatom/stonedb/issues/1953 + public static boolean bug1953 = true; + // https://github.com/stoneatom/stonedb/issues/1945 + public static boolean bug1945 = true; + // CREATE TABLE t0(c0 INT); + // INSERT IGNORE INTO t0(c0) VALUE (DEFAULT); + // SELECT t0.c0 FROM t0 WHERE 0.4; -- expect 1 but got 0 + public static boolean bugNotReported1 = true; + // DELETE statements will result into crash, for example + // DELETE LOW_PRIORITY FROM t0; + // DELETE QUICK IGNORE FROM t0 WHERE -1370759901; + public static boolean bugNotReported2 = true; + // CREATE TABLE t0(c0 INT) ; + // INSERT INTO t0(c0) VALUES (DEFAULT); + // SELECT t0.c0 FROM t0 WHERE (('OC')>=(((t0.c0) IS NULL))); -- expected empty set but got 1 row + public static boolean bugNotReported3 = true; + // CREATE TABLE t0(c0 CHAR PRIMARY KEY); + // INSERT IGNORE INTO t0(c0) VALUE (DEFAULT); + // SELECT t0.c0 FROM t0 WHERE (t0.c0 BETWEEN (0) AND t0.c0); -- 1 + public static boolean bugNotReported4 = true; + // CREATE TABLE t0(c0 INT); + // INSERT INTO t0(c0) VALUE (DEFAULT); + // SELECT * FROM t0 WHERE (('00')>((('')AND(t0.c0)))); -- expected empty set but got 1 row + public static boolean bugNotReported5 = true; + // CREATE TABLE t0(c0 INT) ; + // INSERT INTO t0(c0) VALUE (100); + // SELECT * FROM t0 WHERE (('Xg')>(t0.c0)); -- expected empty set but got 1 row + // SELECT (('Xg')>(t0.c0)) FROM t0; + public static boolean bugNotReported6 = true; + // CREATE TABLE t0(c0 VARCHAR(1000)); + // INSERT INTO t0(c0) VALUE ('GX9vartD'); + // INSERT INTO t0(c0) VALUE ('16'); + // SELECT t0.c0 FROM t0 WHERE t0.c0; -- expected 1 row but got 2 rows + public static boolean bugNotReported7 = true; private StoneDBBugs() { } diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java index e0c317c53..55b7aff70 100644 --- a/src/sqlancer/stonedb/StoneDBProvider.java +++ b/src/sqlancer/stonedb/StoneDBProvider.java @@ -1,5 +1,7 @@ package sqlancer.stonedb; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported2; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; @@ -76,6 +78,9 @@ private static int mapActions(StoneDBGlobalState globalState, Action a) { case TABLE_CREATE: return r.getInteger(0, 1); case TABLE_DELETE: + if (bugNotReported2) { + return 0; + } return r.getInteger(0, 10); case TABLE_INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); diff --git a/src/sqlancer/stonedb/ast/StoneDBConstant.java b/src/sqlancer/stonedb/ast/StoneDBConstant.java index f0abae48a..6760bc25c 100644 --- a/src/sqlancer/stonedb/ast/StoneDBConstant.java +++ b/src/sqlancer/stonedb/ast/StoneDBConstant.java @@ -1,5 +1,7 @@ package sqlancer.stonedb.ast; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported1; + import java.sql.Timestamp; import java.text.SimpleDateFormat; @@ -85,6 +87,9 @@ public String toString() { } public static Node createDoubleConstant(double val) { + if (bugNotReported1 && val < 0.5 && val > 0) { + return new StoneDBDoubleConstant(0.5); + } return new StoneDBDoubleConstant(val); } diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 54d2c4113..6ed9d59bb 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -1,5 +1,9 @@ package sqlancer.stonedb.gen; +import static sqlancer.stonedb.StoneDBBugs.bug1942; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported3; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported6; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -119,10 +123,20 @@ protected Node generateExpression(int depth) { op = StoneDBUnaryPrefixOperator.getRandom(); return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), op); case UNARY_POSTFIX: - op = StoneDBUnaryPostfixOperator.getRandom(); + if (!bug1942) { + op = StoneDBUnaryPostfixOperator.getRandom(); + } else if (!bugNotReported3) { + op = StoneDBUnaryPostfixOperator.IS_NULL; + } else { + throw new IgnoreMeException(); + } return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), op); case BINARY_COMPARISON: - op = StoneDBBinaryComparisonOperator.getRandom(); + if (!bugNotReported6) { + op = StoneDBBinaryComparisonOperator.getRandom(); + } else { + throw new IgnoreMeException(); + } return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); case IN: return new NewInOperatorNode<>(generateExpression(depth + 1), diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index 2fdb7b6fa..f4435ec26 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -1,5 +1,8 @@ package sqlancer.stonedb.gen; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported4; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported5; + import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -38,7 +41,7 @@ private SQLQueryAdapter getQuery() { sb.append(" "); sb.append(Randomly.fromOptions("LOW_PRIORITY", "DELAYED", "HIGH_PRIORITY")); } - if (Randomly.getBoolean()) { + if (!bugNotReported4 && Randomly.getBoolean()) { sb.append(" IGNORE"); } sb.append(" INTO "); @@ -107,7 +110,7 @@ private void appendOneValue(int nrRow) { // append one column of one row @Override protected void insertValue(StoneDBColumn column) { - if (Randomly.getBooleanWithRatherLowProbability()) { + if (!bugNotReported5 && Randomly.getBooleanWithRatherLowProbability()) { sb.append("DEFAULT"); } else { sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 2625bbb7c..534bf030d 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -1,5 +1,7 @@ package sqlancer.stonedb.oracle; +import static sqlancer.stonedb.StoneDBBugs.bug1953; + import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -77,6 +79,9 @@ private int getUnoptimizedQueryCount(List> tableList, select.setFromList(tableList); select.setJoinList(joins); unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; + if (bug1953) { + unoptimizedQueryString = "SELECT * FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; + } SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { From ea85c22fd8611deed2f5aee32d9499bb1a915811 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:21:40 +0800 Subject: [PATCH 0803/1284] fix: fix implementation bugs of randomly (#873) --- src/sqlancer/stonedb/StoneDBSchema.java | 12 ++++++------ .../stonedb/gen/StoneDBTableAlterGenerator.java | 6 ++++-- .../stonedb/gen/StoneDBTableCreateGenerator.java | 2 +- .../stonedb/gen/StoneDBTableUpdateGenerator.java | 9 ++++----- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java index 2150cbba1..346031fba 100644 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -35,7 +35,7 @@ public static StoneDBDataType getRandom() { return Randomly.fromOptions(values()); } - public static Node getRandomValue(StoneDBDataType dataType) { + public static Node getRandomValue(StoneDBDataType dataType, Randomly r) { switch (dataType) { case TINYINT: break; @@ -70,7 +70,7 @@ public static Node getRandomValue(StoneDBDataType dataType) { case TINYTEXT: break; case TEXT: - return StoneDBConstant.createTextConstant(new Randomly().getString()); + return StoneDBConstant.createTextConstant(r.getString()); case MEDIUMTEXT: break; case LONGTEXT: @@ -93,7 +93,7 @@ public static Node getRandomValue(StoneDBDataType dataType) { return null; } - public static String getTypeAndValue(StoneDBDataType dataType) { + public static String getTypeAndValue(StoneDBDataType dataType, Randomly r) { StringBuilder sb = new StringBuilder(); switch (dataType) { case TINYINT: @@ -127,10 +127,10 @@ public static String getTypeAndValue(StoneDBDataType dataType) { case TIMESTAMP: return "TIMESTAMP"; case CHAR: - sb.append("CHAR").append(Randomly.fromOptions("", "(" + new Randomly().getInteger(0, 255) + ")")); + sb.append("CHAR").append(Randomly.fromOptions("", "(" + r.getInteger(0, 255) + ")")); return sb.toString(); case VARCHAR: - sb.append("VARCHAR").append("(").append(new Randomly().getInteger(0, 65535)).append(")"); + sb.append("VARCHAR").append("(").append(r.getInteger(0, 65535)).append(")"); return sb.toString(); case TINYTEXT: return "TINYTEXT"; @@ -143,7 +143,7 @@ public static String getTypeAndValue(StoneDBDataType dataType) { case BINARY: return "BINARY"; case VARBINARY: - sb.append("VARBINARY").append("(").append(new Randomly().getInteger(0, 65535)).append(")"); + sb.append("VARBINARY").append("(").append(r.getInteger(0, 65535)).append(")"); return sb.toString(); case TINYBLOB: return "TINYBLOB"; diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index b1e4872aa..0510f7a78 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -77,7 +77,8 @@ private void appendAlterOption(Action action) { sb.append("ADD COLUMN "); String columnName = table.getFreeColumnName(); sb.append(" ").append(columnName).append(" "); - sb.append(StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull())); + sb.append( + StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT // instead errors.addRegex(Pattern @@ -111,7 +112,8 @@ private void appendAlterOption(Action action) { String oldColumnName = table.getRandomColumn().getName(); String newColumnName = table.getFreeColumnName(); sb.append(oldColumnName).append(" ").append(newColumnName).append(" "); - sb.append(StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull())); + sb.append( + StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT // instead errors.addRegex(Pattern diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java index 3b87e99d5..c288c57fd 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java @@ -261,6 +261,6 @@ private void appendColumnOption(StoneDBDataType type) { } private void appendType(StoneDBDataType randomType) { - sb.append(StoneDBDataType.getTypeAndValue(randomType)); + sb.append(StoneDBDataType.getTypeAndValue(randomType, r)); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java index 4812a6ee3..259136135 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java @@ -49,11 +49,10 @@ public void appendAssignmentList() { } sb.append(table.getColumns().get(i)); sb.append(" = "); - sb.append( - Randomly.fromOptions( - StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType - .getRandomValue(table.getColumns().get(i).getType().getPrimitiveDataType())), - "DEFAULT")); + sb.append(Randomly.fromOptions( + StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType.getRandomValue( + table.getColumns().get(i).getType().getPrimitiveDataType(), globalState.getRandomly())), + "DEFAULT")); } } From 5839a787f6168b9baea2a1f6cf58fb4b7d05d978 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:22:07 +0800 Subject: [PATCH 0804/1284] refactor: refactor stonedb aggregate function (#874) --- .../stonedb/ast/StoneDBAggregate.java | 34 +++++++++++++++++++ .../gen/StoneDBExpressionGenerator.java | 20 +---------- .../oracle/StoneDBAggregateOracle.java | 2 +- 3 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 src/sqlancer/stonedb/ast/StoneDBAggregate.java diff --git a/src/sqlancer/stonedb/ast/StoneDBAggregate.java b/src/sqlancer/stonedb/ast/StoneDBAggregate.java new file mode 100644 index 000000000..2e8fb3bc0 --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBAggregate.java @@ -0,0 +1,34 @@ +package sqlancer.stonedb.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; + +public class StoneDBAggregate extends FunctionNode + implements StoneDBExpression { + + // https://stonedb.io/docs/SQL-reference/functions/aggregate-functions/ + public enum StoneDBAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1), FIRST(1), SUM(1); + + private int nrArgs; + + StoneDBAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static StoneDBAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + protected StoneDBAggregate(StoneDBAggregateFunction function, List args) { + super(function, args); + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 6ed9d59bb..36adf85f2 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -26,6 +26,7 @@ import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -180,25 +181,6 @@ protected List> generateColumns() { return new ArrayList<>(set); } - // https://stonedb.io/docs/SQL-reference/functions/aggregate-functions/ - public enum StoneDBAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1), FIRST(1), SUM(1); - - private int nrArgs; - - StoneDBAggregateFunction(int nrArgs) { - this.nrArgs = nrArgs; - } - - public static StoneDBAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - } - public enum StoneDBUnaryPrefixOperator implements Operator { NOT("NOT"), PLUS("+"), MINUS("-"), INVERSION("!"); diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java index df09e8961..652fa30ed 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -19,9 +19,9 @@ import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; import sqlancer.stonedb.ast.StoneDBExpression; import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBAggregateFunction; import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPostfixOperator; import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPrefixOperator; From ec8c8b03cd44a0567400ae0d8e9ad994c4e2092d Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:22:35 +0800 Subject: [PATCH 0805/1284] feat: stonedb view (#872) --- src/sqlancer/stonedb/StoneDBProvider.java | 10 ++- .../gen/StoneDBViewCreateGenerator.java | 78 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java index 55b7aff70..e4d9c46f8 100644 --- a/src/sqlancer/stonedb/StoneDBProvider.java +++ b/src/sqlancer/stonedb/StoneDBProvider.java @@ -28,6 +28,7 @@ import sqlancer.stonedb.gen.StoneDBTableDeleteGenerator; import sqlancer.stonedb.gen.StoneDBTableInsertGenerator; import sqlancer.stonedb.gen.StoneDBTableUpdateGenerator; +import sqlancer.stonedb.gen.StoneDBViewCreateGenerator; @AutoService(DatabaseProvider.class) public class StoneDBProvider extends SQLProviderAdapter { @@ -54,7 +55,12 @@ enum Action implements AbstractAction { TABLE_UPDATE(StoneDBTableUpdateGenerator::generate), // INDEX_CREATE(StoneDBIndexCreateGenerator::generate), // INDEX_DROP(StoneDBIndexDropGenerator::generate), // - TABLE_INSERT(StoneDBTableInsertGenerator::generate); // + TABLE_INSERT(StoneDBTableInsertGenerator::generate), // + + VIEW_CREATE((g) -> { + String viewName = g.getSchema().getFreeViewName(); + return StoneDBViewCreateGenerator.generate(g, viewName); + }); private final SQLQueryProvider sqlQueryProvider; @@ -90,6 +96,8 @@ private static int mapActions(StoneDBGlobalState globalState, Action a) { return r.getInteger(0, 1); case INDEX_DROP: return r.getInteger(0, 1); + case VIEW_CREATE: + return r.getInteger(0, 1); default: throw new AssertionError(a); } diff --git a/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java new file mode 100644 index 000000000..3ee25ffbd --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java @@ -0,0 +1,78 @@ +package sqlancer.stonedb.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBSchema.StoneDBTables; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; + +public class StoneDBViewCreateGenerator { + // the name of the view to create + private final String viewName; + private final StoneDBGlobalState globalState; + private StoneDBSelect select; + private final StringBuilder sb = new StringBuilder(); + + public StoneDBViewCreateGenerator(StoneDBGlobalState globalState, String viewName) { + this.globalState = globalState; + this.viewName = viewName; + setSelect(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String viewName) { + return new StoneDBViewCreateGenerator(globalState, viewName).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("CREATE "); + sb.append("VIEW "); + sb.append(viewName); + sb.append(" AS "); + sb.append(StoneDBToStringVisitor.asString(select)); + return new SQLQueryAdapter(sb.toString(), true); + } + + private void setSelect() { + StoneDBSchema schema = globalState.getSchema(); + StoneDBTables targetTables = schema.getRandomTableNonEmptyTables(); + StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); + + select = new StoneDBSelect(); + select.setFetchColumns(generateFetchColumns(targetTables)); + List tables = targetTables.getTables(); + List> tableReferenceNodeList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = StoneDBJoin.getJoins(tableReferenceNodeList, globalState); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableReferenceNodeList)); + select.setWhereClause(gen.generateExpression()); + select.setOrderByExpressions(gen.generateOrderBys()); + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + + private List> generateFetchColumns(StoneDBTables tables) { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))); + } else { + columns = Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + } + return columns; + } +} From c7c0d7683399475f67912bc799a111dd773abd6a Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:24:20 +0800 Subject: [PATCH 0806/1284] fix: syntax error (#871) --- .../stonedb/gen/StoneDBTableAlterGenerator.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index 0510f7a78..64ddefa1b 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -1,6 +1,5 @@ package sqlancer.stonedb.gen; -import java.util.List; import java.util.regex.Pattern; import sqlancer.Randomly; @@ -34,7 +33,7 @@ private SQLQueryAdapter getQuery() { sb.append("ALTER TABLE "); sb.append(table.getName()); sb.append(" "); - appendAlterOptions(); + appendAlterOption(Randomly.fromOptions(Action.values())); addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors, true); } @@ -56,19 +55,6 @@ private void addExpectedErrors() { .compile("Column length too big for column 'c\\d{1,3}' \\(max = 16383\\); use BLOB or TEXT instead")); } - private void appendAlterOptions() { - List actions; - if (Randomly.getBooleanWithSmallProbability()) { - actions = Randomly.subset(Action.values()); - } else { - actions = List.of(Randomly.fromOptions(Action.values())); - } - for (Action action : actions) { - appendAlterOption(action); - sb.append(" "); - } - } - private void appendAlterOption(Action action) { StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) .setColumns(table.getColumns()); From 290288235e49a53de9e7663fc59d44d7bbdd4787 Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:39:53 +0800 Subject: [PATCH 0807/1284] feat: write log when running statement reducer (#864) * feat: write log when running statement reducer + write log when start reducing, finish reducing and when a reduction step is successfully performed + fix some bugs of the statement reducer. * fix: check upper bound of subList to avoid outOfBoundException * overwrite log content of reducer * feat: create symbolic link to the latest reduce log * fix format issues * fix: log reduced result overwritting previous steps --- src/sqlancer/Main.java | 53 ++++++++++++++++- src/sqlancer/StatementReducer.java | 58 +++++++++++++------ test/sqlancer/reducer/TestEnvironment.java | 10 +++- .../reducer/TestStatementReducer.java | 7 +++ 4 files changed, 106 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 76d7a36f4..70657c3d6 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -76,12 +76,16 @@ public static final class StateLogger { private final File loggerFile; private File curFile; private File queryPlanFile; + private File reduceFile; private FileWriter logFileWriter; public FileWriter currentFileWriter; private FileWriter queryPlanFileWriter; + private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; private final boolean logQueryPlan; + + private final boolean useReducer; private final DatabaseProvider databaseProvider; private static final class AlsoWriteToConsoleFileWriter extends FileWriter { @@ -118,6 +122,15 @@ public StateLogger(String databaseName, DatabaseProvider provider, Main if (logQueryPlan) { queryPlanFile = new File(dir, databaseName + "-plan.log"); } + this.useReducer = options.useReducer(); + if (useReducer) { + File reduceFileDir = new File(dir, "reduce"); + if (!reduceFileDir.exists()) { + reduceFileDir.mkdir(); + } + this.reduceFile = new File(reduceFileDir, databaseName + "-reduce.log"); + + } this.databaseProvider = provider; } @@ -183,6 +196,20 @@ public FileWriter getQueryPlanFileWriter() { return queryPlanFileWriter; } + public FileWriter getReduceFileWriter() { + if (!useReducer) { + throw new UnsupportedOperationException(); + } + FileWriter fileWriter; + try { + fileWriter = new FileWriter(reduceFile, false); + } catch (IOException e) { + throw new AssertionError(e); + } + + return fileWriter; + } + public void writeCurrent(StateToReproduce state) { if (!logEachSelect) { throw new UnsupportedOperationException(); @@ -229,6 +256,30 @@ public void writeQueryPlan(String queryPlan) { } } + public void logReduced(StateToReproduce state) { + FileWriter reduceFileWriter = getReduceFileWriter(); + + StringBuilder sb = new StringBuilder(); + for (Query s : state.getStatements()) { + sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); + } + try { + reduceFileWriter.write(sb.toString()); + + } catch (IOException e) { + throw new AssertionError(e); + } finally { + try { + reduceFileWriter.flush(); + reduceFileWriter.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + } + public void logException(Throwable reduce, StateToReproduce state) { Loggable stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); @@ -395,7 +446,7 @@ public void run() throws Exception { } if (reproducer != null && options.useReducer()) { System.out.println("EXPERIMENTAL: Trying to reduce queries using a simple reducer."); - System.out.println("Reduced query will be output to stdout but not logs."); + // System.out.println("Reduced query will be output to stdout but not logs."); G newGlobalState = createGlobalState(); newGlobalState.setState(stateToRepro); newGlobalState.setRandomly(r); diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java index 3d3d36711..e066aca84 100644 --- a/src/sqlancer/StatementReducer.java +++ b/src/sqlancer/StatementReducer.java @@ -13,6 +13,14 @@ public class StatementReducer, O extends DBMSSpec private boolean observedChange; private int partitionNum; + private long currentReduceSteps; + private long currentReduceTime; + + private long maxReduceSteps; + private long maxReduceTime; + + Instant timeOfReductionBegins; + public StatementReducer(DatabaseProvider provider) { this.provider = provider; } @@ -28,25 +36,26 @@ private boolean hasNotReachedLimit(long curr, long limit) { @Override public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { - long maxReduceTime = state.getOptions().getMaxStatementReduceTime(); - long maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); + maxReduceTime = state.getOptions().getMaxStatementReduceTime(); + maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); List> knownToReproduceBugStatements = new ArrayList<>(); for (Query stat : state.getState().getStatements()) { knownToReproduceBugStatements.add((Query) stat); } - System.out.println("Starting query:"); - printQueries(knownToReproduceBugStatements); - System.out.println(); + // System.out.println("Starting query:"); + // Main.StateLogger logger = newGlobalState.getLogger(); + // printQueries(knownToReproduceBugStatements); + // System.out.println(); if (knownToReproduceBugStatements.size() <= 1) { return; } - Instant timeOfReductionBegins = Instant.now(); - long currentReduceSteps = 0; - long currentReduceTime = 0; + timeOfReductionBegins = Instant.now(); + currentReduceSteps = 0; + currentReduceTime = 0; partitionNum = 2; while (knownToReproduceBugStatements.size() >= 2 && hasNotReachedLimit(currentReduceSteps, maxReduceSteps) @@ -55,23 +64,21 @@ && hasNotReachedLimit(currentReduceTime, maxReduceTime)) { knownToReproduceBugStatements = tryReduction(state, reproducer, newGlobalState, knownToReproduceBugStatements); + if (!observedChange) { if (partitionNum == knownToReproduceBugStatements.size()) { break; } // increase the search granularity partitionNum = Math.min(partitionNum * 2, knownToReproduceBugStatements.size()); - - currentReduceSteps++; - Instant currentInstant = Instant.now(); - currentReduceTime = Duration.between(currentInstant, timeOfReductionBegins).getSeconds(); } - } - System.out.println("Reduced query:"); - printQueries(knownToReproduceBugStatements); + // System.out.println("Reduced query:"); + // printQueries(knownToReproduceBugStatements); newGlobalState.getState().setStatements(new ArrayList<>(knownToReproduceBugStatements)); + newGlobalState.getLogger().logReduced(newGlobalState.getState()); + } private List> tryReduction(G state, // NOPMD @@ -87,7 +94,8 @@ private List> tryReduction(G state, // NOPMD try (C con2 = provider.createDatabase(newGlobalState)) { newGlobalState.setConnection(con2); List> candidateStatements = new ArrayList<>(statements); - candidateStatements.subList(start, start + subLength).clear(); + int endPoint = Math.min(start + subLength, candidateStatements.size()); + candidateStatements.subList(start, endPoint).clear(); newGlobalState.getState().setStatements(new ArrayList<>(candidateStatements)); for (Query s : candidateStatements) { @@ -102,19 +110,32 @@ private List> tryReduction(G state, // NOPMD observedChange = true; statements = candidateStatements; partitionNum = Math.max(partitionNum - 1, 2); - break; // reproducer.outputHook((SQLite3GlobalState) newGlobalState); - // state.getLogger().logReduced(newGlobalState.getState()); + newGlobalState.getLogger().logReduced(newGlobalState.getState()); + break; + } } catch (Throwable ignoredException) { } + } catch (Exception e) { + e.printStackTrace(); + } + + currentReduceSteps++; + Instant currentInstant = Instant.now(); + + currentReduceTime = Duration.between(timeOfReductionBegins, currentInstant).getSeconds(); + if (!hasNotReachedLimit(currentReduceSteps, maxReduceSteps) + || !hasNotReachedLimit(currentReduceTime, maxReduceTime)) { + return statements; } start = start + subLength; } return statements; } + @SuppressWarnings("unused") private void printQueries(List> statements) { System.out.println("==============================="); for (Query q : statements) { @@ -122,5 +143,4 @@ private void printQueries(List> statements) { } System.out.println("==============================="); } - } diff --git a/test/sqlancer/reducer/TestEnvironment.java b/test/sqlancer/reducer/TestEnvironment.java index 3ba1fc11e..7c1155490 100644 --- a/test/sqlancer/reducer/TestEnvironment.java +++ b/test/sqlancer/reducer/TestEnvironment.java @@ -6,6 +6,7 @@ import sqlancer.reducer.VirtualDB.VirtualDBProvider; import sqlancer.reducer.VirtualDB.VirtualDBQuery; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; @@ -88,6 +89,10 @@ private void setUpTestingEnvironment() throws Exception { state.setState(stateToReproduce); state.setDatabaseName(databaseName); + // A really hacky way to enable reducer... + Field field = options.getClass().getDeclaredField("useReducer"); + field.setAccessible(true); + field.set(options, true); state.setMainOptions(options); // Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); @@ -96,8 +101,9 @@ private void setUpTestingEnvironment() throws Exception { try (SQLConnection con = provider.createDatabase(state)) { state.setConnection(con); newGlobalState = createGlobalState(); - // Main.StateLogger newLogger = new Main.StateLogger(databaseName, provider, options); - // newGlobalState.setStateLogger(newLogger); + Main.StateLogger newLogger = new Main.StateLogger(databaseName, provider, options); + newGlobalState.setStateLogger(newLogger); + state.setStateLogger(newLogger); newGlobalState.setState(stateToReproduce); newGlobalState.setDatabaseName(databaseName); newGlobalState.setMainOptions(options); diff --git a/test/sqlancer/reducer/TestStatementReducer.java b/test/sqlancer/reducer/TestStatementReducer.java index 66a9a3c4c..d9e364c84 100644 --- a/test/sqlancer/reducer/TestStatementReducer.java +++ b/test/sqlancer/reducer/TestStatementReducer.java @@ -1,6 +1,7 @@ package sqlancer.reducer; import org.junit.jupiter.api.Test; +import sqlancer.Main; import sqlancer.common.query.Query; import java.util.ArrayList; @@ -72,4 +73,10 @@ void testDeltaDebuggingWithStatementsCombination() throws Exception { assertEquals(queriesString, "Statement_2;\nStatement_318;\nStatement_990;"); } + @Test + void testSQLite3WithStatementReducer() { + Main.executeMain(new String[] { "--random-seed", "0", "--use-reducer", "--timeout-seconds", "60", + "--num-threads", "4", "sqlite3", "--oracle", "NoREC" }); + } + } From b6cf5c03e474ed171136c0789d8aa8b3efc6e2b6 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Fri, 11 Aug 2023 10:28:19 +0800 Subject: [PATCH 0808/1284] feat: add expected errors for stonedb (#867) * feat: add expected errors * feat: add expected errors --- src/sqlancer/stonedb/StoneDBErrors.java | 29 +++++++++++++++++++ .../gen/StoneDBTableAlterGenerator.java | 15 ++++++++++ .../gen/StoneDBTableDeleteGenerator.java | 6 ++++ .../gen/StoneDBTableInsertGenerator.java | 4 +++ .../oracle/StoneDBAggregateOracle.java | 2 ++ .../stonedb/oracle/StoneDBNoRECOracle.java | 2 ++ 6 files changed, 58 insertions(+) diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java index bdbd6a447..1e72e5685 100644 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -1,6 +1,35 @@ package sqlancer.stonedb; +import sqlancer.common.query.ExpectedErrors; + public final class StoneDBErrors { private StoneDBErrors() { } + + public static void addExpectedExpressionErrors(ExpectedErrors errors) { + // java.sql.SQLException: Incorrect DATE value: '292269055-12-02' + errors.add("Incorrect DATE value: "); + // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 + errors.add("Incorrect string value: "); + // java.sql.SQLException: Incorrect integer value: 'ST' for column 'c1' at row 1 + errors.add("Incorrect integer value: "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Truncated incorrect INTEGER value: '#Q' + errors.add("Data truncation: Truncated incorrect INTEGER value: "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT value is out of range in + // '-((`database0`.`t0`.`c1` >> (not(`database0`.`t0`.`c1`))))' + errors.add("Data truncation: BIGINT value is out of range in "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT UNSIGNED value is out of range in + // '(`database10`.`t0`.`c0` + (`database10`.`t0`.`c0` & (not(0.5))))' + errors.add("Data truncation: BIGINT UNSIGNED value is out of range in "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect time value: '0Sly6xqF0' for + // column 'c1' at row 1 + errors.add("Data truncation: Incorrect time value: "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: DOUBLE value is out of range in + // '(`database0`.`t0`.`c0` * `database0`.`t0`.`c0`)' + errors.add("Data truncation: DOUBLE value is out of range in "); + // java.sql.SQLException: Numeric result of an expression is too large and cannot be handled by tianmu. + errors.add("Numeric result of an expression is too large and cannot be handled by tianmu."); + // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order clause' + errors.add("Unknown column "); + } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index 64ddefa1b..e9c283cc9 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -41,6 +41,14 @@ private SQLQueryAdapter getQuery() { private void addExpectedErrors() { // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Data too long for column 'c0' at row 2 errors.addRegex(Pattern.compile("Data truncation: Data too long for column 'c\\d{1,3}' at row \\d{1,3}")); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: + // '0.571272522740968' for column 'c1' at row 1 + errors.add("Incorrect datetime value: "); + // java.sql.SQLSyntaxErrorException: Invalid default value for 'c0' + errors.add("Invalid default value for "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row + // 2 + errors.add("Data truncation: Out of range value for column "); // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes errors.add("Specified key was too long; max key length is 3072 bytes"); // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead @@ -98,12 +106,19 @@ private void appendAlterOption(Action action) { String oldColumnName = table.getRandomColumn().getName(); String newColumnName = table.getFreeColumnName(); sb.append(oldColumnName).append(" ").append(newColumnName).append(" "); + errors.add("Incorrect integer value: "); + // java.sql.SQLException: Data truncated for column 'c1' at row 1 + errors.add("Data truncated for column "); sb.append( StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT // instead errors.addRegex(Pattern .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); + // java.sql.SQLSyntaxErrorException: BLOB column 'c1' can't be used in key specification with the used table + // type + errors.addRegex(Pattern + .compile("BLOB column 'c\\d{1,3}' can't be used in key specification with the used table type")); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index ab0828d30..16b9c1061 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -55,6 +55,12 @@ public SQLQueryAdapter getQuery() { sb.append(" LIMIT "); sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); } + addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } + + private void addExpectedErrors() { + // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 + errors.add("Incorrect string value: "); + } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index f4435ec26..19b650b82 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -53,6 +53,10 @@ private SQLQueryAdapter getQuery() { } private void addExpectedErrors() { + // java.sql.SQLException: Incorrect DATE value: '292278994-08-17' + errors.add("Incorrect DATE value: '"); + // java.sql.SQLIntegrityConstraintViolationException: Duplicate entry '1970-01-14' for key 'PRIMARY' + errors.add("Duplicate entry "); // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row errors.add("Data truncation: Out of range value for column "); // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 'field list' diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java index 652fa30ed..27ba91013 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -15,6 +15,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.stonedb.StoneDBErrors; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; @@ -30,6 +31,7 @@ public class StoneDBAggregateOracle extends StoneDBQueryPartitioningBase { public StoneDBAggregateOracle(StoneDBGlobalState state) { super(state); + StoneDBErrors.addExpectedExpressionErrors(errors); } @Override diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 534bf030d..e042cd44a 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -21,6 +21,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.stonedb.StoneDBErrors; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; @@ -42,6 +43,7 @@ public class StoneDBNoRECOracle extends NoRECBase implements public StoneDBNoRECOracle(StoneDBGlobalState globalState) { super(globalState); this.schema = globalState.getSchema(); + StoneDBErrors.addExpectedExpressionErrors(errors); } @Override From 0009112ada915f120bb5b2209455d3794093d939 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Sat, 12 Aug 2023 10:05:10 +0800 Subject: [PATCH 0809/1284] fix: fix implementation bugs (#876) * fix: drop features StoneDB not supported * fix: add supported operator * git: merge latest changes --- src/sqlancer/stonedb/ast/StoneDBAggregate.java | 2 +- src/sqlancer/stonedb/ast/StoneDBJoin.java | 2 +- src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/stonedb/ast/StoneDBAggregate.java b/src/sqlancer/stonedb/ast/StoneDBAggregate.java index 2e8fb3bc0..f65b54d74 100644 --- a/src/sqlancer/stonedb/ast/StoneDBAggregate.java +++ b/src/sqlancer/stonedb/ast/StoneDBAggregate.java @@ -11,7 +11,7 @@ public class StoneDBAggregate extends FunctionNode Date: Mon, 14 Aug 2023 12:07:50 +0800 Subject: [PATCH 0810/1284] feat: bypass logical XOR (#877) * feat: bypass logical XOR * fix: change import order * fix: fix style check error --- src/sqlancer/stonedb/StoneDBBugs.java | 6 ++++++ src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java index 11a144dad..4864c56e5 100644 --- a/src/sqlancer/stonedb/StoneDBBugs.java +++ b/src/sqlancer/stonedb/StoneDBBugs.java @@ -39,6 +39,12 @@ public final class StoneDBBugs { // INSERT INTO t0(c0) VALUE ('16'); // SELECT t0.c0 FROM t0 WHERE t0.c0; -- expected 1 row but got 2 rows public static boolean bugNotReported7 = true; + // CREATE TABLE t0(c0 INT) ; + // SELECT * FROM t0 WHERE ((t0.c0)XOR((t0.c0))); + // ERROR 6 (HY000): The query includes syntax that is not supported by the storage engine. Either restructure the + // query with supported syntax, or enable the MySQL core::Query Path in config file to execute the query with + // reduced performance. + public static boolean bugNotReportedXOR = true; private StoneDBBugs() { } diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 36adf85f2..6ff9f1b5b 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -3,12 +3,14 @@ import static sqlancer.stonedb.StoneDBBugs.bug1942; import static sqlancer.stonedb.StoneDBBugs.bugNotReported3; import static sqlancer.stonedb.StoneDBBugs.bugNotReported6; +import static sqlancer.stonedb.StoneDBBugs.bugNotReportedXOR; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -146,7 +148,12 @@ protected Node generateExpression(int depth) { return new NewInOperatorNode<>(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), true); case BINARY_LOGICAL: - op = StoneDBBinaryLogicalOperator.getRandom(); + if (bugNotReportedXOR) { + op = Randomly.fromList(Arrays.stream(StoneDBBinaryLogicalOperator.values()) + .filter(p -> p != StoneDBBinaryLogicalOperator.XOR).collect(Collectors.toList())); + } else { + op = StoneDBBinaryLogicalOperator.getRandom(); + } return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_ARITHMETIC: op = StoneDBBinaryArithmeticOperator.getRandom(); From 9bcc3ffe819e87e80fe9cd2e0d1f7058501059ee Mon Sep 17 00:00:00 2001 From: Jiansen Song Date: Mon, 14 Aug 2023 12:08:30 +0800 Subject: [PATCH 0811/1284] fix: avoid logging CREATE TABLE exception (#875) * fix: avoid logging CREATE TABLE exception * Update TestLoggableFactory.java * Update SQLLoggableFactory.java We need to check the query in the constructor * Update SQLLoggableFactory.java Fix wrong indent. * fix: check couldAffectSchema before setting false * fix: revise check query string --------- Co-authored-by: Yichen Yan --- src/sqlancer/common/query/SQLQueryAdapter.java | 12 +++++++++--- test/sqlancer/TestLoggableFactory.java | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 test/sqlancer/TestLoggableFactory.java diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 22eef268a..8e698676a 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -24,7 +24,11 @@ public SQLQueryAdapter(String query, boolean couldAffectSchema) { } public SQLQueryAdapter(String query, ExpectedErrors expectedErrors) { - this(query, expectedErrors, false); + this(query, expectedErrors, guessAffectSchemaFromQuery(query)); + } + + private static boolean guessAffectSchemaFromQuery(String query) { + return query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN"); } public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema) { @@ -46,8 +50,10 @@ private String canonicalizeString(String s) { } private void checkQueryString() { - if (query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN") && !couldAffectSchema) { - throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); + if (!couldAffectSchema) { + if (guessAffectSchemaFromQuery(query)) { + throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); + } } } diff --git a/test/sqlancer/TestLoggableFactory.java b/test/sqlancer/TestLoggableFactory.java new file mode 100644 index 000000000..2679cf740 --- /dev/null +++ b/test/sqlancer/TestLoggableFactory.java @@ -0,0 +1,17 @@ +package sqlancer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.log.SQLLoggableFactory; +import sqlancer.common.query.SQLQueryAdapter; + +public class TestLoggableFactory { + + @Test + public void testLogCreateTable() { + String query = "CREATE TABLE t1 (c1 INT)"; + SQLLoggableFactory logger = new SQLLoggableFactory(); + SQLQueryAdapter queryAdapter = logger.getQueryForStateToReproduce(query); + assert (queryAdapter.couldAffectSchema()); + } + +} From b694a3abda768ab9f562067d57f15a0a5b9db3b1 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 22 Aug 2023 23:33:23 +0800 Subject: [PATCH 0812/1284] feat: add support of insert on duplicate key update (#883) --- src/sqlancer/common/query/SQLQueryAdapter.java | 6 ++---- .../stonedb/gen/StoneDBTableInsertGenerator.java | 9 +++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 8e698676a..248403458 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -50,10 +50,8 @@ private String canonicalizeString(String s) { } private void checkQueryString() { - if (!couldAffectSchema) { - if (guessAffectSchemaFromQuery(query)) { - throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); - } + if (!couldAffectSchema && guessAffectSchemaFromQuery(query)) { + throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index 19b650b82..a9823be6d 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -48,6 +48,7 @@ private SQLQueryAdapter getQuery() { sb.append(table.getName()); appendPartition(); appendColumnsAndValues(columns); + appendOnDuplicateUpdate(); addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } @@ -83,6 +84,14 @@ private void appendColumnsAndValues(List columns) { appendValues(); } + private void appendOnDuplicateUpdate() { + sb.append("on duplicate key update "); + StoneDBColumn randomColumn = table.getRandomColumn(); + sb.append(randomColumn.getName()); + sb.append("="); + insertValue(randomColumn); + } + // append nrRows rows private void appendValues() { int nrRows; From 7f37778cae1064c03f8c4721fbe55d88253996d5 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 22 Aug 2023 23:34:12 +0800 Subject: [PATCH 0813/1284] feat: add support of advanced functions (#882) --- .../stonedb/ast/StoneDBAdvancedFunction.java | 34 +++++++++++++++++++ .../gen/StoneDBExpressionGenerator.java | 6 ++++ 2 files changed, 40 insertions(+) create mode 100644 src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java diff --git a/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java b/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java new file mode 100644 index 000000000..9c49f2678 --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java @@ -0,0 +1,34 @@ +package sqlancer.stonedb.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.stonedb.ast.StoneDBAdvancedFunction.StoneDBAdvancedFunc; + +public class StoneDBAdvancedFunction extends FunctionNode + implements StoneDBExpression { + + // https://stonedb.io/docs/SQL-reference/functions/advanced-functions + public enum StoneDBAdvancedFunc { + IFNULL(2), IF(3), NULLIF(2), BIN(1), BINARY(1), CONV(3); + + private int nrArgs; + + StoneDBAdvancedFunc(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static StoneDBAdvancedFunc getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + protected StoneDBAdvancedFunction(StoneDBAdvancedFunc function, List args) { + super(function, args); + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 6ff9f1b5b..3e1dcd348 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -28,6 +28,7 @@ import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.ast.StoneDBAdvancedFunction.StoneDBAdvancedFunc; import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -118,6 +119,11 @@ protected Node generateExpression(int depth) { return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs(), depth + 1), aggregateFunction); } + if (Randomly.getBooleanWithRatherLowProbability()) { + StoneDBAdvancedFunc advancedFunction = StoneDBAdvancedFunc.getRandom(); + return new NewFunctionNode<>(generateExpressions(advancedFunction.getNrArgs(), depth + 1), + advancedFunction); + } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); Expression expr = Randomly.fromList(possibleOptions); Operator op; From 11d57fde3d4870ebfadf4f1417c503803005dec3 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Wed, 23 Aug 2023 10:12:50 +0800 Subject: [PATCH 0814/1284] feat: add support of Fuzz oracle (#869) * feat: add expected ExpressionErrors feat: add Fuzz oracle feat: add expected errors feat: add one test oracle * ignore: let CI run * feat: generate random columns * fix: remove redundant select * fix: change enum name * fix: change enum name --- .github/workflows/main.yml | 31 +-------- src/sqlancer/stonedb/StoneDBOptions.java | 7 ++ .../stonedb/oracle/StoneDBFuzzOracle.java | 67 +++++++++++++++++++ test/sqlancer/dbms/TestStoneDBFuzz.java | 21 ++++++ 4 files changed, 98 insertions(+), 28 deletions(-) create mode 100644 src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java create mode 100644 test/sqlancer/dbms/TestStoneDBFuzz.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9aaed3c70..ec8698f03 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -375,8 +375,8 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test - stonedb-norec: - name: DBMS Tests (StoneDB NoRec) + stonedb: + name: DBMS Tests (StoneDB) runs-on: ubuntu-latest steps: @@ -391,32 +391,7 @@ jobs: sleep 60s docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 - with: - java-version: 11 - - name: Build - run: mvn -B package -DskipTests=true - - name: StoneDB Tests - run: | - STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBNoRec - - stonedb-tlp: - name: DBMS Tests (StoneDB TLP) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up StoneDB - run: | - docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb - ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb --format "{{.ID}}") - docker exec $ContainerID bash sleep 60s - docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf - docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Set up JDK 11 uses: actions/setup-java@v1.4.4 with: @@ -425,7 +400,7 @@ jobs: run: mvn -B package -DskipTests=true - name: StoneDB Tests run: | - STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBTLP + STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBFuzz tidb: name: DBMS Tests (TiDB) diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java index 7e6a2f09b..a9728323f 100644 --- a/src/sqlancer/stonedb/StoneDBOptions.java +++ b/src/sqlancer/stonedb/StoneDBOptions.java @@ -14,6 +14,7 @@ import sqlancer.stonedb.StoneDBOptions.StoneDBOracleFactory; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.oracle.StoneDBAggregateOracle; +import sqlancer.stonedb.oracle.StoneDBFuzzOracle; import sqlancer.stonedb.oracle.StoneDBNoRECOracle; import sqlancer.stonedb.oracle.StoneDBQueryPartitioningDistinctTester; import sqlancer.stonedb.oracle.StoneDBQueryPartitioningGroupByTester; @@ -30,6 +31,12 @@ public class StoneDBOptions implements DBMSSpecificOptions public List oracles = List.of(StoneDBOracleFactory.NOREC); public enum StoneDBOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBFuzzOracle(globalState); + } + }, NOREC { @Override public TestOracle create(StoneDBGlobalState globalState) throws SQLException { diff --git a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java new file mode 100644 index 000000000..7e4d6a622 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java @@ -0,0 +1,67 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBErrors; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBSchema.StoneDBTables; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator; + +public class StoneDBFuzzOracle implements TestOracle { + private final StoneDBGlobalState globalState; + private final StoneDBSchema schema; + private final ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBFuzzOracle(StoneDBGlobalState globalState) { + this.globalState = globalState; + this.schema = globalState.getSchema(); + StoneDBErrors.addExpectedExpressionErrors(errors); + } + + @Override + public void check() throws Exception { + StoneDBTables randomTables = schema.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(globalState).setColumns(columns); + Node randomWhereCondition = gen.generateExpression(); + List tables = randomTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List> joins = StoneDBJoin.getJoins(tableList, globalState); + + StoneDBSelect select = new StoneDBSelect(); + select.setFetchColumns( + Randomly.fromOptions(List.of(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))), + allColumns, Randomly.subset(allColumns))); + select.setFromList(new ArrayList<>(tableList)); + select.setJoinList(joins); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions( + new StoneDBExpressionGenerator(globalState).setColumns(columns).generateOrderBys()); + } + select.setJoinList(joins); + + String queryString = StoneDBToStringVisitor.asString(select); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + q.executeAndGetLogged(globalState); + } +} diff --git a/test/sqlancer/dbms/TestStoneDBFuzz.java b/test/sqlancer/dbms/TestStoneDBFuzz.java new file mode 100644 index 000000000..2f5d52424 --- /dev/null +++ b/test/sqlancer/dbms/TestStoneDBFuzz.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestStoneDBFuzz { + + String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); + boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); + + @Test + public void testStoneDB() { + assumeTrue(stoneDBIsAvailable); + assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", + "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "FUZZER")); + } + +} \ No newline at end of file From 48aadfd27a889bb7544db6cbd18ad4608606b3bb Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Wed, 23 Aug 2023 23:04:43 +0800 Subject: [PATCH 0815/1284] add doc of test case reduction (#880) * add doc of test case reduction * made some improvement --- docs/testCaseReduction.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docs/testCaseReduction.md diff --git a/docs/testCaseReduction.md b/docs/testCaseReduction.md new file mode 100644 index 000000000..35aa9cd08 --- /dev/null +++ b/docs/testCaseReduction.md @@ -0,0 +1,31 @@ +# Test Case Reduction +SQLancer generates a large number of statements, but not all of them are relevant to the bug. To automatically reduce the test cases, two reducers were implemented: the statement reducer and the AST-based reducer. + +## Statement Reducer +The statement reducer utilizes the delta-debugging technique to remove irrelevant statements. More details of delta-debugging could be found in this paper: [Simplifying and Isolating Failure-Inducing Input](https://www.cs.purdue.edu/homes/xyzhang/fall07/Papers/delta-debugging.pdf). + +Using the statement reducer, SQLancer reduces the set of statements to a minimal subset that reproduces the bug. + +## AST-Based Reducer +The AST-based reducer can shorten a statement by applying AST level transformations, including removing unnecessary clauses, irrelevant elements in a list, simplify complicated expressions and etc. + +The transformations are implemented by [JSQLParser](https://github.com/JSQLParser/JSqlParser), a RDBMS agnostic SQL statement parser that can translate SQL statements into a traversable hierarchy of Java classes. JSQLParser provides support for the SQL standard as well as major SQL dialects. The AST-based reducer works for any SQL dialects that can be parsed by this tool. + +## Enable reducers +Test-case reduction is disabled by default. The statement reducer can be enabled by passing `--use-reducer` when starting SQLancer. If you wish to further shorten each statements, you need to additionally pass the `--reduce-AST` parameter so that the AST-based reduction is applied. + +Note: if `--reduce-AST` is set, `--use-reducer` option must be enabled first. + +There are also options to define timeout seconds and max steps of reduction for both statement reducer and AST-based reducer. + +``` +--statement-reducer-max-steps= +--statement-reducer-max-time= +--ast-reducer-max-steps= +--ast-reducer-max-time= +``` + +## Reduction logs +If test-case reduction is enabled, each time the reducer performs a reduction step successfully,it prints the reduced statements to the log file, overwriting the previous ones. + +The log files will be stored in the following format: `logs//reduce/-reduce.log`. For instance, if the tested DBMS is SQLite3 and the current database is named database0, the log file will be located at `logs/sqlite3/reduce/database0-reduce.log`. \ No newline at end of file From 706ac16bc554bf096ee049521960f1beff40e212 Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Thu, 24 Aug 2023 16:35:21 +0800 Subject: [PATCH 0816/1284] feat: refactor AST-based reducer (#879) * feat: refactor AST-based reducer + define different kinds of transformers separately, making the reducer extenable. + add options to enable AST Based Reducer * modify the code based on the feedback of the review * add expected value to test cases for AST-based reducer. * support removing columns, clauses for common table expressions * feat: add time-out and max-steps options for AST-based-reducer * feat: add support for simplifying double constant * feat: simplify double constants when using AST-based reducer * refactor: modify the structure of the code to make it appropriate * adjust the format of comments and code * change reduce-AST to reduce-ast --- docs/testCaseReduction.md | 4 +- src/sqlancer/ASTBasedReducer.java | 527 +++--------------- src/sqlancer/Main.java | 10 + src/sqlancer/MainOptions.java | 24 +- .../JSQLParserBasedTransformation.java | 36 ++ .../RemoveClausesOfSelect.java | 102 ++++ .../RemoveColumnsOfSelect.java | 64 +++ .../RemoveElementsOfExpressionList.java | 87 +++ .../transformations/RemoveRowsOfInsert.java | 45 ++ .../transformations/RemoveUnions.java | 52 ++ .../transformations/RoundDoubleConstant.java | 74 +++ .../transformations/SimplifyConstant.java | 103 ++++ .../transformations/SimplifyExpressions.java | 98 ++++ .../transformations/Transformation.java | 122 ++++ .../sqlancer/reducer/TestASTBasedReducer.java | 194 ++++++- 15 files changed, 1054 insertions(+), 488 deletions(-) create mode 100644 src/sqlancer/transformations/JSQLParserBasedTransformation.java create mode 100644 src/sqlancer/transformations/RemoveClausesOfSelect.java create mode 100644 src/sqlancer/transformations/RemoveColumnsOfSelect.java create mode 100644 src/sqlancer/transformations/RemoveElementsOfExpressionList.java create mode 100644 src/sqlancer/transformations/RemoveRowsOfInsert.java create mode 100644 src/sqlancer/transformations/RemoveUnions.java create mode 100644 src/sqlancer/transformations/RoundDoubleConstant.java create mode 100644 src/sqlancer/transformations/SimplifyConstant.java create mode 100644 src/sqlancer/transformations/SimplifyExpressions.java create mode 100644 src/sqlancer/transformations/Transformation.java diff --git a/docs/testCaseReduction.md b/docs/testCaseReduction.md index 35aa9cd08..c95770688 100644 --- a/docs/testCaseReduction.md +++ b/docs/testCaseReduction.md @@ -12,9 +12,9 @@ The AST-based reducer can shorten a statement by applying AST level transformati The transformations are implemented by [JSQLParser](https://github.com/JSQLParser/JSqlParser), a RDBMS agnostic SQL statement parser that can translate SQL statements into a traversable hierarchy of Java classes. JSQLParser provides support for the SQL standard as well as major SQL dialects. The AST-based reducer works for any SQL dialects that can be parsed by this tool. ## Enable reducers -Test-case reduction is disabled by default. The statement reducer can be enabled by passing `--use-reducer` when starting SQLancer. If you wish to further shorten each statements, you need to additionally pass the `--reduce-AST` parameter so that the AST-based reduction is applied. +Test-case reduction is disabled by default. The statement reducer can be enabled by passing `--use-reducer` when starting SQLancer. If you wish to further shorten each statements, you need to additionally pass the `--reduce-ast` parameter so that the AST-based reduction is applied. -Note: if `--reduce-AST` is set, `--use-reducer` option must be enabled first. +Note: if `--reduce-ast` is set, `--use-reducer` option must be enabled first. There are also options to define timeout seconds and max steps of reduction for both statement reducer and AST-based reducer. diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java index 705d9d307..36f16c2ac 100644 --- a/src/sqlancer/ASTBasedReducer.java +++ b/src/sqlancer/ASTBasedReducer.java @@ -1,101 +1,22 @@ package sqlancer; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.function.BiConsumer; -import net.sf.jsqlparser.expression.BinaryExpression; -import net.sf.jsqlparser.expression.CaseExpression; -import net.sf.jsqlparser.expression.DoubleValue; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; -import net.sf.jsqlparser.expression.LongValue; -import net.sf.jsqlparser.expression.NullValue; -import net.sf.jsqlparser.expression.Parenthesis; -import net.sf.jsqlparser.expression.StringValue; -import net.sf.jsqlparser.expression.WhenClause; -import net.sf.jsqlparser.expression.operators.relational.Between; -import net.sf.jsqlparser.expression.operators.relational.ExpressionList; -import net.sf.jsqlparser.expression.operators.relational.InExpression; -import net.sf.jsqlparser.expression.operators.relational.ItemsList; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.Statement; -import net.sf.jsqlparser.statement.StatementVisitorAdapter; -import net.sf.jsqlparser.statement.select.GroupByElement; -import net.sf.jsqlparser.statement.select.PlainSelect; -import net.sf.jsqlparser.statement.select.Select; -import net.sf.jsqlparser.statement.select.SelectBody; -import net.sf.jsqlparser.statement.select.SelectExpressionItem; -import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; -import net.sf.jsqlparser.statement.select.SetOperationList; -import net.sf.jsqlparser.statement.select.SubSelect; -import net.sf.jsqlparser.statement.select.WithItem; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.transformations.RemoveClausesOfSelect; +import sqlancer.transformations.RemoveColumnsOfSelect; +import sqlancer.transformations.RemoveElementsOfExpressionList; +import sqlancer.transformations.RemoveRowsOfInsert; +import sqlancer.transformations.RemoveUnions; +import sqlancer.transformations.RoundDoubleConstant; +import sqlancer.transformations.SimplifyConstant; +import sqlancer.transformations.SimplifyExpressions; +import sqlancer.transformations.Transformation; -final class ExpressionTransformer { - - private static List flattenChildren(BinaryExpression expr) { - List candidates = new ArrayList<>(); - Expression lhs = expr.getLeftExpression(); - Expression rhs = expr.getRightExpression(); - candidates.add(lhs); - candidates.add(rhs); - return candidates; - } - - private static List flattenChildren(Between expr) { - Expression lhs = expr.getBetweenExpressionStart(); - Expression rhs = expr.getBetweenExpressionEnd(); - return List.of(lhs, rhs); - } - - public static List candidateExpressions(Expression expr) { - if (expr instanceof Parenthesis) { - // try removing a pair of brackets. - Parenthesis paren = (Parenthesis) expr; - return List.of(paren.getExpression()); - } else if (expr instanceof BinaryExpression) { - return flattenChildren((BinaryExpression) expr); - } else if (expr instanceof Between) { - return flattenChildren((Between) expr); - } else if (expr instanceof LongValue) { - LongValue longValue = (LongValue) expr; - if (String.valueOf(longValue).length() >= 4) { - return List.of(new NullValue(), new LongValue(10), new LongValue(0), new LongValue(1)); - } - return new ArrayList<>(); - } else if (expr instanceof DoubleValue) { - DoubleValue doubleValue = (DoubleValue) expr; - double literal = doubleValue.getValue(); - if (String.valueOf(literal).length() <= 4) { - return new ArrayList<>(); - } - double roundedValue = Math.round(literal * 10.0) / 10.0; - return List.of(new NullValue(), new DoubleValue(String.valueOf(roundedValue))); - } else if (expr instanceof StringValue) { - StringValue sv = (StringValue) expr; - String str = sv.getValue(); - if (str.length() > 4) { - return List.of(new NullValue(), new StringValue(" ")); - } - return new ArrayList<>(); - } else if (expr instanceof CaseExpression) { - CaseExpression caseExpression = (CaseExpression) expr; - return List.of(caseExpression.getSwitchExpression(), caseExpression.getElseExpression()); - } else { - return new ArrayList<>(); - } - } - - private ExpressionTransformer() throws Exception { - throw new AssertionError("Do not initialize the util class"); - } -} - -@SuppressWarnings("unchecked") public class ASTBasedReducer, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> implements Reducer { @@ -105,406 +26,97 @@ public class ASTBasedReducer, O extends DBMSSpeci private G state; private G newGlobalState; private Reproducer reproducer; - private int reduceTargetIndex; - private Statement targetStatement; - // statement after reduction. private List> reducedStatements; + // statement after reduction. public ASTBasedReducer(DatabaseProvider provider) { this.provider = provider; } - private void updateStatements() { - String queryString = targetStatement.toString(); + @SuppressWarnings("unchecked") + private void updateStatements(String queryString, int index) { boolean couldAffectSchema = queryString.contains("CREATE TABLE") || queryString.contains("EXPLAIN"); - reducedStatements.set(reduceTargetIndex, (Query) new SQLQueryAdapter(queryString, couldAffectSchema)); - } - - public

void expressionReduce(P parent, Expression subExpr, // NOPMD - BiConsumer setter) { - boolean observeChange; - do { - observeChange = false; - List candidates = ExpressionTransformer.candidateExpressions(subExpr); - for (Expression candidate : candidates) { - try { - setter.accept(parent, candidate); - if (bugStillTriggers()) { - subExpr = candidate; - observeChange = true; - } - } catch (Exception ignoredException) { - } - } - setter.accept(parent, subExpr); - } while (observeChange); + reducedStatements.set(index, (Query) new SQLQueryAdapter(queryString, couldAffectSchema)); } - public void listElementRemovingReduce(P parent, List elms, // NOPMD - BiConsumer> setter) { - // TODO: For AST-Reducer, is delta-debugging needed ? Or just use the naive approach ? - boolean observeChange; - do { - observeChange = false; - for (int i = elms.size() - 1; i >= 0; i--) { - List reducedElms = new ArrayList<>(elms); - reducedElms.subList(i, i + 1).clear(); - setter.accept(parent, reducedElms); - try { - if (bugStillTriggers()) { - elms = reducedElms; - observeChange = true; - } - } catch (Exception e) { - System.out.println("An error occurred when trying executing reduced statements"); - e.printStackTrace(); - } - } - setter.accept(parent, elms); - } while (observeChange); - - } - - ExpressionVisitorAdapter expressionReducerVisitor = new ExpressionVisitorAdapter() { - - @Override - public void visit(InExpression expr) { - Expression rhs = expr.getRightExpression(); - - if (rhs instanceof SubSelect) { - SubSelect subSelect = (SubSelect) rhs; - subSelect.getSelectBody().accept(selectReducerVisitor); - } else { - ItemsList itemslist = expr.getRightItemsList(); - itemslist.accept(this); - } - } - - @Override - protected void visitBinaryExpression(BinaryExpression expr) { - Expression lhs = expr.getLeftExpression(); - Expression rhs = expr.getRightExpression(); - expressionReduce(expr, lhs, (expression, candidate) -> { - expression.setLeftExpression(candidate); - updateStatements(); - }); - expressionReduce(expr, rhs, (expression, candidate) -> { - expression.setRightExpression(candidate); - updateStatements(); - }); - lhs.accept(this); - rhs.accept(this); - } - - // @Override - // public void visit(DateValue value) { - // super.visit(value); - // } - // - // @Override - // public void visit(TimeValue value) { - // super.visit(value); - // } - // - // @Override - // public void visit(LikeExpression expr) { - // super.visit(expr); - // } - - @Override - public void visit(CaseExpression expr) { - Expression switchExpr = expr.getSwitchExpression(); - Expression elseExpr = expr.getElseExpression(); - - expressionReduce(expr, switchExpr, (parent, sw) -> { - parent.setSwitchExpression(sw); - updateStatements(); - }); - expressionReduce(expr, elseExpr, (parent, els) -> { - parent.setElseExpression(els); - updateStatements(); - }); - - super.visit(expr); - } - - @Override - public void visit(WhenClause whenClause) { - Expression when = whenClause.getWhenExpression(); - Expression then = whenClause.getThenExpression(); - - expressionReduce(whenClause, when, (wc, w) -> { - wc.setWhenExpression(w); - updateStatements(); - }); + @SuppressWarnings("unchecked") + @Override + public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + this.state = state; + this.newGlobalState = newGlobalState; + this.reproducer = reproducer; - expressionReduce(whenClause, then, (wc, t) -> { - wc.setThenExpression(t); - updateStatements(); - }); + long maxReduceTime = state.getOptions().getMaxStatementReduceTime(); + long maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); - super.visit(whenClause); - } + List> initialBugInducingStatements = state.getState().getStatements(); + newGlobalState.getState().setStatements(new ArrayList<>(initialBugInducingStatements)); - @Override - public void visit(Parenthesis parenthesis) { - Expression closedExpr = parenthesis.getExpression(); - expressionReduce(parenthesis, closedExpr, (p, s) -> { - p.setExpression(s); - updateStatements(); - }); - closedExpr.accept(this); - } + List transformations = new ArrayList<>(); - // @Override - // public void visit(Function function) { - // super.visit(function); - // } + transformations.add(new RemoveUnions()); + transformations.add(new RemoveClausesOfSelect()); + transformations.add(new RemoveRowsOfInsert()); + transformations.add(new RemoveColumnsOfSelect()); + transformations.add(new RemoveElementsOfExpressionList()); + transformations.add(new SimplifyExpressions()); + transformations.add(new SimplifyConstant()); + transformations.add(new RoundDoubleConstant()); - @Override - public void visit(ExpressionList expressionList) { - List expressions = expressionList.getExpressions(); - listElementRemovingReduce(expressionList, expressions, (l, es) -> { - l.setExpressions(es); - updateStatements(); - }); - expressions = expressionList.getExpressions(); - for (int i = 0; i < expressions.size(); i++) { - Expression expr = expressions.get(i); - int index = i; - expressionReduce(expressions, expr, (l, e) -> { - l.set(index, e); - updateStatements(); - }); + Transformation.setBugJudgement(() -> { + try { + return this.bugStillTriggers(); + } catch (Exception ignored) { } - super.visit(expressionList); - } - - @Override - public void visit(SelectExpressionItem selectExpressionItem) { - Expression expr = selectExpressionItem.getExpression(); - expressionReduce(selectExpressionItem, expr, (item, e) -> { - item.setExpression(e); - updateStatements(); - }); - super.visit(selectExpressionItem); - } - - @Override - public void visit(SubSelect subSelect) { - subSelect.getSelectBody().accept(selectReducerVisitor); - } - - }; - - SelectVisitorAdapter selectReducerVisitor = new SelectVisitorAdapter() { - - // Clauses that would be tried removing. - // examples: - // Remove when: select * from table when 1 -> select * from table - // Remove limit: select * from table limit 1 -> select * from table - private final String[] removeList = { "Limit", "Offset", "Where", "Having", "GroupBy", "Distinct", - "OrderByElements", "Joins" }; + return false; + }); - // Clauses that would be tried transforming. - // examples: - // select * from t where a + b < c + d - // where clause might become one of the statement below after transformation: - // -> select * from table where c + d - // -> select * from table where a + b - - private final String[] transformList = { "Where", "Having", "FromItem", "SelectItems", "GroupBy", "Joins" }; - - // Clauses that would be visited for further reduction. - // example: - // select * from t where a + b < c + d - // Assuming that the coexistence of a and c would trigger the bug. - // The where clause : a + b < c + d would be visited and a + b, c + d would be reduced respectively. - // a + b < c + d might become a + c - private final String[] descendList = { "Where", "Having", "FromItem", "SelectItems", "GroupBy" }; - - private String getterName(String astNodeName) { - return "get" + astNodeName; - } - - private String setterName(String astNodeName) { - if (astNodeName.equals("GroupBy")) { - return "set" + astNodeName + "Element"; - } else { - return "set" + astNodeName; - } + boolean observeChange; + reducedStatements = new ArrayList<>(); + for (Query query : initialBugInducingStatements) { + reducedStatements.add((Query) query); } - @Override - public void visit(WithItem withItem) { - // withItem.getItemsList(); - } + Instant startTime = Instant.now(); + reduceProcess: do { + observeChange = false; + for (Transformation t : transformations) { + for (int i = 0; i < reducedStatements.size(); i++) { - @Override - public void visit(PlainSelect plainSelect) { - // transform section. Lists defined above would be iterated to get the corresponding clause name. Reflection - // is used to avoid repetitive code. e.g. The current astNodeName is When `getWhen`, `setWhen` would be - // called. - for (String astNodeName : removeList) { - try { - Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); - Object astNode = nodeGetter.invoke(plainSelect); - if (astNode == null) { - continue; + Instant currentTime = Instant.now(); + if (maxReduceTime != MainOptions.NO_REDUCE_LIMIT + && Duration.between(startTime, currentTime).getSeconds() >= maxReduceTime) { + break reduceProcess; } - Method nodeSetter = plainSelect.getClass().getMethod(setterName(astNodeName), - nodeGetter.getReturnType()); - nodeSetter.invoke(plainSelect, new Object[] { null }); - updateStatements(); - if (!bugStillTriggers()) { - nodeSetter.invoke(plainSelect, astNode); - updateStatements(); - } - } catch (Exception e) { - throw new AssertionError(e); - } - } - // Pull Up Section - for (String astNodeName : transformList) { - try { - Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); - Object astNode = nodeGetter.invoke(plainSelect); - if (astNode == null) { - continue; + if (maxReduceSteps != MainOptions.NO_REDUCE_LIMIT + && Transformation.getReduceSteps() >= maxReduceSteps) { + break reduceProcess; } - Method nodeSetter = plainSelect.getClass().getMethod(setterName(astNodeName), - nodeGetter.getReturnType()); - if (astNode instanceof Expression) { - Expression expr = (Expression) astNode; - expressionReduce(plainSelect, expr, (select, expression) -> { - try { - nodeSetter.invoke(select, expression); - updateStatements(); - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - } - }); - } else if (astNode instanceof List) { - List elms = (List) astNode; - if (elms.size() <= 1) { - continue; - } - listElementRemovingReduce(plainSelect, elms, (select, items) -> { - try { - nodeSetter.invoke(select, items); - updateStatements(); - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - } - }); - } else if (astNode instanceof GroupByElement) { - GroupByElement groupByElement = (GroupByElement) astNode; - ExpressionList expressionList = groupByElement.getGroupByExpressionList(); - if (expressionList == null) { - groupByElement.getGroupingSets(); - // TODO: TO BE IMPLEMENTED. - } else { - List elms = expressionList.getExpressions(); - if (elms.size() <= 1) { - continue; - } - listElementRemovingReduce(groupByElement, elms, (select, items) -> { - groupByElement.setGroupByExpressionList(new ExpressionList(items)); - updateStatements(); - }); - } - } - } catch (Exception e) { - throw new AssertionError(e); - } - } + Query query = reducedStatements.get(i); + boolean initFlag = t.init(query.getQueryString()); + int index = i; + t.setStatementChangedCallBack((statementString) -> { + updateStatements(statementString, index); + }); - for (String astNodeName : descendList) { - try { - Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); - Object astNode = nodeGetter.invoke(plainSelect); - if (astNode == null) { + if (!initFlag) { + System.out.println("Error when parsing the statement at transformer :" + t); continue; } - if (astNode instanceof Expression) { - ((Expression) astNode).accept(expressionReducerVisitor); - } else if (astNode instanceof List) { - // Really hacky... Some other ways to simplify it ? - List elms = (List) astNode; - for (Object obj : elms) { - if (obj instanceof SelectExpressionItem) { - ((SelectExpressionItem) obj).accept(expressionReducerVisitor); - } else if (obj instanceof Expression) { - ((Expression) obj).accept(expressionReducerVisitor); - } - } - } else if (astNode instanceof GroupByElement) { - GroupByElement groupByElement = (GroupByElement) astNode; - ExpressionList expressionList = groupByElement.getGroupByExpressionList(); - if (expressionList != null) { - expressionList.accept(expressionReducerVisitor); - } - // TODO: groupByElement.getGroupingSets() TO BE IMPLEMENTED - - } - } catch (Exception e) { - throw new AssertionError(e); - } - } - } - - @Override - public void visit(SetOperationList setOpList) { - List selectBodies = setOpList.getSelects(); - listElementRemovingReduce(setOpList, selectBodies, (optionList, selects) -> { - optionList.setSelects(selects); - updateStatements(); - }); - for (SelectBody selectBody : selectBodies) { - if (selectBody instanceof PlainSelect) { - visit((PlainSelect) selectBody); + t.apply(); + observeChange |= t.changed(); } } - } - - }; - - StatementVisitorAdapter statementReducerVisitor = new StatementVisitorAdapter() { - @Override - public void visit(Select select) { - SelectBody selectBody = select.getSelectBody(); - if (selectBody != null) { - selectBody.accept(selectReducerVisitor); - } - } - }; - - @Override - public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { - this.state = state; - this.newGlobalState = newGlobalState; - this.reproducer = reproducer; - - List> initialBugInducingStatements = state.getState().getStatements(); - reducedStatements = new ArrayList<>(); - for (Query query : initialBugInducingStatements) { - reducedStatements.add((Query) query); - } - - for (int i = 0; i < reducedStatements.size(); i++) { - reduceTargetIndex = i; - Query query = reducedStatements.get(reduceTargetIndex); - targetStatement = CCJSqlParserUtil.parse(query.getQueryString()); - targetStatement.accept(statementReducerVisitor); - } + } while (observeChange); newGlobalState.getState().setStatements(new ArrayList<>(reducedStatements)); + newGlobalState.getLogger().logReduced(newGlobalState.getState()); } - private boolean bugStillTriggers() throws Exception { + public boolean bugStillTriggers() throws Exception { try (C con2 = provider.createDatabase(newGlobalState)) { newGlobalState.setConnection(con2); List> candidateStatements = new ArrayList<>(reducedStatements); @@ -519,6 +131,7 @@ private boolean bugStillTriggers() throws Exception { } try { if (reproducer.bugStillTriggers(newGlobalState)) { + newGlobalState.getLogger().logReduced(newGlobalState.getState()); return true; } } catch (Throwable ignoredException) { diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 70657c3d6..4b480e595 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -444,6 +444,10 @@ public void run() throws Exception { } catch (IOException e) { throw new AssertionError(e); } + + if (options.reduceAST() && !options.useReducer()) { + throw new AssertionError("To reduce AST, use-reducer option must be enabled first"); + } if (reproducer != null && options.useReducer()) { System.out.println("EXPERIMENTAL: Trying to reduce queries using a simple reducer."); // System.out.println("Reduced query will be output to stdout but not logs."); @@ -459,6 +463,12 @@ public void run() throws Exception { Reducer reducer = new StatementReducer<>(provider); reducer.reduce(state, reproducer, newGlobalState); + + if (options.reduceAST()) { + Reducer astBasedReducer = new ASTBasedReducer<>(provider); + astBasedReducer.reduce(state, reproducer, newGlobalState); + } + throw new AssertionError("Found a potential bug"); } } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index c653a358f..232856f94 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -126,10 +126,19 @@ public class MainOptions { @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") private boolean useReducer = false; // NOPMD + @Parameter(names = "--reduce-ast", description = "EXPERIMENTAL perform AST reduction after statement reduction") + private boolean reduceAST = false; // NOPMD + @Parameter(names = "--statement-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the statement reducer will do") private long maxStatementReduceSteps = NO_REDUCE_LIMIT; // NOPMD - @Parameter(names = "--statement-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the statement reducer will do") + @Parameter(names = "--statement-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the AST-based reducer will do") + private long maxASTReduceTime = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--ast-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the AST-based reducer will do") + private long maxASTReduceSteps = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--ast-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the statement reducer will do") private long maxStatementReduceTime = NO_REDUCE_LIMIT; // NOPMD public int getMaxExpressionDepth() { @@ -293,6 +302,10 @@ public boolean useReducer() { return useReducer; } + public boolean reduceAST() { + return reduceAST; + } + public long getMaxStatementReduceSteps() { return maxStatementReduceSteps; } @@ -300,4 +313,13 @@ public long getMaxStatementReduceSteps() { public long getMaxStatementReduceTime() { return maxStatementReduceTime; } + + public long getMaxASTReduceSteps() { + return maxASTReduceSteps; + } + + public long getMaxASTReduceTime() { + return maxASTReduceTime; + } + } diff --git a/src/sqlancer/transformations/JSQLParserBasedTransformation.java b/src/sqlancer/transformations/JSQLParserBasedTransformation.java new file mode 100644 index 000000000..e7b7b6145 --- /dev/null +++ b/src/sqlancer/transformations/JSQLParserBasedTransformation.java @@ -0,0 +1,36 @@ +package sqlancer.transformations; + +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; + +/** + * Transformations based on JSQLParser should be derived from this class. + */ + +public class JSQLParserBasedTransformation extends Transformation { + + protected Statement statement; + + public JSQLParserBasedTransformation(String desc) { + super(desc); + } + + @Override + protected void onStatementChanged() { + if (statementChangedHandler != null) { + statementChangedHandler.accept(this.statement.toString()); + } + } + + @Override + public boolean init(String sql) { + this.current = sql; + try { + statement = CCJSqlParserUtil.parse(current); + } catch (Exception e) { + return false; + } + return true; + } + +} diff --git a/src/sqlancer/transformations/RemoveClausesOfSelect.java b/src/sqlancer/transformations/RemoveClausesOfSelect.java new file mode 100644 index 000000000..832f82762 --- /dev/null +++ b/src/sqlancer/transformations/RemoveClausesOfSelect.java @@ -0,0 +1,102 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.statement.select.Distinct; +import net.sf.jsqlparser.statement.select.GroupByElement; +import net.sf.jsqlparser.statement.select.Limit; +import net.sf.jsqlparser.statement.select.Offset; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SubSelect; +import net.sf.jsqlparser.statement.select.WithItem; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * remove clauses of a select, such as join, where, group by, distinct, offset, limit. + * + * e.g. select * from t where a = b offset 1 limit 1 -> select * from t; + */ + +public class RemoveClausesOfSelect extends JSQLParserBasedTransformation { + private final SelectDeParser remover = new SelectDeParser() { + @Override + public void visit(PlainSelect plainSelect) { + handleSelect(plainSelect); + super.visit(plainSelect); + } + }; + + public RemoveClausesOfSelect() { + super("remove clauses of select"); + } + + @Override + public boolean init(String original) { + + boolean baseSuc = super.init(original); + if (!baseSuc) { + return false; + } + + this.remover.setExpressionVisitor(new ExpressionDeParser(remover, new StringBuilder())); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(remover); + + List withItemsList = select.getWithItemsList(); + if (withItemsList == null) { + return; + } + tryRemoveElms(select, withItemsList, Select::setWithItemsList); + + for (WithItem withItem : withItemsList) { + SubSelect subSelect = withItem.getSubSelect(); + if (subSelect == null) { + return; + } + + if (subSelect.getSelectBody() != null) { + subSelect.getSelectBody().accept(remover); + } + } + } + } + + private void handleSelect(PlainSelect plainSelect) { + + Expression where = plainSelect.getWhere(); + if (where != null) { + tryRemove(plainSelect, where, PlainSelect::setWhere); + } + + GroupByElement groupByElement = plainSelect.getGroupBy(); + if (groupByElement != null) { + tryRemove(plainSelect, groupByElement, PlainSelect::setGroupByElement); + } + + Distinct distinct = plainSelect.getDistinct(); + if (distinct != null) { + tryRemove(plainSelect, distinct, PlainSelect::setDistinct); + } + + Offset offset = plainSelect.getOffset(); + if (offset != null) { + tryRemove(plainSelect, offset, PlainSelect::setOffset); + } + + Limit limit = plainSelect.getLimit(); + if (offset != null) { + tryRemove(plainSelect, limit, PlainSelect::setLimit); + } + } + +} diff --git a/src/sqlancer/transformations/RemoveColumnsOfSelect.java b/src/sqlancer/transformations/RemoveColumnsOfSelect.java new file mode 100644 index 000000000..6cdd44d15 --- /dev/null +++ b/src/sqlancer/transformations/RemoveColumnsOfSelect.java @@ -0,0 +1,64 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SubSelect; +import net.sf.jsqlparser.statement.select.WithItem; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * remove columns of a select: e.g. select a, b, c from t -> select a from t. + */ +public class RemoveColumnsOfSelect extends JSQLParserBasedTransformation { + + private final SelectDeParser remover = new SelectDeParser() { + @Override + public void visit(PlainSelect plainSelect) { + tryRemoveElms(plainSelect, plainSelect.getSelectItems(), PlainSelect::setSelectItems); + super.visit(plainSelect); + } + }; + + public RemoveColumnsOfSelect() { + super("remove columns of a select"); + } + + @Override + public boolean init(String original) { + + boolean baseSucc = super.init(original); + if (!baseSucc) { + return false; + } + this.remover.setExpressionVisitor(new ExpressionDeParser(remover, new StringBuilder())); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(remover); + + List withItemsList = select.getWithItemsList(); + if (withItemsList == null) { + return; + } + for (WithItem withItem : withItemsList) { + SubSelect subSelect = withItem.getSubSelect(); + if (subSelect == null) { + return; + } + + if (subSelect.getSelectBody() != null) { + subSelect.getSelectBody().accept(remover); + } + } + + } + } +} diff --git a/src/sqlancer/transformations/RemoveElementsOfExpressionList.java b/src/sqlancer/transformations/RemoveElementsOfExpressionList.java new file mode 100644 index 000000000..ebeb0b0af --- /dev/null +++ b/src/sqlancer/transformations/RemoveElementsOfExpressionList.java @@ -0,0 +1,87 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.statement.select.GroupByElement; +import net.sf.jsqlparser.statement.select.Join; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.InsertDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * remove elements of an expression list. + * + * NOTE: this only works for select statements and targets at ExpressionList type in JSQLParser, such as groupBy list + */ +public class RemoveElementsOfExpressionList extends JSQLParserBasedTransformation { + private final ExpressionDeParser expressionHandler = new ExpressionDeParser(); + private final SelectDeParser simplifier = new SelectDeParser() { + @Override + public void visit(PlainSelect plainSelect) { + handleSelect(plainSelect); + super.visit(plainSelect); + } + + @Override + public void visit(ExpressionList expressionList) { + List expressions = expressionList.getExpressions(); + tryRemoveElms(expressionList, expressions, ExpressionList::setExpressions); + super.visit(expressionList); + } + }; + private final InsertDeParser insertDeParser = new InsertDeParser() { + @Override + public void visit(ExpressionList expressionList) { + List expressions = expressionList.getExpressions(); + tryRemoveElms(expressionList, expressions, ExpressionList::setExpressions); + super.visit(expressionList); + } + }; + + public RemoveElementsOfExpressionList() { + super("remove elements of expression lists"); + } + + @Override + public boolean init(String sql) { + boolean baseSuc = super.init(sql); + if (!baseSuc) { + return false; + } + this.simplifier.setExpressionVisitor(expressionHandler); + this.expressionHandler.setSelectVisitor(simplifier); + + this.insertDeParser.setExpressionVisitor(expressionHandler); + this.insertDeParser.setSelectVisitor(simplifier); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(simplifier); + } + } + + private void handleSelect(PlainSelect plainSelect) { + + GroupByElement groupByElement = plainSelect.getGroupBy(); + + if (groupByElement != null && groupByElement.getGroupByExpressionList() != null) { + ExpressionList expressionList = groupByElement.getGroupByExpressionList(); + List list = expressionList.getExpressions(); + tryRemoveElms(expressionList, list, ExpressionList::setExpressions); + } + + List expressionList = plainSelect.getJoins(); + if (expressionList != null) { + tryRemoveElms(plainSelect, expressionList, PlainSelect::setJoins); + } + } +} diff --git a/src/sqlancer/transformations/RemoveRowsOfInsert.java b/src/sqlancer/transformations/RemoveRowsOfInsert.java new file mode 100644 index 000000000..b49f80149 --- /dev/null +++ b/src/sqlancer/transformations/RemoveRowsOfInsert.java @@ -0,0 +1,45 @@ +package sqlancer.transformations; + +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.expression.operators.relational.ItemsList; +import net.sf.jsqlparser.statement.insert.Insert; +import net.sf.jsqlparser.statement.select.SelectBody; +import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.values.ValuesStatement; + +/** + * This Transformer remove rows of insert. Given a sql statement: + * + * INSERT INTO t1(c2, c0) VALUES (1508438260, 2929), (1508438260, TIMESTAMP '1969-12-26 01:57:21'), (0.5347171705591047, + * 398662142); -> INSERT INTO t1 (c2, c0) VALUES (0.5347171705591047, 398662142); + */ +public class RemoveRowsOfInsert extends JSQLParserBasedTransformation { + public RemoveRowsOfInsert() { + super("remove rows of an insert statement"); + } + + @Override + public void apply() { + super.apply(); + if (!(statement instanceof Insert)) { + return; + } + SelectBody selectBody = ((Insert) statement).getSelect().getSelectBody(); + if (!(selectBody instanceof SetOperationList)) { + return; + } + SetOperationList insertingList = (SetOperationList) selectBody; + for (SelectBody selBody : insertingList.getSelects()) { + if (!(selBody instanceof ValuesStatement)) { + continue; + } + ValuesStatement valuesStatement = (ValuesStatement) selBody; + ItemsList itemsList = valuesStatement.getExpressions(); + if (!(itemsList instanceof ExpressionList)) { + continue; + } + tryRemoveElms((ExpressionList) itemsList, ((ExpressionList) itemsList).getExpressions(), + ExpressionList::setExpressions); + } + } +} diff --git a/src/sqlancer/transformations/RemoveUnions.java b/src/sqlancer/transformations/RemoveUnions.java new file mode 100644 index 000000000..4f9593345 --- /dev/null +++ b/src/sqlancer/transformations/RemoveUnions.java @@ -0,0 +1,52 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectBody; +import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * try removing sub selects of a union statement. + * + * e.g. select 1 union select 2 -> select 1 + */ + +public class RemoveUnions extends JSQLParserBasedTransformation { + + private final SelectDeParser remover = new SelectDeParser() { + @Override + public void visit(SetOperationList list) { + List selectBodyList = list.getSelects(); + tryRemoveElms(list, selectBodyList, SetOperationList::setSelects); + super.visit(list); + } + }; + + public RemoveUnions() { + super("remove union selects"); + } + + @Override + public boolean init(String sql) { + + boolean baseSuc = super.init(sql); + if (!baseSuc) { + return false; + } + + this.remover.setExpressionVisitor(new ExpressionDeParser(remover, new StringBuilder())); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(remover); + } + } +} diff --git a/src/sqlancer/transformations/RoundDoubleConstant.java b/src/sqlancer/transformations/RoundDoubleConstant.java new file mode 100644 index 000000000..d22496c4e --- /dev/null +++ b/src/sqlancer/transformations/RoundDoubleConstant.java @@ -0,0 +1,74 @@ +package sqlancer.transformations; + +import java.text.DecimalFormat; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Round double values which are longer than a certain length. e.g. 2.4782565267 -> 2.478. + * + * This transformation is not based on JSQLParser. + */ +public class RoundDoubleConstant extends Transformation { + + private Set doubleValueCollector; + + private String currentString; + + private static final int ROUND_LENGTH = 3; + private DecimalFormat decimalFormat; + + public RoundDoubleConstant() { + super("round double constant values"); + } + + @Override + public boolean init(String sql) { + super.init(sql); + decimalFormat = new DecimalFormat("#." + "#".repeat(ROUND_LENGTH)); + + currentString = sql; + doubleValueCollector = new HashSet<>(); + + String regex = "\\b-?\\d+\\.\\d+\\b"; + + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(sql); + + while (matcher.find()) { + String matchedText = matcher.group(); + String decimalPart = matchedText.replaceAll("\\d+\\.", ""); + int decimalPlaces = decimalPart.length(); + if (decimalPlaces > ROUND_LENGTH) { + doubleValueCollector.add(matchedText); + } + } + return true; + } + + @Override + public void apply() { + for (String doubleValue : doubleValueCollector) { + + double targetNumber = Double.parseDouble(doubleValue); + String roundedNumberStr = decimalFormat.format(targetNumber); + + String replacement = currentString.replace(doubleValue, roundedNumberStr); + String original = currentString; + + tryReplace(null, original, replacement, (p, r) -> { + currentString = r; + }); + } + super.apply(); + } + + @Override + protected void onStatementChanged() { + if (statementChangedHandler != null) { + statementChangedHandler.accept(currentString); + } + } +} diff --git a/src/sqlancer/transformations/SimplifyConstant.java b/src/sqlancer/transformations/SimplifyConstant.java new file mode 100644 index 000000000..6638dca73 --- /dev/null +++ b/src/sqlancer/transformations/SimplifyConstant.java @@ -0,0 +1,103 @@ +package sqlancer.transformations; + +import java.util.ArrayList; +import java.util.List; + +import net.sf.jsqlparser.expression.DoubleValue; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.statement.StatementVisitorAdapter; +import net.sf.jsqlparser.statement.insert.Insert; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * Shorten the constant of a statement e.g. "a_very_long_str" -> "_", 12341234->1. + * + * Note: The API of JSQLParser may have some problems with double values: `setValue` can't change the literal value of a + * DoubleValue object. Therefore, double values are handled at RoundDoubleConstant class. + */ +public class SimplifyConstant extends JSQLParserBasedTransformation { + static class ConstantCollector extends ExpressionDeParser { + private final List candidates = new ArrayList<>(); + + @Override + public void visit(DoubleValue doubleValue) { + candidates.add(doubleValue); + super.visit(doubleValue); + } + + @Override + public void visit(LongValue longValue) { + candidates.add(longValue); + super.visit(longValue); + } + + @Override + public void visit(StringValue stringValue) { + candidates.add(stringValue); + super.visit(stringValue); + } + + public List getCandidates() { + return candidates; + } + } + + public SimplifyConstant() { + super("simplify constant expressions"); + } + + @Override + public void apply() { + super.apply(); + ConstantCollector collector = new ConstantCollector(); + StringBuilder buffer = new StringBuilder(); + SelectDeParser collectorDeParser = new SelectDeParser(collector, buffer); + collector.setSelectVisitor(collectorDeParser); + collector.setBuffer(buffer); + + List candidates = collector.getCandidates(); + + StatementVisitorAdapter statementVisitor = new StatementVisitorAdapter() { + @Override + public void visit(Insert insert) { + insert.getSelect().getSelectBody().accept(collectorDeParser); + super.visit(insert); + } + + @Override + public void visit(Select select) { + select.getSelectBody().accept(collectorDeParser); + super.visit(select); + } + }; + + statement.accept(statementVisitor); + + for (Expression e : candidates) { + if (e instanceof LongValue) { + simplify((LongValue) e); + } else if (e instanceof StringValue) { + simplify((StringValue) e); + } + } + } + + private void simplify(LongValue longValue) { + long variant = 0; + if (!longValue.getStringValue().equals(String.valueOf(variant))) { + tryReplace(longValue, longValue.getStringValue(), String.valueOf(variant), LongValue::setStringValue); + } + } + + private void simplify(StringValue stringValue) { + String variant = "_"; + if (!stringValue.getValue().equals(variant)) { + tryReplace(stringValue, stringValue.getValue(), variant, StringValue::setValue); + } + } + +} diff --git a/src/sqlancer/transformations/SimplifyExpressions.java b/src/sqlancer/transformations/SimplifyExpressions.java new file mode 100644 index 000000000..244629b4d --- /dev/null +++ b/src/sqlancer/transformations/SimplifyExpressions.java @@ -0,0 +1,98 @@ +package sqlancer.transformations; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; + +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * This transformation simplifies complicated expressions e.g: a + (b + c) -> b. + */ + +public class SimplifyExpressions extends JSQLParserBasedTransformation { + public SimplifyExpressions() { + super("simplify expressions. e.g. a + b -> a"); + } + + @Override + public boolean init(String sql) { + boolean baseSuc = super.init(sql); + if (!baseSuc) { + return false; + } + this.simplifier.setExpressionVisitor(expressionHandler); + this.expressionHandler.setSelectVisitor(simplifier); + return true; + } + + private final ExpressionDeParser expressionHandler = new ExpressionDeParser() { + @Override + protected void visitBinaryExpression(BinaryExpression binaryExpression, String operator) { + + Expression lhs = binaryExpression.getLeftExpression(); + Expression rhs = binaryExpression.getRightExpression(); + + handleExpression(binaryExpression, lhs, BinaryExpression::setLeftExpression); + handleExpression(binaryExpression, rhs, BinaryExpression::setRightExpression); + + super.visitBinaryExpression(binaryExpression, operator); + } + + }; + private final SelectDeParser simplifier = new SelectDeParser() { + + @Override + public void visit(PlainSelect plainSelect) { + handleSelect(plainSelect); + super.visit(plainSelect); + } + }; + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(simplifier); + } + } + + private void handleSelect(PlainSelect plainSelect) { + Expression where = plainSelect.getWhere(); + if (where != null) { + handleExpression(plainSelect, where, PlainSelect::setWhere); + } + Expression having = plainSelect.getHaving(); + if (having != null) { + handleExpression(plainSelect, having, PlainSelect::setHaving); + } + } + + private List flattenExpression(Expression expression) { + if (expression instanceof BinaryExpression) { + BinaryExpression binaryExpression = (BinaryExpression) expression; + return List.of(binaryExpression.getLeftExpression(), binaryExpression.getRightExpression()); + } else if (expression instanceof Parenthesis) { + return List.of(((Parenthesis) expression).getExpression()); + } + return new ArrayList<>(); + } + + private

void handleExpression(P parent, Expression expr, BiConsumer setter) { + + List expressions = flattenExpression(expr); + for (Expression variant : expressions) { + boolean suc = tryReplace(parent, expr, variant, setter); + if (suc) { + break; + } + } + } +} diff --git a/src/sqlancer/transformations/Transformation.java b/src/sqlancer/transformations/Transformation.java new file mode 100644 index 000000000..affa4c281 --- /dev/null +++ b/src/sqlancer/transformations/Transformation.java @@ -0,0 +1,122 @@ +package sqlancer.transformations; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * The base class of transformations. Defines APIs to remove, replace, remove elements of a list. + */ +public class Transformation { + + private static Supplier bugJudgement; + private static long reduceSteps; + + protected boolean isChanged; + protected String current; + protected String desc = ""; + + protected Consumer statementChangedHandler; + + public Transformation(String desc) { + this.desc = desc; + } + + @SuppressWarnings("unused") + protected Transformation() { + } + + public static void setBugJudgement(Supplier judgement) { + bugJudgement = judgement; + } + + @Override + public String toString() { + return desc; + } + + public boolean init(String sql) { + isChanged = false; + return true; + } + + public boolean tryRemove(P parent, T target, BiConsumer setter) { + setter.accept(parent, null); + onStatementChanged(); + if (!bugStillTriggers()) { + setter.accept(parent, target); + onStatementChanged(); + return false; + } + reduceSteps++; + isChanged = true; + return true; + } + + public boolean tryReplace(P parent, T original, T vari, BiConsumer setter) { + setter.accept(parent, vari); + onStatementChanged(); + if (!bugStillTriggers()) { + setter.accept(parent, original); + onStatementChanged(); + return false; + } + reduceSteps++; + isChanged = true; + return true; + } + + public void tryRemoveElms(P parent, List elms, // NOPMD + BiConsumer> setter) { + boolean observeChange; + do { + observeChange = false; + for (int i = elms.size() - 1; i >= 0; i--) { + List reducedElms = new ArrayList<>(elms); + reducedElms.subList(i, i + 1).clear(); + setter.accept(parent, reducedElms); + onStatementChanged(); + if (bugStillTriggers()) { + elms = reducedElms; + onStatementChanged(); + observeChange = true; + } + } + isChanged |= observeChange; + setter.accept(parent, elms); + reduceSteps++; + onStatementChanged(); + } while (observeChange); + + } + + public boolean bugStillTriggers() { + try { + return Transformation.bugJudgement.get(); + } catch (Exception ignored) { + } + return false; + } + + public void apply() { + isChanged = false; + } + + public boolean changed() { + return isChanged; + } + + public static long getReduceSteps() { + return reduceSteps; + } + + protected void onStatementChanged() { + } + + public void setStatementChangedCallBack(Consumer statementChangedHandler) { + this.statementChangedHandler = statementChangedHandler; + } + +} diff --git a/test/sqlancer/reducer/TestASTBasedReducer.java b/test/sqlancer/reducer/TestASTBasedReducer.java index b65ceef21..549bbe84d 100644 --- a/test/sqlancer/reducer/TestASTBasedReducer.java +++ b/test/sqlancer/reducer/TestASTBasedReducer.java @@ -1,14 +1,19 @@ package sqlancer.reducer; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.junit.jupiter.api.Test; import sqlancer.common.query.Query; -import java.util.Arrays; import java.util.List; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestASTBasedReducer { @Test - void testLongStatement() throws Exception { + void testRemovingClauses() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); String[] queriesStr = { @@ -16,12 +21,17 @@ void testLongStatement() throws Exception { env.setInitialStatementsFromStrings(List.of(queriesStr)); env.setBugInducingCondition(statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } return queriesString.contains("&&"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStr)); - System.out.println(reducedResult); + String outcome = TestEnvironment.getQueriesString(reducedResult); + assertEquals(outcome, "SELECT * FROM v0 WHERE v0.c && v0.c;"); } @Test @@ -33,12 +43,16 @@ void testReducingMultipleTokensToOne() throws Exception { env.setInitialStatementsFromStrings(List.of(queriesStr)); env.setBugInducingCondition(statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } return queriesString.contains("||"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStr)); - System.out.println(reducedResult); + assertEquals(TestEnvironment.getQueriesString(reducedResult), "SELECT row_id FROM v0 WHERE v0.rowid || 0;"); } @Test @@ -47,83 +61,207 @@ void testMultipleStatements() throws Exception { String[] queriesStrs = { "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)", - "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)", - "SELECT * FROM table_3;" }; + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); env.setBugInducingCondition(statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + return queriesString.toUpperCase().contains("UNION"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals(TestEnvironment.getQueriesString(reducedResult), + "SELECT row_id FROM v0;\nSELECT row_id FROM v0 UNION SELECT * FROM v0;"); } @Test - void testJoin() throws Exception { + void removeJoins() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); String[] queriesStrs = { "SELECT * FROM t0, t1, t2, t3, t4 Where t2.val = t1.val" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); env.setBugInducingCondition(statements -> { + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } String queriesString = TestEnvironment.getQueriesString(statements); - return queriesString.contains("t1") && queriesString.contains("WHERE"); + if (!queriesString.contains("WHERE")) { + return false; + } + String[] split = queriesString.split("WHERE"); + String columns = split[0]; + String condition = split[1]; + return columns.contains("t1") && condition.contains("t1"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals(TestEnvironment.getQueriesString(reducedResult), "SELECT * FROM t0, t1 WHERE t1.val;"); } @Test - void testCase() throws Exception { + void testComplicated() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); String[] queriesStrs = { "SELECT STRING_AGG(v0.c2) FROM t0, v0 WHERE (CASE true WHEN (ABS(true) BETWEEN (v0.c0 LIKE NULL ESCAPE v0.c2) AND (DATE '1970-01-23' NOT IN (v0.c2))) THEN (0.07914839711718646 NOT BETWEEN '' AND ((v0.c0)OR(v0.c2))) WHEN v0.c1 THEN ((v0.c1)-(v0.c0)) WHEN t0.c1 THEN (TIMESTAMP '1969-12-29 20:22:33' IN (PI(), v0.c2, (v0.c1 BETWEEN '' AND v0.rowid))) WHEN v0.c1 THEN TIMESTAMP '1969-12-16 17:24:43' WHEN ((((v0.c1)-(t0.c0)))||(t0.c0)) THEN true ELSE ((0.279978719843174)/(((v0.c1)>(DATE '1969-12-19')))) END ) GROUP BY ((DATE '1970-01-24') IS NULL), t0.c1, (CASE (v0.c1 LIKE ((0.9833120083624495)SIMILAR TO(t0.rowid)) ESCAPE CEIL(TIMESTAMP '1970-01-11 16:38:26')) WHEN t0.rowid THEN 0.27742217994251717 ELSE ((v0.c0) IS NOT NULL) END );" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); - env.setBugInducingCondition(statements -> { + Function>, Boolean> condition = statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } return queriesString.toUpperCase().contains("CASE"); + }; + env.setBugInducingCondition(condition); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertTrue(condition.apply(reducedResult)); + } + + @Test + void testSimplifyingConstantStringValue() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { "SELECT * FROM t0 WHERE v LIKE '[vQ3㭫oQ';" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("LIKE"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals("SELECT * FROM t0 WHERE v LIKE '_';", TestEnvironment.getQueriesString(reducedResult)); } @Test - void testConstantVar() throws Exception { + void testSimplifyingConstantLongValue() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); - String[] queriesStrs = { - "SELECT STRING_AGG(v0.c2) FROM t0 GROUP BY ( (CASE (t0.rowid) WHEN t0.rowid THEN 0.27742217994251717 ELSE ((v0.c0) IS NOT NULL) END) )" }; + String[] queriesStrs = { "SELECT * FROM t0 where t0.v = 314598267;" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); env.setBugInducingCondition(statements -> { + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + String queriesString = TestEnvironment.getQueriesString(statements); - return queriesString.toUpperCase().contains("CASE"); + return queriesString.contains("t0.v = "); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals("SELECT * FROM t0 WHERE t0.v = 0;", TestEnvironment.getQueriesString(reducedResult)); } @Test - void testFunction() throws Exception { + void testSubSelects() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); String[] queriesStrs = { - "SELECT DATE '1970-01-11', false, t1.c1, t1.c1, (t1.c1 NOT IN (((('' LIKE t1.c2 ESCAPE t1.c1)) IS NOT NULL))) FROM t1 WHERE t1.c0 GROUP BY (((('Zlb)' IN (t1.c0)) LIKE t1.c1 ESCAPE (0.6419925594156123 BETWEEN t1.c1 AND ')-'))) ::BOOL) HAVING ((LAST_DAY(1630554083))&(AVG((CASE t1.c0 WHEN t1.c2 THEN DATE '1970-01-09' ELSE '' END )))) LIMIT 714775291;" }; + "SELECT AVG(c0) FROM (SELECT SUM(c1) AS c0 FROM t1 GROUP BY c2 LIMIT 32 OFFSET 128) AS t1;" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); env.setBugInducingCondition(statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } return queriesString.contains("AVG"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals("SELECT AVG(c0) FROM (SELECT SUM(c1) AS c0 FROM t1) AS t1;", + TestEnvironment.getQueriesString(reducedResult)); + } + + @Test + void testInsert() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "INSERT INTO t1(c2, c0) VALUES (1508438260, 2929), (1508438260, TIMESTAMP '1969-12-26 01:57:21'), (0.5347171705591047, 398662142);" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("(0.5347171705591047, 398662142)"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("INSERT INTO t1 (c2, c0) VALUES (0.5347171705591047, 398662142);", + TestEnvironment.getQueriesString(reducedResult)); } + + @Test + void testWithSelect() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "WITH cte1 AS (SELECT a, b FROM table1 where a < b), cte2 AS (SELECT c, d FROM table2 where c = d) SELECT b, d FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("table1"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("WITH cte1 AS (SELECT a FROM table1) SELECT b FROM cte1;", + TestEnvironment.getQueriesString(reducedResult)); + } + + @Test + void testRoundDouble() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + String[] queriesStrs = { "SELECT * FROM t0 WHERE (2.1427572639 IS NULL);" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("WHERE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("SELECT * FROM t0 WHERE 2.143 IS NULL;", TestEnvironment.getQueriesString(reducedResult)); + } + } From f852f8cc08a98db34d40d20011b7c8fe47abf453 Mon Sep 17 00:00:00 2001 From: Branimir Vujicic Date: Thu, 24 Aug 2023 17:52:20 +0200 Subject: [PATCH 0817/1284] Add support for Presto - common classes (#884) To implement support for Presto some changes in common classes are required. ## Changes in common classes: 1. class sqlancer.MainOptions - added global parameters : * canonicalizeString (boolean) - presto doesn't support JDBC queries with ";" at the end of statement * compareResultsContent (boolean) - comparing content of VARBINARY columns fails 2. class sqlancer.ComparatorHelper - compare result based on parameter ``` boolean compare = state.getOptions().compareResultsContent(); if (compare && !firstHashSet.equals(secondHashSet)) { ``` 3. sqlancer.common.query.SQLancerResultSet - added method: ``` public String getType(int i) throws SQLException { return rs.getMetaData().getColumnTypeName(i); } ``` 4. sqlancer.common.query.SQLQueryAdapter : added constructor ``` public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema, boolean canonicalizeString) { ``` --- src/sqlancer/ComparatorHelper.java | 6 +++-- src/sqlancer/MainOptions.java | 14 ++++++++++ .../common/query/SQLQueryAdapter.java | 11 +++++++- .../common/query/SQLancerResultSet.java | 4 +++ test/sqlancer/TestComparatorHelper.java | 26 ++++++++++++++++--- 5 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 41d2cf996..5da635de2 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -48,7 +48,8 @@ public static List getResultSetFirstColumnAsString(String queryString, E e.printStackTrace(); } } - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + boolean canonicalizeString = state.getOptions().canonicalizeSqlString(); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, true, canonicalizeString); List resultSet = new ArrayList<>(); SQLancerResultSet result = null; try { @@ -106,7 +107,8 @@ public static void assumeResultSetsAreEqual(List resultSet, List Set firstHashSet = new HashSet<>(resultSet); Set secondHashSet = new HashSet<>(secondResultSet); - if (!firstHashSet.equals(secondHashSet)) { + boolean validateResultSizeOnly = state.getOptions().validateResultSizeOnly(); + if (!validateResultSizeOnly && !firstHashSet.equals(secondHashSet)) { Set firstResultSetMisses = new HashSet<>(firstHashSet); firstResultSetMisses.removeAll(secondHashSet); Set secondResultSetMisses = new HashSet<>(secondHashSet); diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 232856f94..a6a252fa3 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -141,6 +141,12 @@ public class MainOptions { @Parameter(names = "--ast-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the statement reducer will do") private long maxStatementReduceTime = NO_REDUCE_LIMIT; // NOPMD + @Parameter(names = "--validate-result-size-only", description = "Should validate result size only and skip comparing content of the result set ", arity = 1) + private boolean validateResultSizeOnly = false; // NOPMD + + @Parameter(names = "--canonicalize-sql-strings", description = "Should canonicalize query string (add ';' at the end", arity = 1) + private boolean canonicalizeSqlString = true; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -322,4 +328,12 @@ public long getMaxASTReduceTime() { return maxASTReduceTime; } + public boolean validateResultSizeOnly() { + return validateResultSizeOnly; + } + + public boolean canonicalizeSqlString() { + return canonicalizeSqlString; + } + } diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 248403458..3784ed635 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -32,7 +32,16 @@ private static boolean guessAffectSchemaFromQuery(String query) { } public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema) { - this.query = canonicalizeString(query); + this(query, expectedErrors, couldAffectSchema, true); + } + + public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema, + boolean canonicalizeString) { + if (canonicalizeString) { + this.query = canonicalizeString(query); + } else { + this.query = query; + } this.expectedErrors = expectedErrors; this.couldAffectSchema = couldAffectSchema; checkQueryString(); diff --git a/src/sqlancer/common/query/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java index f83cdd3a5..d6f2a67cb 100644 --- a/src/sqlancer/common/query/SQLancerResultSet.java +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -46,6 +46,10 @@ public long getLong(int i) throws SQLException { return rs.getLong(i); } + public String getType(int i) throws SQLException { + return rs.getMetaData().getColumnTypeName(i); + } + public void registerEpilogue(Runnable runnableEpilogue) { this.runnableEpilogue = runnableEpilogue; } diff --git a/test/sqlancer/TestComparatorHelper.java b/test/sqlancer/TestComparatorHelper.java index 6815b9258..e8b06388b 100644 --- a/test/sqlancer/TestComparatorHelper.java +++ b/test/sqlancer/TestComparatorHelper.java @@ -2,19 +2,37 @@ import static org.junit.jupiter.api.Assertions.assertThrowsExactly; +import java.sql.SQLException; import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.Test; +import sqlancer.h2.H2Options; +import sqlancer.h2.H2Schema; + public class TestComparatorHelper { // TODO: Implement tests for the other ComparatorHelper methods + // TODO: create test state that not depends on specific database + final SQLGlobalState state = new SQLGlobalState() { + + @Override + protected H2Schema readSchema() throws SQLException { + return H2Schema.fromConnection(getConnection(), getDatabaseName()); + } + + @Override + public MainOptions getOptions() { + return new MainOptions(); + } + }; + @Test public void testAssumeResultSetsAreEqualWithEqualSets() { List r1 = Arrays.asList("a", "b", "c"); List r2 = Arrays.asList("a", "b", "c"); - ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state); } @@ -26,7 +44,7 @@ public void testAssumeResultSetsAreEqualWithUnequalLengthSets() { // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions // is raised assertThrowsExactly(NullPointerException.class, () -> { - ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state); }); } @@ -38,7 +56,7 @@ public void testAssumeResultSetsAreEqualWithUnequalValueSets() { // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions // is raised assertThrowsExactly(NullPointerException.class, () -> { - ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state); }); } @@ -46,7 +64,7 @@ public void testAssumeResultSetsAreEqualWithUnequalValueSets() { public void testAssumeResultSetsAreEqualWithCanonicalizationRule() { List r1 = Arrays.asList("a", "b", "c"); List r2 = Arrays.asList("a", "b", "d"); - ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null, (String s) -> { + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state, (String s) -> { return s.equals("d") ? "c" : s; }); } From 7804a3adec0962ad6d24687c42ec473aa49669fe Mon Sep 17 00:00:00 2001 From: Branimir Vujicic Date: Wed, 30 Aug 2023 03:41:55 +0200 Subject: [PATCH 0818/1284] Add support for Presto (#881) This is the first Pull Request for [Presto](https://prestodb.io/) implementation. The following oracles are currently completed: * NoRec oracle * TLP Where * TLP Aggregate (basic aggregate function) ## Data types The following data types are currently supported: BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, JSON, DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND, ARRAY. Others such as MAP, ROW, IPADDRESS, UID, IPPREFIX, HyperLogLog, P4HyperLogLog, KHyperLogLog, QDigest and TDigest will be supported in the future. ## Functions Large number of Presto supported are implemented, but only small number are actually used in test. Most of the functions will be supported in the future. --- README.md | 8 +- pom.xml | 7 +- src/check_names.py | 1 + src/sqlancer/Main.java | 2 + src/sqlancer/presto/PrestoConstantUtils.java | 40 + src/sqlancer/presto/PrestoErrors.java | 127 +++ .../presto/PrestoExpressionToNode.java | 25 + src/sqlancer/presto/PrestoGlobalState.java | 13 + src/sqlancer/presto/PrestoOptions.java | 166 ++++ src/sqlancer/presto/PrestoProvider.java | 195 +++++ src/sqlancer/presto/PrestoSchema.java | 484 +++++++++++ .../presto/PrestoToStringVisitor.java | 149 ++++ .../presto/ast/PrestoAggregateFunction.java | 714 +++++++++++++++ .../presto/ast/PrestoAtTimeZoneOperator.java | 22 + .../presto/ast/PrestoCastFunction.java | 24 + .../presto/ast/PrestoColumnReference.java | 12 + .../presto/ast/PrestoComparisonFunction.java | 81 ++ .../presto/ast/PrestoConditionalFunction.java | 95 ++ src/sqlancer/presto/ast/PrestoConstant.java | 810 ++++++++++++++++++ .../presto/ast/PrestoDateFunction.java | 523 +++++++++++ .../presto/ast/PrestoDefaultFunction.java | 233 +++++ src/sqlancer/presto/ast/PrestoExpression.java | 5 + src/sqlancer/presto/ast/PrestoFunction.java | 127 +++ .../ast/PrestoFunctionWithoutParenthesis.java | 24 + src/sqlancer/presto/ast/PrestoJoin.java | 118 +++ .../ast/PrestoMultiValuedComparison.java | 39 + .../PrestoMultiValuedComparisonOperator.java | 43 + .../ast/PrestoMultiValuedComparisonType.java | 11 + .../ast/PrestoQuantifiedComparison.java | 36 + src/sqlancer/presto/ast/PrestoSelect.java | 18 + .../presto/ast/PrestoTableReference.java | 11 + .../ast/PrestoUnaryPostfixOperation.java | 52 ++ .../ast/PrestoUnaryPrefixOperation.java | 62 ++ .../presto/gen/PrestoAlterTableGenerator.java | 67 ++ .../presto/gen/PrestoDeleteGenerator.java | 32 + .../presto/gen/PrestoIndexGenerator.java | 57 ++ .../presto/gen/PrestoInsertGenerator.java | 52 ++ .../gen/PrestoRandomQuerySynthesizer.java | 72 ++ .../presto/gen/PrestoTableGenerator.java | 69 ++ .../gen/PrestoTypedExpressionGenerator.java | 798 +++++++++++++++++ .../presto/gen/PrestoUpdateGenerator.java | 53 ++ .../presto/gen/PrestoViewGenerator.java | 36 + .../presto/test/PrestoNoRECOracle.java | 138 +++ ...restoQueryPartitioningAggregateTester.java | 206 +++++ .../test/PrestoQueryPartitioningBase.java | 90 ++ ...PrestoQueryPartitioningDistinctTester.java | 44 + .../PrestoQueryPartitioningGroupByTester.java | 53 ++ .../PrestoQueryPartitioningHavingTester.java | 64 ++ .../PrestoQueryPartitioningWhereTester.java | 46 + 49 files changed, 6150 insertions(+), 4 deletions(-) create mode 100644 src/sqlancer/presto/PrestoConstantUtils.java create mode 100644 src/sqlancer/presto/PrestoErrors.java create mode 100644 src/sqlancer/presto/PrestoExpressionToNode.java create mode 100644 src/sqlancer/presto/PrestoGlobalState.java create mode 100644 src/sqlancer/presto/PrestoOptions.java create mode 100644 src/sqlancer/presto/PrestoProvider.java create mode 100644 src/sqlancer/presto/PrestoSchema.java create mode 100644 src/sqlancer/presto/PrestoToStringVisitor.java create mode 100644 src/sqlancer/presto/ast/PrestoAggregateFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java create mode 100644 src/sqlancer/presto/ast/PrestoCastFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoColumnReference.java create mode 100644 src/sqlancer/presto/ast/PrestoComparisonFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoConditionalFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoConstant.java create mode 100644 src/sqlancer/presto/ast/PrestoDateFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoDefaultFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoExpression.java create mode 100644 src/sqlancer/presto/ast/PrestoFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java create mode 100644 src/sqlancer/presto/ast/PrestoJoin.java create mode 100644 src/sqlancer/presto/ast/PrestoMultiValuedComparison.java create mode 100644 src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java create mode 100644 src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java create mode 100644 src/sqlancer/presto/ast/PrestoQuantifiedComparison.java create mode 100644 src/sqlancer/presto/ast/PrestoSelect.java create mode 100644 src/sqlancer/presto/ast/PrestoTableReference.java create mode 100644 src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java create mode 100644 src/sqlancer/presto/gen/PrestoAlterTableGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoDeleteGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoIndexGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoInsertGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java create mode 100644 src/sqlancer/presto/gen/PrestoTableGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoUpdateGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoViewGenerator.java create mode 100644 src/sqlancer/presto/test/PrestoNoRECOracle.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningBase.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java diff --git a/README.md b/README.md index 5c18c887d..d84a993b1 100644 --- a/README.md +++ b/README.md @@ -68,11 +68,13 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | | OceanBase | Working | Untyped | | | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | -| Databend | Working | Typed | | -| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | +| Databend | Working | Typed | | +| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | | CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | | Materialize |Working | Typed | -| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. +| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. +| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. + # Using SQLancer diff --git a/pom.xml b/pom.xml index 4944369fa..292378933 100644 --- a/pom.xml +++ b/pom.xml @@ -316,6 +316,11 @@ duckdb_jdbc 0.5.1 + + com.facebook.presto + presto-jdbc + 0.283 + org.junit.jupiter junit-jupiter-engine @@ -467,4 +472,4 @@ - \ No newline at end of file + diff --git a/src/check_names.py b/src/check_names.py index a88c94e89..048da7669 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -48,6 +48,7 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["MySQL"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mysql")) name_to_files["OceanBase"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "oceanbase")) name_to_files["Postgres"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "postgres")) + name_to_files["Presto"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "presto")) name_to_files["QuestDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "questdb")) name_to_files["SQLite3"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "sqlite3")) name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 4b480e595..89fbafd38 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -43,6 +43,7 @@ import sqlancer.mysql.MySQLProvider; import sqlancer.oceanbase.OceanBaseProvider; import sqlancer.postgres.PostgresProvider; +import sqlancer.presto.PrestoProvider; import sqlancer.questdb.QuestDBProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.stonedb.StoneDBProvider; @@ -719,6 +720,7 @@ private static void checkForIssue799(List> providers) providers.add(new MongoDBProvider()); providers.add(new MySQLProvider()); providers.add(new OceanBaseProvider()); + providers.add(new PrestoProvider()); providers.add(new PostgresProvider()); providers.add(new QuestDBProvider()); providers.add(new SQLite3Provider()); diff --git a/src/sqlancer/presto/PrestoConstantUtils.java b/src/sqlancer/presto/PrestoConstantUtils.java new file mode 100644 index 000000000..6de744019 --- /dev/null +++ b/src/sqlancer/presto/PrestoConstantUtils.java @@ -0,0 +1,40 @@ +package sqlancer.presto; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +public final class PrestoConstantUtils { + + private PrestoConstantUtils() { + } + + public static String removeNoneAscii(String str) { + return str.replaceAll("[^\\x00-\\x7F]", ""); + } + + public static String removeNonePrintable(String str) { // All Control Char + return str.replaceAll("[\\p{C}]", ""); + } + + public static String removeOthersControlChar(String str) { // Some Control Char + return str.replaceAll("[\\p{Cntrl}\\p{Cc}\\p{Cf}\\p{Co}\\p{Cn}]", ""); + } + + public static String removeAllControlChars(String str) { + return removeOthersControlChar(removeNonePrintable(str)).replaceAll("[\\r\\n\\t]", ""); + } + + public static BigDecimal getDecimal(double val, int scale, int precision) { + int part = precision - scale; + // long part + long lng = (long) val; + // decimal places + double d1 = val - lng; + String xStr = Long.toString(lng); + String substring = xStr.substring(xStr.length() - part); + long newX = substring.isEmpty() ? 0 : Long.parseLong(substring); + double finalD = newX + d1; + return new BigDecimal(finalD).setScale(scale, RoundingMode.CEILING); + } + +} diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java new file mode 100644 index 000000000..603addb5c --- /dev/null +++ b/src/sqlancer/presto/PrestoErrors.java @@ -0,0 +1,127 @@ +package sqlancer.presto; + +import sqlancer.common.query.ExpectedErrors; + +public final class PrestoErrors { + + private PrestoErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + // Presto errors + errors.add("cannot be applied to"); + errors.add("LIKE expression must evaluate to a varchar"); + errors.add("JOIN ON clause must evaluate to a boolean"); + // errors.add("Unexpected parameters"); + + // SELECT SUM(count) FROM (SELECT + // CAST((-179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0000 + // IS NOT NULL AND + // -179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0000) + // AS BIGINT)as count FROM t0) as res + errors.add("Decimal overflow"); + errors.add("multiplication overflow"); + errors.add("addition overflow"); + errors.add("subtraction overflow"); + + // cast + // errors.add("Cannot cast"); + errors.add("Value cannot be cast to"); + errors.add("Cannot cast DECIMAL"); + errors.add("Cannot cast BIGINT"); + errors.add("Cannot cast INTEGER"); + + // TODO: check + errors.add("io.airlift.slice.Slice cannot be cast to java.lang.Number"); + errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); + errors.add("Unexpected subquery expression in logical plan"); + + // 9223372036854775808 + errors.add("Invalid numeric literal"); + + errors.add("Division by zero"); + errors.add("/ by zero"); + + errors.add("Cannot subtract hour, minutes or seconds from a date"); + errors.add("Cannot add hour, minutes or seconds to a date"); + + errors.add("DECIMAL scale must be in range"); + errors.add("multiplication overflow"); + errors.add("addition overflow"); + errors.add("subtraction overflow"); + errors.add("Decimal overflow"); + errors.add("IN value and list items must be the same type"); + errors.add("is not a valid timestamp literal"); + errors.add("Unknown time-zone ID"); + errors.add("GROUP BY position"); + + // ARRAY + errors.add("Unknown type: ARRAY"); + } + + private static void addRegexErrors(ExpectedErrors errors) { + errors.add("missing ]"); + errors.add("missing )"); + errors.add("invalid escape sequence"); + errors.add("no argument for repetition operator: "); + errors.add("bad repetition operator"); + errors.add("trailing \\"); + errors.add("invalid perl operator"); + errors.add("invalid character class range"); + errors.add("width is not integer"); + } + + private static void addFunctionErrors(ExpectedErrors errors) { + errors.add("SUBSTRING cannot handle negative lengths"); + errors.add("is undefined outside [-1,1]"); // ACOS etc + errors.add("invalid type specifier"); // PRINTF + errors.add("argument index out of range"); // PRINTF + errors.add("invalid format string"); // PRINTF + errors.add("number is too big"); // PRINTF + errors.add("Like pattern must not end with escape character!"); // LIKE + errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part + errors.add("extract specifier"); // date_part + errors.add("not recognized"); // date_part + errors.add("not supported"); // date_part + errors.add("Failed to cast"); + errors.add("Conversion Error"); + errors.add("Could not cast value"); + errors.add("Insufficient padding in RPAD"); // RPAD + errors.add("Could not choose a best candidate function for the function call"); // monthname + errors.add("expected a numeric precision field"); // ROUND + errors.add("with non-constant precision is not supported"); // ROUND + } + + // TODO: cover presto error + public static void addInsertErrors(ExpectedErrors errors) { + addRegexErrors(errors); + addFunctionErrors(errors); + + errors.add("NOT NULL constraint failed"); + errors.add("PRIMARY KEY or UNIQUE constraint violated"); + errors.add("duplicate key"); + errors.add("can't be cast because the value is out of range for the destination type"); + errors.add("Could not convert string"); + errors.add("Unimplemented type for cast"); + errors.add("field value out of range"); + errors.add("CHECK constraint failed"); + errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid + errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if + // the table has a primary key + errors.add("Could not cast value"); + errors.add("create unique index, table contains duplicate data"); + errors.add("Failed to cast"); + + errors.add("Values rows have mismatched types"); + errors.add("Mismatch at column"); + errors.add("This connector does not support updates or deletes"); + errors.add("Values rows have mismatched types"); + errors.add("Invalid numeric literal"); + + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.add("must be an aggregate expression or appear in GROUP BY clause"); + } + +} diff --git a/src/sqlancer/presto/PrestoExpressionToNode.java b/src/sqlancer/presto/PrestoExpressionToNode.java new file mode 100644 index 000000000..96497e47c --- /dev/null +++ b/src/sqlancer/presto/PrestoExpressionToNode.java @@ -0,0 +1,25 @@ +package sqlancer.presto; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.ast.PrestoExpression; + +public final class PrestoExpressionToNode { + + private PrestoExpressionToNode() { + + } + + @SuppressWarnings("unchecked") + public static Node cast(PrestoExpression expression) { + return (Node) expression; + } + + @SuppressWarnings("unchecked") + public static List> casts(List expressions) { + return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/presto/PrestoGlobalState.java b/src/sqlancer/presto/PrestoGlobalState.java new file mode 100644 index 000000000..eb053712e --- /dev/null +++ b/src/sqlancer/presto/PrestoGlobalState.java @@ -0,0 +1,13 @@ +package sqlancer.presto; + +import java.sql.SQLException; + +import sqlancer.SQLGlobalState; + +public class PrestoGlobalState extends SQLGlobalState { + + @Override + protected PrestoSchema readSchema() throws SQLException { + return PrestoSchema.fromConnection(getConnection(), getDatabaseName()); + } +} diff --git a/src/sqlancer/presto/PrestoOptions.java b/src/sqlancer/presto/PrestoOptions.java new file mode 100644 index 000000000..78deab27c --- /dev/null +++ b/src/sqlancer/presto/PrestoOptions.java @@ -0,0 +1,166 @@ +package sqlancer.presto; + +import java.util.ArrayList; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.presto.test.PrestoNoRECOracle; +import sqlancer.presto.test.PrestoQueryPartitioningAggregateTester; +import sqlancer.presto.test.PrestoQueryPartitioningDistinctTester; +import sqlancer.presto.test.PrestoQueryPartitioningGroupByTester; +import sqlancer.presto.test.PrestoQueryPartitioningHavingTester; +import sqlancer.presto.test.PrestoQueryPartitioningWhereTester; + +@Parameters(commandDescription = "Presto") +public class PrestoOptions implements DBMSSpecificOptions { + + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 8080; + + @Parameter(names = "--test-collate", arity = 1) + public boolean testCollate = true; + + @Parameter(names = "--test-check", description = "Allow generating CHECK constraints in tables", arity = 1) + public boolean testCheckConstraints = true; + + @Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1) + public boolean testDefaultValues = true; + + @Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1) + public boolean testNotNullConstraints = true; + + @Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1) + public boolean testFunctions = true; + + @Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1) + public boolean testCasts = true; + + @Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions (FALSE by default : Presto null handling in BETWEEN operator : https://prestodb.io/docs/current/functions/comparison.html )", arity = 1) + public boolean testBetween; + + @Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1) + public boolean testIn = true; + + @Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1) + public boolean testCase = true; + + @Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1) + public boolean testBinaryLogicals = true; + + @Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1) + public boolean testIntConstants = true; + + @Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1) + public boolean testStringConstants = true; + + @Parameter(names = "--test-time-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testDateConstants = true; + + @Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testTimeConstants = true; + + @Parameter(names = "--test-timestamp-constants", description = "Allow generating TIMESTAMP constants", arity = 1) + public boolean testTimestampConstants = true; + + @Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1) + public boolean testFloatConstants = true; + + @Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1) + public boolean testBooleanConstants = true; + + @Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1) + public boolean testBinaryComparisons = true; + + @Parameter(names = "--test-indexes", description = "Allow explicit (i.e. CREATE INDEX) and implicit (i.e., UNIQUE and PRIMARY KEY) indexes", arity = 1) + public boolean testIndexes = true; + + @Parameter(names = "--test-rowid", description = "Test tables' rowid columns", arity = 1) + public boolean testRowid = true; + + @Parameter(names = "--max-num-views", description = "The maximum number of views that can be generated for a database", arity = 1) + public int maxNumViews = 1; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates = 5; + + @Parameter(names = "--oracle") + public List oracles = List.of(PrestoOracleFactory.NOREC); + + @Parameter(names = "--catalog") + public String catalog = "memory"; + + @Parameter(names = "--schema") + public String schema = "test"; + + @Parameter(names = "--typed-generator", description = "the expression generator type - typed and untyped ") + public boolean typedGenerator = true; + + @Override + public List getTestOracleFactory() { + return oracles; + } + + public enum PrestoOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoNoRECOracle(globalState); + } + + }, + HAVING { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PrestoGlobalState globalState) { + List> oracles = new ArrayList<>(); + oracles.add(new PrestoQueryPartitioningWhereTester(globalState)); + oracles.add(new PrestoQueryPartitioningHavingTester(globalState)); + oracles.add(new PrestoQueryPartitioningAggregateTester(globalState)); + oracles.add(new PrestoQueryPartitioningDistinctTester(globalState)); + oracles.add(new PrestoQueryPartitioningGroupByTester(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + } + + } + +} diff --git a/src/sqlancer/presto/PrestoProvider.java b/src/sqlancer/presto/PrestoProvider.java new file mode 100644 index 000000000..be3ef4325 --- /dev/null +++ b/src/sqlancer/presto/PrestoProvider.java @@ -0,0 +1,195 @@ +package sqlancer.presto; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.presto.gen.PrestoInsertGenerator; +import sqlancer.presto.gen.PrestoTableGenerator; + +@AutoService(DatabaseProvider.class) +public class PrestoProvider extends SQLProviderAdapter { + + public PrestoProvider() { + super(PrestoGlobalState.class, PrestoOptions.class); + } + + // TODO : check actions based on connector + // returns number of actions + private static int mapActions(PrestoGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + if (Objects.requireNonNull(a) == Action.INSERT) { + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + // case UPDATE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + // case EXPLAIN: + // return r.getInteger(0, 2); + // case DELETE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + // case CREATE_VIEW: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + } + throw new AssertionError(a); + } + + @Override + public void generateDatabase(PrestoGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new PrestoTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + PrestoProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(PrestoGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + boolean useSSl = true; + if (globalState.getOptions().isDefaultUsername() && globalState.getOptions().isDefaultPassword()) { + username = "presto"; + password = null; + useSSl = false; + } + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = PrestoOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = PrestoOptions.DEFAULT_PORT; + } + String catalogName = globalState.getDbmsSpecificOptions().catalog; + String databaseName = globalState.getDatabaseName(); + String url = String.format("jdbc:presto://%s:%d/%s?SSL=%b", host, port, catalogName, useSSl); + Connection con = DriverManager.getConnection(url, username, password); + List schemaNames = getSchemaNames(con, catalogName, databaseName); + dropExistingTables(con, catalogName, databaseName, schemaNames); + dropSchema(globalState, con, catalogName, databaseName); + createSchema(globalState, con, catalogName, databaseName); + useSchema(globalState, con, catalogName, databaseName); + return new SQLConnection(con); + + } + + private static void useSchema(PrestoGlobalState globalState, Connection con, String catalogName, + String databaseName) throws SQLException { + globalState.getState().logStatement("USE " + catalogName + "." + databaseName); + try (Statement s = con.createStatement()) { + s.execute("USE " + catalogName + "." + databaseName); + } + } + + private static void createSchema(PrestoGlobalState globalState, Connection con, String catalogName, + String databaseName) throws SQLException { + globalState.getState().logStatement("CREATE SCHEMA IF NOT EXISTS " + catalogName + "." + databaseName); + try (Statement s = con.createStatement()) { + s.execute("CREATE SCHEMA IF NOT EXISTS " + catalogName + "." + databaseName); + } + } + + private static void dropSchema(PrestoGlobalState globalState, Connection con, String catalogName, + String databaseName) throws SQLException { + globalState.getState().logStatement("DROP SCHEMA IF EXISTS " + catalogName + "." + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP SCHEMA IF EXISTS " + catalogName + "." + databaseName); + } + } + + private static List getSchemaNames(Connection con, String catalogName, String databaseName) + throws SQLException { + List schemaNames = new ArrayList<>(); + final String showSchemasSql = "SHOW SCHEMAS FROM " + catalogName + " LIKE '" + databaseName + "'"; + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(showSchemasSql)) { + while (rs.next()) { + schemaNames.add(rs.getString("Schema")); + } + } + } + return schemaNames; + } + + private static void dropExistingTables(Connection con, String catalogName, String databaseName, + List schemaNames) throws SQLException { + if (!schemaNames.isEmpty()) { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES FROM " + catalogName + "." + databaseName)) { + while (rs.next()) { + tableNames.add(rs.getString("Table")); + } + } + } + try (Statement s = con.createStatement()) { + for (String tableName : tableNames) { + s.execute("DROP TABLE IF EXISTS " + catalogName + "." + databaseName + "." + tableName); + } + } + } + } + + @Override + public String getDBMSName() { + return "presto"; + } + + public enum Action implements AbstractAction { + // SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES", new ExpectedErrors(), false, false)), // + INSERT(PrestoInsertGenerator::getQuery); + // TODO : check actions based on connector + // DELETE(PrestoDeleteGenerator::generate), // + // UPDATE(PrestoUpdateGenerator::getQuery), // + // CREATE_VIEW(PrestoViewGenerator::generate), // + // EXPLAIN((g) -> { + // ExpectedErrors errors = new ExpectedErrors(); + // PrestoErrors.addExpressionErrors(errors); + // PrestoErrors.addGroupByErrors(errors); + // return new SQLQueryAdapter( + // "EXPLAIN " + PrestoToStringVisitor + // .asString(PrestoRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), + // errors); + // }); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(PrestoGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + +} diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java new file mode 100644 index 000000000..70323a18b --- /dev/null +++ b/src/sqlancer/presto/PrestoSchema.java @@ -0,0 +1,484 @@ +package sqlancer.presto; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; + +public class PrestoSchema extends AbstractSchema { + + public PrestoSchema(List databaseTables) { + super(databaseTables); + } + + public static PrestoSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, databaseName, tableName); + boolean isView = tableName.startsWith("v"); + PrestoTable t = new PrestoTable(tableName, databaseColumns, isView); + for (PrestoColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new PrestoSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + // TODO: UPDATE + // SHOW TABLES [ FROM schema ] [ LIKE pattern [ ESCAPE 'escape_character' ] ] + try (ResultSet rs = s.executeQuery("SHOW TABLES")) { + while (rs.next()) { + tableNames.add(rs.getString("Table")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("select " + " table_catalog " + " , table_schema " + + " , table_name " + " , column_name " + " , is_nullable " + " , data_type " + + " from information_schema.columns " + " where table_schema = '%s' and table_name = '%s'", + databaseName, tableName))) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + boolean isNullable = rs.getString("is_nullable").contentEquals("YES"); + PrestoColumn c = new PrestoColumn(columnName, getColumnType(dataType), false, isNullable); + columns.add(c); + } + } + } + + return columns; + } + + private static PrestoCompositeDataType getColumnType(String typeString) { + int bracesStart = typeString.indexOf('('); + String type; + int size = 0; + int precision = 0; + if (bracesStart != -1) { + type = typeString.substring(0, bracesStart); + } else { + type = typeString; + } + type = type.toUpperCase(); + + PrestoDataType primitiveType; + switch (type) { + case "INTEGER": + primitiveType = PrestoDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = PrestoDataType.INT; + size = 2; + break; + case "BIGINT": + primitiveType = PrestoDataType.INT; + size = 8; + break; + case "TINYINT": + primitiveType = PrestoDataType.INT; + size = 1; + break; + case "VARCHAR": + primitiveType = PrestoDataType.VARCHAR; + break; + case "VARBINARY": + primitiveType = PrestoDataType.VARBINARY; + break; + case "CHAR": + primitiveType = PrestoDataType.CHAR; + break; + case "FLOAT": + case "REAL": + primitiveType = PrestoDataType.FLOAT; + size = 4; + break; + case "DOUBLE": + primitiveType = PrestoDataType.FLOAT; + size = 8; + break; + case "DECIMAL": + primitiveType = PrestoDataType.DECIMAL; + break; + case "BOOLEAN": + primitiveType = PrestoDataType.BOOLEAN; + break; + case "DATE": + primitiveType = PrestoDataType.DATE; + break; + case "TIME": + primitiveType = PrestoDataType.TIME; + break; + case "TIME WITH TIME ZONE": + primitiveType = PrestoDataType.TIME_WITH_TIME_ZONE; + break; + case "TIMESTAMP": + primitiveType = PrestoDataType.TIMESTAMP; + break; + case "TIMESTAMP WITH TIME ZONE": + primitiveType = PrestoDataType.TIMESTAMP_WITH_TIME_ZONE; + break; + case "INTERVAL DAY TO SECOND": + primitiveType = PrestoDataType.INTERVAL_DAY_TO_SECOND; + break; + case "INTERVAL YEAR TO MONTH": + primitiveType = PrestoDataType.INTERVAL_YEAR_TO_MONTH; + break; + case "JSON": + primitiveType = PrestoDataType.JSON; + break; + case "ARRAY": + primitiveType = PrestoDataType.ARRAY; + break; + case "NULL": + primitiveType = PrestoDataType.NULL; + break; + default: + throw new AssertionError(typeString); + } + return new PrestoCompositeDataType(primitiveType, size, precision); + } + + public PrestoTables getRandomTableNonEmptyTables() { + return new PrestoTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public enum PrestoDataType { + BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, JSON, DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, + TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND, ARRAY, + // MAP, + // ROW, + // IPADDRESS, + // UID, + // IPPREFIX, + // HyperLogLog, + // P4HyperLogLog, + // KHyperLogLog, + // QDigest, + // TDigest, + NULL; + + public static PrestoDataType getRandomWithoutNull() { + PrestoDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == PrestoDataType.NULL); + return dt; + } + + public static List getNumericTypes() { + return Arrays.asList(INT, FLOAT, DECIMAL, DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, + TIMESTAMP_WITH_TIME_ZONE); + } + + public static List getComparableTypes() { + return Arrays.asList(BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, JSON, DATE, TIME, TIMESTAMP, + TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND); + } + + public static List getOrderableTypes() { + return Arrays.asList(BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, + // JSON, + DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, + INTERVAL_DAY_TO_SECOND, ARRAY); + } + + public static List getNumberTypes() { + return Arrays.asList(INT, FLOAT, DECIMAL); + } + + public static List getTemporalTypes() { + return Arrays.asList(DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE); + } + + public static List getIntervalTypes() { + return Arrays.asList(INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND); + } + + public static List getTextTypes() { + return Arrays.asList(VARCHAR, CHAR, VARBINARY, JSON); + } + + public boolean isNumeric() { + switch (this) { + case INT: + case FLOAT: + case DECIMAL: + return true; + default: + return false; + } + } + + public boolean isOrderable() { + return getOrderableTypes().contains(this); + } + + public PrestoCompositeDataType get() { + return PrestoCompositeDataType.fromDataType(this); + } + } + + public static class PrestoCompositeDataType { + + private final PrestoDataType dataType; + + private final int size; + + private final int scale; + + private final PrestoCompositeDataType elementType; + + public PrestoCompositeDataType(PrestoDataType dataType, int dataSize, int dataScale) { + this.dataType = dataType; + this.size = dataSize; + this.scale = dataScale; + this.elementType = null; + } + + public PrestoCompositeDataType(PrestoDataType dataType, PrestoCompositeDataType elementType) { + if (dataType != PrestoDataType.ARRAY) { + throw new IllegalArgumentException(); + } + this.dataType = dataType; + this.size = -1; + this.scale = -1; + this.elementType = elementType; + } + + public static PrestoCompositeDataType getRandomWithoutNull() { + PrestoDataType type = PrestoDataType.getRandomWithoutNull(); + int size; + int scale = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case DECIMAL: + size = Math.toIntExact(8); + scale = Math.toIntExact(4); + break; + case VARBINARY: + case JSON: + case VARCHAR: + case CHAR: + size = Math.toIntExact(Randomly.getNotCachedInteger(10, 250)); + break; + case ARRAY: + return new PrestoCompositeDataType(type, PrestoCompositeDataType.getRandomWithoutNull()); + case BOOLEAN: + case DATE: + case TIME: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP: + case TIMESTAMP_WITH_TIME_ZONE: + case INTERVAL_DAY_TO_SECOND: + case INTERVAL_YEAR_TO_MONTH: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new PrestoCompositeDataType(type, size, scale); + } + + public static PrestoCompositeDataType fromDataType(PrestoDataType type) { + int size; + int scale = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case DECIMAL: + size = Math.toIntExact(8); + scale = Math.toIntExact(4); + break; + case JSON: + case VARCHAR: + case CHAR: + size = Math.toIntExact(Randomly.getNotCachedInteger(10, 250)); + break; + case ARRAY: + return new PrestoCompositeDataType(type, PrestoCompositeDataType.getRandomWithoutNull()); + case BOOLEAN: + case VARBINARY: + case DATE: + case TIME: + case TIMESTAMP: + case TIMESTAMP_WITH_TIME_ZONE: + case TIME_WITH_TIME_ZONE: + case INTERVAL_DAY_TO_SECOND: + case INTERVAL_YEAR_TO_MONTH: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new PrestoCompositeDataType(type, size, scale); + } + + public PrestoDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public int getScale() { + if (scale == -1) { + throw new AssertionError(this); + } + return scale; + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 8: + return "BIGINT"; + case 4: + return "INTEGER"; + case 2: + return "SMALLINT"; + case 1: + return "TINYINT"; + default: + throw new AssertionError(size); + } + case VARBINARY: + return "VARBINARY"; + case JSON: + return "JSON"; + case VARCHAR: + return "VARCHAR" + "(" + size + ")"; + case CHAR: + return "CHAR" + "(" + size + ")"; + case FLOAT: + switch (size) { + case 4: + return "REAL"; + case 8: + return "DOUBLE"; + default: + throw new AssertionError(size); + } + case DECIMAL: + return "DECIMAL" + "(" + size + ", " + scale + ")"; + case BOOLEAN: + return "BOOLEAN"; + case TIMESTAMP_WITH_TIME_ZONE: + return "TIMESTAMP WITH TIME ZONE"; + case TIMESTAMP: + return "TIMESTAMP"; + case INTERVAL_YEAR_TO_MONTH: + return "INTERVAL YEAR TO MONTH"; + case INTERVAL_DAY_TO_SECOND: + return "INTERVAL DAY TO SECOND"; + case DATE: + return "DATE"; + case TIME: + return "TIME"; + case TIME_WITH_TIME_ZONE: + return "TIME WITH TIME ZONE"; + case ARRAY: + return "ARRAY(" + elementType + ")"; + case NULL: + return "NULL"; + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + public PrestoCompositeDataType getElementType() { + return elementType; + } + + public boolean isOrderable() { + if (dataType == PrestoDataType.ARRAY) { + assert elementType != null; + return elementType.isOrderable(); + } + return dataType.isOrderable(); + } + + } + + public static class PrestoColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + + public PrestoColumn(String name, PrestoCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + public boolean isOrderable() { + return getType().getPrimitiveDataType().isOrderable(); + } + + } + + public static class PrestoTables extends AbstractTables { + + public PrestoTables(List tables) { + super(tables); + } + + } + + public static class PrestoTable extends AbstractRelationalTable { + + public PrestoTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + +} diff --git a/src/sqlancer/presto/PrestoToStringVisitor.java b/src/sqlancer/presto/PrestoToStringVisitor.java new file mode 100644 index 000000000..1342ff635 --- /dev/null +++ b/src/sqlancer/presto/PrestoToStringVisitor.java @@ -0,0 +1,149 @@ +package sqlancer.presto; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.ast.PrestoAtTimeZoneOperator; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoConstant; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoFunctionWithoutParenthesis; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoMultiValuedComparison; +import sqlancer.presto.ast.PrestoQuantifiedComparison; +import sqlancer.presto.ast.PrestoSelect; + +public class PrestoToStringVisitor extends NewToStringVisitor { + + public static String asString(Node expr) { + PrestoToStringVisitor visitor = new PrestoToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof PrestoConstant) { + visit((PrestoConstant) expr); + } else if (expr instanceof PrestoSelect) { + visit((PrestoSelect) expr); + } else if (expr instanceof PrestoJoin) { + visit((PrestoJoin) expr); + } else if (expr instanceof PrestoCastFunction) { + visit((PrestoCastFunction) expr); + } else if (expr instanceof PrestoFunctionWithoutParenthesis) { + visit((PrestoFunctionWithoutParenthesis) expr); + } else if (expr instanceof PrestoAtTimeZoneOperator) { + visit((PrestoAtTimeZoneOperator) expr); + } else if (expr instanceof PrestoMultiValuedComparison) { + visit((PrestoMultiValuedComparison) expr); + } else if (expr instanceof PrestoQuantifiedComparison) { + visit((PrestoQuantifiedComparison) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(PrestoJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getOuterType() != null) { + sb.append(join.getOuterType()); + } + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(PrestoConstant constant) { + sb.append(constant.toString()); + } + + private void visit(PrestoAtTimeZoneOperator timeZoneOperator) { + visit(timeZoneOperator.getExpr()); + sb.append(" AT TIME ZONE "); + sb.append(timeZoneOperator.getTimeZone()); + } + + private void visit(PrestoFunctionWithoutParenthesis prestoFunctionWithoutParenthesis) { + sb.append(prestoFunctionWithoutParenthesis.getExpr()); + } + + private void visit(PrestoSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public void visit(PrestoCastFunction cast) { + sb.append("CAST(("); + visit(cast.getExpr()); + sb.append(") AS "); + sb.append(cast.getType().toString()); + sb.append(")"); + } + + public void visit(PrestoMultiValuedComparison comp) { + sb.append("("); + visit(comp.getLeft()); + sb.append(" "); + sb.append(comp.getOp().getStringRepresentation()); + sb.append(" "); + sb.append(comp.getType()); + sb.append(" (VALUES "); + visit(comp.getRight()); + sb.append(")"); + sb.append(")"); + } + + public void visit(PrestoQuantifiedComparison comp) { + sb.append("("); + visit(comp.getLeft()); + sb.append(" "); + sb.append(comp.getOp().getStringRepresentation()); + sb.append(" "); + sb.append(comp.getType()); + sb.append(" ( "); + visit(comp.getRight()); + sb.append(" ) "); + sb.append(")"); + } +} diff --git a/src/sqlancer/presto/ast/PrestoAggregateFunction.java b/src/sqlancer/presto/ast/PrestoAggregateFunction.java new file mode 100644 index 000000000..0b118393f --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoAggregateFunction.java @@ -0,0 +1,714 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public enum PrestoAggregateFunction implements PrestoFunction { + + // General Aggregate Functions + + // arbitrary(x) → [same as input] + // Returns an arbitrary non-null value of x, if one exists. + ARBITRARY("arbitrary", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return PrestoDataType.getRandomWithoutNull(); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + + // TODO: + // + // array_agg(x) → array<[same as input]># + // Returns an array created from the input x elements. + + // avg(x) → double + // Returns the average (arithmetic mean) of all input values. + AVG("avg", PrestoDataType.FLOAT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { + Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL) }; + } + }, + // avg(time interval type) → time interval type# + // Returns the average interval length of all input values. + AVG_INTERVAL_YM("avg", PrestoDataType.INTERVAL_YEAR_TO_MONTH, PrestoDataType.INTERVAL_YEAR_TO_MONTH), + AVG_INTERVAL_DS("avg", PrestoDataType.INTERVAL_DAY_TO_SECOND, PrestoDataType.INTERVAL_DAY_TO_SECOND), + + // bool_and(boolean) → boolean# + // Returns TRUE if every input value is TRUE, otherwise FALSE. + BOOL_AND("bool_and", PrestoDataType.BOOLEAN, PrestoDataType.BOOLEAN), + // bool_or(boolean) → boolean# + // Returns TRUE if any input value is TRUE, otherwise FALSE. + BOOL_OR("bool_or", PrestoDataType.BOOLEAN, PrestoDataType.BOOLEAN), + // checksum(x) → varbinary# + // Returns an order-insensitive checksum of the given values. + CHECKSUM("checksum", PrestoDataType.VARBINARY) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromList(PrestoDataType.getComparableTypes()) }; + } + }, + + // count(*) → bigint# + // Returns the number of input rows. + COUNT_ALL("count(*)", PrestoDataType.INT), + // count(x) → bigint# + // Returns the number of non-null input values. + COUNT_NOARGS("count", PrestoDataType.INT), COUNT("count", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.getRandomWithoutNull()) }; + } + }, + // count_if(x) → bigint# + // Returns the number of TRUE input values. This function is equivalent to count(CASE WHEN x THEN 1 END). + COUNT_IF("count_if", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.getRandomWithoutNull()) }; + } + }, + // every(boolean) → boolean# + // This is an alias for bool_and(). + EVERY("every", PrestoDataType.BOOLEAN, PrestoDataType.BOOLEAN), + // geometric_mean(x) → double# + // Returns the geometric mean of all input values. + GEOMETRIC_MEAN("geometric_mean", PrestoDataType.FLOAT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { + Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL) }; + } + }, + // max_by(x, y) → [same as x]# + // Returns the value of x associated with the maximum value of y over all input values. + MAX_BY("max_by", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), + Randomly.fromList(PrestoDataType.getOrderableTypes()) }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + + // TODO: + // + // max_by(x, y, n) → array<[same as x]># + // Returns n values of x associated with the n largest of all input values of y in descending order of y. + + // min_by(x, y) → [same as x]# + // Returns the value of x associated with the minimum value of y over all input values. + MIN_BY("min_by", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), + Randomly.fromList(PrestoDataType.getOrderableTypes()) }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + // TODO: + // + // min_by(x, y, n) → array<[same as x]> + // Returns n values of x associated with the n smallest of all input values of y in ascending order of y. + + // max(x) → [same as input] + // Returns the maximum value of all input values. + MAX("max", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + boolean isCompatible = PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + if (returnType.getPrimitiveDataType() == PrestoDataType.ARRAY && returnType.toString().contains("JSON")) { + isCompatible = false; + } + return isCompatible; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public PrestoCompositeDataType getCompositeReturnType() { + PrestoDataType dataType = Randomly.fromList(PrestoDataType.getOrderableTypes()); + PrestoCompositeDataType returnType; + do { + returnType = PrestoCompositeDataType.fromDataType(dataType); + } while (!isCompatibleWithReturnType(returnType)); + return returnType; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, true); + } + + }, + + // TODO: + // + // max(x, n) → array<[same as x]># + // Returns n largest values of all input values of x. + + // min(x) → [same as input]# + // Returns the minimum value of all input values. + MIN("min", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + boolean orderable = PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + if (returnType.getPrimitiveDataType() == PrestoDataType.ARRAY && returnType.toString().contains("JSON")) { + orderable = false; + } + return orderable; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + + // TODO: + // + // min(x, n) → array<[same as x]># + // Returns n smallest values of all input values of x. + + // TODO: + // + // reduce_agg(inputValue T, initialState S, inputFunction(S, T, S), combineFunction(S, S, S)) → S# + // Reduces all input values into a single value. inputFunction will be invoked for each input value. In addition to + // taking the input value, inputFunction takes the current state, initially initialState, and returns the new state. + // combineFunction will be invoked to combine two states into a new state. The final state is returned: + // + // SELECT id, reduce_agg(value, (a, b) -> a + b, (a, b) -> a + b) + // FROM ( + // VALUES + // (1, 2), + // (1, 3), + // (1, 4), + // (2, 20), + // (2, 30), + // (2, 40) + // ) AS t(id, value) + // GROUP BY id; + // -- (1, 9) + // -- (2, 90) + // + // SELECT id, reduce_agg(value, (a, b) -> a * b, (a, b) -> a * b) + // FROM ( + // VALUES + // (1, 2), + // (1, 3), + // (1, 4), + // (2, 20), + // (2, 30), + // (2, 40) + // ) AS t(id, value) + // GROUP BY id; + // -- (1, 24) + // -- (2, 24000) + // The state type must be a boolean, integer, floating-point, or date/time/interval. + + // TODO: + // + // set_agg(x) → array<[same as input]># + // Returns an array created from the distinct input x elements. + + // TODO: + // + // set_union(array(T)) -> array(T)# + // Returns an array of all the distinct values contained in each array of the input + // + // Example: + // + // SELECT set_union(elements) + // FROM ( + // VALUES + // ARRAY[1, 3], + // ARRAY[2, 4] + // ) AS t(elements); + // Returns ARRAY[1, 3, 4] + + // sum(x) → [same as input]# + // Returns the sum of all input values. + SUM("sum", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL) + .contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + // sum(time interval type) → time interval type# + // Returns the average interval length of all input values. + SUM_INTERVAL_YM("sum", PrestoDataType.INTERVAL_YEAR_TO_MONTH, PrestoDataType.INTERVAL_YEAR_TO_MONTH), + SUM_INTERVAL_DS("sum", PrestoDataType.INTERVAL_DAY_TO_SECOND, PrestoDataType.INTERVAL_DAY_TO_SECOND), + + // Bitwise Aggregate Functions# + + // bitwise_and_agg(x) → bigint# + // Returns the bitwise AND of all input values in 2’s complement representation. + BITWISE_AND_AGG("bitwise_and_agg", PrestoDataType.INT, PrestoDataType.INT), + + // bitwise_or_agg(x) → bigint# + // Returns the bitwise OR of all input values in 2’s complement representation. + BITWISE_OR_AGG("bitwise_or_agg", PrestoDataType.INT, PrestoDataType.INT), + + // TODO: + // + // Map Aggregate Functions + + // histogram(x)# + // Returns a map containing the count of the number of times each input value occurs. + // + // map_agg(key, value)# + // Returns a map created from the input key / value pairs. + // + // map_union(x(K, V)) -> map(K, V)# + // Returns the union of all the input maps. If a key is found in multiple input maps, that key’s value in the + // resulting map comes from an arbitrary input map. + // + // map_union_sum(x(K, V)) -> map(K, V)# + // Returns the union of all the input maps summing the values of matching keys in all the maps. All null values in + // the original maps are coalesced to 0. + // + // multimap_agg(key, value)# + // Returns a multimap created from the input key / value pairs. Each key can be associated with multiple values. + + // Approximate Aggregate Functions# + // approx_distinct(x) → bigint# + // Returns the approximate number of distinct input values. This function provides an approximation of + // count(DISTINCT x). + // Zero is returned if all input values are null. + // This function should produce a standard error of 2.3%, which is the standard deviation of the (approximately + // normal) + // error distribution over all possible sets. It does not guarantee an upper bound on the error for any specific + // input set. + APPROX_DISTINCT("approx_distinct", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromList(PrestoDataType.getOrderableTypes()) }; + } + }, + // + // approx_distinct(x, e) → bigint# + // Returns the approximate number of distinct input values. This function provides an approximation of + // count(DISTINCT x). Zero is returned if all input values are null. + // + // This function should produce a standard error of no more than e, which is the standard deviation of the + // (approximately normal) error distribution over all possible sets. It does not guarantee an upper bound on the + // error for any specific input set. The current implementation of this function requires that e be in the range of + // [0.0040625, 0.26000]. + APPROX_DISTINCT_2("approx_distinct", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromList(PrestoDataType.getOrderableTypes()), PrestoDataType.FLOAT }; + } + }, + // approx_percentile(x, percentage) → [same as x]# + // Returns the approximate percentile for all input values of x at the given percentage. + // The value of percentage must be between zero and one and must be constant for all input rows. + APPROX_PERCENTILE("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.FLOAT }; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List> arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + + // approx_percentile(x, percentage, accuracy) → [same as x]# + // As approx_percentile(x, percentage), but with a maximum rank error of accuracy. + // The value of accuracy must be between zero and one (exclusive) and must be constant for all input rows. + // Note that a lower “accuracy” is really a lower error threshold, and thus more accurate. The default accuracy is + // 0.01. + APPROX_PERCENTILE_ACCURACY("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.FLOAT, PrestoDataType.FLOAT }; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List> arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoFloatConstant(0.01D)); + } else { + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + } + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType + .fromDataType(Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT))); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + + // TODO: + // + // approx_percentile(x, percentages) → array<[same as x]># + // Returns the approximate percentile for all input values of x at each of the specified percentages. Each element + // of the percentages array must be between zero and one, and the array must be constant for all input rows. + // + // approx_percentile(x, percentages, accuracy) → array<[same as x]># + // As approx_percentile(x, percentages), but with a maximum rank error of accuracy. + + // approx_percentile(x, w, percentage) → [same as x]# + // Returns the approximate weighed percentile for all input values of x using the per-item weight w at the + // percentage p. + // The weight must be an integer value of at least one. + // It is effectively a replication count for the value x in the percentile set. + // The value of p must be between zero and one and must be constant for all input rows. + APPROX_PERCENTILE_WEIGHT("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.INT, PrestoDataType.FLOAT }; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List> arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoIntConstant(1)); + } else { + arguments.add(new PrestoConstant.PrestoIntConstant(Randomly.smallNumber())); + } + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType + .fromDataType(Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT))); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + + // approx_percentile(x, w, percentage, accuracy) → [same as x]# + // As approx_percentile(x, w, percentage), but with a maximum rank error of accuracy. + APPROX_PERCENTILE_PERCENTAGE_ACCURACY("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.FLOAT }; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List> arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoIntConstant(1)); + } else { + arguments.add(new PrestoConstant.PrestoIntConstant(Randomly.smallNumber())); + } + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoFloatConstant(0.01D)); + } else { + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + } + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType + .fromDataType(Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT))); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }; + + // TODO: + // + // approx_percentile(x, w, percentages) → array<[same as x]># + // Returns the approximate weighed percentile for all input values of x using the per-item weight w at each of the + // given percentages specified in the array. The weight must be an integer value of at least one. It is effectively + // a replication count for the value x in the percentile set. Each element of the array must be between zero and + // one, and the array must be constant for all input rows. + // + // approx_percentile(x, w, percentages, accuracy) → array<[same as x]># + // As approx_percentile(x, w, percentages), but with a maximum rank error of accuracy. + // + // approx_set(x) → HyperLogLog + // See HyperLogLog Functions. + // + // merge(x) → HyperLogLog + // See HyperLogLog Functions. + // + // khyperloglog_agg(x) → KHyperLogLog + // See KHyperLogLog Functions. + + // TODO: + // + // merge(qdigest(T)) -> qdigest(T) + // See Quantile Digest Functions. + // + // qdigest_agg(x) → qdigest<[same as x]> + // See Quantile Digest Functions. + // + // qdigest_agg(x, w) → qdigest<[same as x]> + // See Quantile Digest Functions. + // + // qdigest_agg(x, w, accuracy) → qdigest<[same as x]> + // See Quantile Digest Functions. + // + // numeric_histogram(buckets, value, weight) → map# + // Computes an approximate histogram with up to buckets number of buckets for all values with a per-item weight of + // weight. + // The keys of the returned map are roughly the center of the bin, and the entry is the total weight of the bin. + // The algorithm is based loosely on [BenHaimTomTov2010]. + // + // buckets must be a bigint. value and weight must be numeric. + // + // numeric_histogram(buckets, value) → map# + // Computes an approximate histogram with up to buckets number of buckets for all values. This function is + // equivalent to the variant of numeric_histogram() that takes a weight, with a per-item weight of 1. In this case, + // the total weight in the returned map is the count of items in the bin. + + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoAggregateFunction(String functionName, PrestoDataType returnType) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = new PrestoDataType[0]; + } + + PrestoAggregateFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + public static PrestoAggregateFunction getRandomMetamorphicOracle() { + return Randomly.fromOptions(ARBITRARY, AVG, AVG_INTERVAL_YM, AVG_INTERVAL_DS, BOOL_AND, BOOL_OR, CHECKSUM, + COUNT_ALL, COUNT_NOARGS, COUNT, COUNT_IF, EVERY, GEOMETRIC_MEAN, MAX_BY, MIN_BY, MAX, MIN, SUM, + SUM_INTERVAL_YM, SUM_INTERVAL_DS, BITWISE_AND_AGG, BITWISE_OR_AGG); + } + + public static PrestoAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public static List getFunctionsCompatibleWith(PrestoCompositeDataType returnType) { + return Stream.of(values()).filter(f -> f.isCompatibleWithReturnType(returnType)).collect(Collectors.toList()); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + + @Override + public int getNumberOfArguments() { + return 1; + } + + public List getReturnTypes(PrestoSchema.PrestoDataType dataType) { + return Collections.singletonList(dataType); + } + + public PrestoDataType getReturnType() { + if (returnType == null) { + return PrestoDataType.getRandomWithoutNull(); + } + return returnType; + } + + public PrestoCompositeDataType getCompositeReturnType() { + PrestoDataType dataType = getReturnType(); + return PrestoCompositeDataType.fromDataType(dataType); + } +} diff --git a/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java new file mode 100644 index 000000000..7bb01bb98 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java @@ -0,0 +1,22 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.Node; + +public class PrestoAtTimeZoneOperator implements Node { + + private final Node expr; + private final Node timeZone; + + public PrestoAtTimeZoneOperator(Node expr, Node timeZone) { + this.expr = expr; + this.timeZone = timeZone; + } + + public Node getExpr() { + return expr; + } + + public Node getTimeZone() { + return timeZone; + } +} diff --git a/src/sqlancer/presto/ast/PrestoCastFunction.java b/src/sqlancer/presto/ast/PrestoCastFunction.java new file mode 100644 index 000000000..89b139314 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoCastFunction.java @@ -0,0 +1,24 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; + +public class PrestoCastFunction implements Node { + + private final Node expr; + private final PrestoSchema.PrestoCompositeDataType type; + + public PrestoCastFunction(Node expr, PrestoSchema.PrestoCompositeDataType type) { + this.expr = expr; + this.type = type; + } + + public Node getExpr() { + return expr; + } + + public PrestoSchema.PrestoCompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoColumnReference.java b/src/sqlancer/presto/ast/PrestoColumnReference.java new file mode 100644 index 000000000..4f96a013d --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.presto.PrestoSchema; + +public class PrestoColumnReference extends ColumnReferenceNode { + + public PrestoColumnReference(PrestoSchema.PrestoColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoComparisonFunction.java b/src/sqlancer/presto/ast/PrestoComparisonFunction.java new file mode 100644 index 000000000..1125c3e35 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoComparisonFunction.java @@ -0,0 +1,81 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; + +public enum PrestoComparisonFunction implements PrestoFunction { + + // comparison + + // Returns the largest of the provided values. + // → [same as input] + GREATEST("greatest", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return -1; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + }, + // Returns the smallest of the provided values. + // → [same as input]# + LEAST("least", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return -1; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + ArrayList prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + }; + + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoComparisonFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoConditionalFunction.java b/src/sqlancer/presto/ast/PrestoConditionalFunction.java new file mode 100644 index 000000000..e603a7e5d --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoConditionalFunction.java @@ -0,0 +1,95 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; + +public enum PrestoConditionalFunction implements PrestoFunction { + + // Conditional functions + IF_TRUE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { PrestoDataType.BOOLEAN, returnType.getPrimitiveDataType() }; + } + }, + + IF_TRUE_FALSE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { PrestoDataType.BOOLEAN, returnType.getPrimitiveDataType(), + returnType.getPrimitiveDataType() }; + } + }, + + NULLIF("nullif", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), returnType.getPrimitiveDataType() }; + } + }, + + COALESCE("coalesce", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public int getNumberOfArguments() { + return -1; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + List prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + }; + + private final PrestoDataType returnType; + private final String functionName; + + PrestoConditionalFunction(String functionName, PrestoDataType returnType) { + this.functionName = functionName; + this.returnType = returnType; + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public int getNumberOfArguments() { + return getArgumentTypes(PrestoCompositeDataType.fromDataType(returnType)).length; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoConstant.java b/src/sqlancer/presto/ast/PrestoConstant.java new file mode 100644 index 000000000..97b115a4e --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoConstant.java @@ -0,0 +1,810 @@ +package sqlancer.presto.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoConstantUtils; +import sqlancer.presto.PrestoSchema; + +public abstract class PrestoConstant implements Node, PrestoExpression { + + private static final String[] TIME_ZONES = { "Africa/Abidjan", "Africa/Accra", "Africa/Addis_Ababa", + "Africa/Algiers", "Africa/Asmara", "Africa/Asmera", "Africa/Bamako", "Africa/Bangui", "Africa/Banjul", + "Africa/Bissau", "Africa/Blantyre", "Africa/Brazzaville", "Africa/Bujumbura", "Africa/Cairo", + "Africa/Casablanca", "Africa/Ceuta", "Africa/Conakry", "Africa/Dakar", "Africa/Dar_es_Salaam", + "Africa/Djibouti", "Africa/Douala", "Africa/El_Aaiun", "Africa/Freetown", "Africa/Gaborone", + "Africa/Harare", "Africa/Johannesburg", "Africa/Juba", "Africa/Kampala", "Africa/Khartoum", "Africa/Kigali", + "Africa/Kinshasa", "Africa/Lagos", "Africa/Libreville", "Africa/Lome", "Africa/Luanda", "Africa/Lubumbashi", + "Africa/Lusaka", "Africa/Malabo", "Africa/Maputo", "Africa/Maseru", "Africa/Mbabane", "Africa/Mogadishu", + "Africa/Monrovia", "Africa/Nairobi", "Africa/Ndjamena", "Africa/Niamey", "Africa/Nouakchott", + "Africa/Ouagadougou", "Africa/Porto-Novo", "Africa/Sao_Tome", "Africa/Timbuktu", "Africa/Tripoli", + "Africa/Tunis", "Africa/Windhoek", "America/Adak", "America/Anchorage", "America/Anguilla", + "America/Antigua", "America/Araguaina", "America/Argentina/Buenos_Aires", "America/Argentina/Catamarca", + "America/Argentina/ComodRivadavia", "America/Argentina/Cordoba", "America/Argentina/Jujuy", + "America/Argentina/La_Rioja", "America/Argentina/Mendoza", "America/Argentina/Rio_Gallegos", + "America/Argentina/Salta", "America/Argentina/San_Juan", "America/Argentina/San_Luis", + "America/Argentina/Tucuman", "America/Argentina/Ushuaia", "America/Aruba", "America/Asuncion", + "America/Atikokan", "America/Atka", "America/Bahia", "America/Barbados", "America/Belem", "America/Belize", + "America/Blanc-Sablon", "America/Boa_Vista", "America/Bogota", "America/Boise", "America/Buenos_Aires", + "America/Cambridge_Bay", "America/Campo_Grande", "America/Cancun", "America/Caracas", "America/Catamarca", + "America/Cayenne", "America/Cayman", "America/Chicago", "America/Chihuahua", "America/Coral_Harbour", + "America/Cordoba", "America/Costa_Rica", "America/Creston", "America/Cuiaba", "America/Curacao", + "America/Danmarkshavn", "America/Dawson", "America/Dawson_Creek", "America/Denver", "America/Detroit", + "America/Dominica", "America/Edmonton", "America/Eirunepe", "America/El_Salvador", "America/Ensenada", + "America/Fort_Nelson", "America/Fort_Wayne", "America/Fortaleza", "America/Glace_Bay", "America/Godthab", + "America/Goose_Bay", "America/Grand_Turk", "America/Grenada", "America/Guadeloupe", "America/Guatemala", + "America/Guayaquil", "America/Guyana", "America/Halifax", "America/Havana", "America/Hermosillo", + "America/Indiana/Indianapolis", "America/Indiana/Knox", "America/Indiana/Marengo", + "America/Indiana/Petersburg", "America/Indiana/Tell_City", "America/Indiana/Vevay", + "America/Indiana/Vincennes", "America/Indiana/Winamac", "America/Indianapolis", "America/Inuvik", + "America/Iqaluit", "America/Jamaica", "America/Jujuy", "America/Juneau", "America/Kentucky/Louisville", + "America/Kentucky/Monticello", "America/Knox_IN", "America/Kralendijk", "America/La_Paz", "America/Lima", + "America/Los_Angeles", "America/Louisville", "America/Lower_Princes", "America/Maceio", "America/Managua", + "America/Manaus", "America/Marigot", "America/Martinique", "America/Matamoros", "America/Mendoza", + "America/Menominee", "America/Merida", "America/Metlakatla", "America/Mexico_City", "America/Miquelon", + "America/Moncton", "America/Monterrey", "America/Montevideo", "America/Montreal", "America/Montserrat", + "America/Nassau", "America/New_York", "America/Nipigon", "America/Nome", "America/Noronha", + "America/North_Dakota/Beulah", "America/North_Dakota/Center", "America/North_Dakota/New_Salem", + "America/Nuuk", "America/Ojinaga", "America/Panama", "America/Pangnirtung", "America/Paramaribo", + "America/Phoenix", "America/Port-au-Prince", "America/Port_of_Spain", "America/Porto_Acre", + "America/Porto_Velho", "America/Puerto_Rico", "America/Punta_Arenas", "America/Rainy_River", + "America/Rankin_Inlet", "America/Recife", "America/Regina", "America/Resolute", "America/Rio_Branco", + "America/Rosario", "America/Santa_Isabel", "America/Santarem", "America/Santiago", "America/Santo_Domingo", + "America/Sao_Paulo", "America/Scoresbysund", "America/Shiprock", "America/Sitka", "America/St_Barthelemy", + "America/St_Johns", "America/St_Kitts", "America/St_Lucia", "America/St_Thomas", "America/St_Vincent", + "America/Swift_Current", "America/Tegucigalpa", "America/Thule", "America/Thunder_Bay", "America/Tijuana", + "America/Toronto", "America/Tortola", "America/Vancouver", "America/Virgin", "America/Whitehorse", + "America/Winnipeg", "America/Yakutat", "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis", + "Antarctica/DumontDUrville", "Antarctica/Macquarie", "Antarctica/Mawson", "Antarctica/McMurdo", + "Antarctica/Palmer", "Antarctica/Rothera", "Antarctica/South_Pole", "Antarctica/Syowa", "Antarctica/Troll", + "Antarctica/Vostok", "Arctic/Longyearbyen", "Asia/Aden", "Asia/Almaty", "Asia/Amman", "Asia/Anadyr", + "Asia/Aqtau", "Asia/Aqtobe", "Asia/Ashgabat", "Asia/Ashkhabad", "Asia/Atyrau", "Asia/Baghdad", + "Asia/Bahrain", "Asia/Baku", "Asia/Bangkok", "Asia/Barnaul", "Asia/Beirut", "Asia/Bishkek", "Asia/Brunei", + "Asia/Calcutta", "Asia/Chita", "Asia/Choibalsan", "Asia/Chongqing", "Asia/Chungking", "Asia/Colombo", + "Asia/Dacca", "Asia/Dhaka", "Asia/Dili", "Asia/Dubai", "Asia/Dushanbe", "Asia/Famagusta", "Asia/Gaza", + "Asia/Harbin", "Asia/Hebron", "Asia/Ho_Chi_Minh", "Asia/Hong_Kong", "Asia/Hovd", "Asia/Irkutsk", + "Asia/Istanbul", "Asia/Jakarta", "Asia/Jayapura", "Asia/Jerusalem", "Asia/Kabul", "Asia/Kamchatka", + "Asia/Karachi", "Asia/Kashgar", "Asia/Kathmandu", "Asia/Katmandu", "Asia/Khandyga", "Asia/Kolkata", + "Asia/Krasnoyarsk", "Asia/Kuala_Lumpur", "Asia/Kuching", "Asia/Kuwait", "Asia/Macao", "Asia/Macau", + "Asia/Magadan", "Asia/Makassar", "Asia/Manila", "Asia/Muscat", "Asia/Nicosia", "Asia/Novokuznetsk", + "Asia/Novosibirsk", "Asia/Omsk", "Asia/Oral", "Asia/Phnom_Penh", "Asia/Pontianak", "Asia/Pyongyang", + "Asia/Qatar", "Asia/Qostanay", "Asia/Qyzylorda", "Asia/Rangoon", "Asia/Riyadh", "Asia/Saigon", + "Asia/Sakhalin", "Asia/Samarkand", "Asia/Seoul", "Asia/Shanghai", "Asia/Singapore", "Asia/Srednekolymsk", + "Asia/Taipei", "Asia/Tashkent", "Asia/Tbilisi", "Asia/Tehran", "Asia/Tel_Aviv", "Asia/Thimbu", + "Asia/Thimphu", "Asia/Tokyo", "Asia/Tomsk", "Asia/Ujung_Pandang", "Asia/Ulaanbaatar", "Asia/Ulan_Bator", + "Asia/Urumqi", "Asia/Ust-Nera", "Asia/Vientiane", "Asia/Vladivostok", "Asia/Yakutsk", "Asia/Yangon", + "Asia/Yekaterinburg", "Asia/Yerevan", "Atlantic/Azores", "Atlantic/Bermuda", "Atlantic/Canary", + "Atlantic/Cape_Verde", "Atlantic/Faeroe", "Atlantic/Faroe", "Atlantic/Jan_Mayen", "Atlantic/Madeira", + "Atlantic/Reykjavik", "Atlantic/South_Georgia", "Atlantic/St_Helena", "Atlantic/Stanley", "Australia/ACT", + "Australia/Adelaide", "Australia/Brisbane", "Australia/Broken_Hill", "Australia/Canberra", + "Australia/Currie", "Australia/Darwin", "Australia/Eucla", "Australia/Hobart", "Australia/LHI", + "Australia/Lindeman", "Australia/Lord_Howe", "Australia/Melbourne", "Australia/North", "Australia/Perth", + "Australia/Queensland", "Australia/South", "Australia/Sydney", "Australia/Tasmania", "Australia/Victoria", + "Australia/West", "Australia/Yancowinna", "Brazil/Acre", "Brazil/DeNoronha", "Brazil/East", "Brazil/West", + "CET", "CST6CDT", "Canada/Atlantic", "Canada/Central", "Canada/Eastern", "Canada/Mountain", + "Canada/Newfoundland", "Canada/Pacific", "Canada/Saskatchewan", "Canada/Yukon", "Chile/Continental", + "Chile/EasterIsland", "Cuba", "EET", "EST5EDT", "Egypt", "Eire", "Etc/GMT", "Etc/GMT+0", "Etc/GMT+1", + "Etc/GMT+10", "Etc/GMT+11", "Etc/GMT+12", "Etc/GMT+2", "Etc/GMT+3", "Etc/GMT+4", "Etc/GMT+5", "Etc/GMT+6", + "Etc/GMT+7", "Etc/GMT+8", "Etc/GMT+9", "Etc/GMT-0", "Etc/GMT-1", "Etc/GMT-10", "Etc/GMT-11", "Etc/GMT-12", + "Etc/GMT-13", "Etc/GMT-14", "Etc/GMT-2", "Etc/GMT-3", "Etc/GMT-4", "Etc/GMT-5", "Etc/GMT-6", "Etc/GMT-7", + "Etc/GMT-8", "Etc/GMT-9", "Etc/GMT0", "Etc/Greenwich", "Etc/UCT", "Etc/UTC", "Etc/Universal", "Etc/Zulu", + "Europe/Amsterdam", "Europe/Andorra", "Europe/Astrakhan", "Europe/Athens", "Europe/Belfast", + "Europe/Belgrade", "Europe/Berlin", "Europe/Bratislava", "Europe/Brussels", "Europe/Bucharest", + "Europe/Budapest", "Europe/Busingen", "Europe/Chisinau", "Europe/Copenhagen", "Europe/Dublin", + "Europe/Gibraltar", "Europe/Guernsey", "Europe/Helsinki", "Europe/Isle_of_Man", "Europe/Istanbul", + "Europe/Jersey", "Europe/Kaliningrad", "Europe/Kiev", "Europe/Kirov", "Europe/Lisbon", "Europe/Ljubljana", + "Europe/London", "Europe/Luxembourg", "Europe/Madrid", "Europe/Malta", "Europe/Mariehamn", "Europe/Minsk", + "Europe/Monaco", "Europe/Moscow", "Europe/Nicosia", "Europe/Oslo", "Europe/Paris", "Europe/Podgorica", + "Europe/Prague", "Europe/Riga", "Europe/Rome", "Europe/Samara", "Europe/San_Marino", "Europe/Sarajevo", + "Europe/Saratov", "Europe/Simferopol", "Europe/Skopje", "Europe/Sofia", "Europe/Stockholm", "Europe/Tirane", + "Europe/Tiraspol", "Europe/Ulyanovsk", "Europe/Uzhgorod", "Europe/Vaduz", "Europe/Vatican", "Europe/Vienna", + "Europe/Vilnius", "Europe/Volgograd", "Europe/Warsaw", "Europe/Zagreb", "Europe/Zaporozhye", + "Europe/Zurich", "GB", "GB-Eire", "GMT", "GMT0", "Greenwich", "Hongkong", "Iceland", "Indian/Antananarivo", + "Indian/Chagos", "Indian/Christmas", "Indian/Cocos", "Indian/Comoro", "Indian/Kerguelen", "Indian/Mahe", + "Indian/Maldives", "Indian/Mauritius", "Indian/Mayotte", "Indian/Reunion", "Iran", "Israel", "Jamaica", + "Japan", "Kwajalein", "Libya", "MET", "MST7MDT", "Mexico/General", "NZ", "NZ-CHAT", "Navajo", "PRC", + "PST8PDT", "Pacific/Apia", "Pacific/Auckland", "Pacific/Bougainville", "Pacific/Chatham", "Pacific/Chuuk", + "Pacific/Easter", "Pacific/Efate", "Pacific/Enderbury", "Pacific/Fakaofo", "Pacific/Fiji", + "Pacific/Funafuti", "Pacific/Galapagos", "Pacific/Gambier", "Pacific/Guadalcanal", "Pacific/Guam", + "Pacific/Honolulu", "Pacific/Johnston", "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein", + "Pacific/Majuro", "Pacific/Marquesas", "Pacific/Midway", "Pacific/Nauru", "Pacific/Niue", "Pacific/Norfolk", + "Pacific/Noumea", "Pacific/Pago_Pago", "Pacific/Palau", "Pacific/Pitcairn", "Pacific/Pohnpei", + "Pacific/Ponape", "Pacific/Port_Moresby", "Pacific/Rarotonga", "Pacific/Saipan", "Pacific/Samoa", + "Pacific/Tahiti", "Pacific/Tarawa", "Pacific/Tongatapu", "Pacific/Truk", "Pacific/Wake", "Pacific/Wallis", + "Pacific/Yap", "Poland", "Portugal", "ROK", "Singapore", "Turkey", "UCT", "US/Alaska", "US/Aleutian", + "US/Arizona", "US/Central", "US/East-Indiana", "US/Eastern", "US/Hawaii", "US/Indiana-Starke", + "US/Michigan", "US/Mountain", "US/Pacific", "US/Samoa", "UTC", "Universal", "W-SU", "WET", "Zulu" }; + private static final String FALSE = "false"; + private static final String TRUE = "true"; + + private PrestoConstant() { + } + + public static Node createStringConstant(String text) { + return new PrestoTextConstant(text); + } + + public static Node createStringConstant(String text, int size) { + return new PrestoTextConstant(text, size); + } + + public static Node createJsonConstant() { + return new PrestoJsonConstant(); + } + + public static Node createFloatConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + assert type.getSize() == 4; + float floatValue = (float) val; + return new PrestoFloatConstant(floatValue); + } + + public static Node createFloatConstant(double val) { + return new PrestoFloatConstant(val); + } + + public static Node createDecimalConstant(double val) { + return new PrestoDecimalConstant(val); + } + + public static Node createDecimalConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + int scale = type.getScale(); + int precision = type.getSize(); + BigDecimal finalBD = PrestoConstantUtils.getDecimal(val, scale, precision); + return new PrestoDecimalConstant(finalBD.doubleValue()); + } + + public static Node createIntConstant(long val) { + return new PrestoIntConstant(val); + } + + public static Node createIntConstant(PrestoSchema.PrestoCompositeDataType type, long val, + boolean castInteger) { + PrestoIntConstant intConstant; + assert List.of(1, 2, 4, 8).contains(type.getSize()); + switch (type.getSize()) { + case 1: + intConstant = new PrestoIntConstant((byte) val); + break; + case 2: + intConstant = new PrestoIntConstant((short) val); + break; + case 4: + intConstant = new PrestoIntConstant((int) val); + break; + default: + intConstant = new PrestoIntConstant(val); + } + if (castInteger) { + return new PrestoCastFunction(intConstant, type); + } else { + return intConstant; + } + } + + public static Node createNullConstant() { + return new PrestoNullConstant(); + } + + public static Node createBooleanConstant(boolean val) { + return new PrestoBooleanConstant(val); + } + + public static Node createDateConstant(long integer) { + return new PrestoDateConstant(integer); + } + + public static Node createTimeConstant(long integer) { + return new PrestoTimeConstant(integer); + } + + public static Node createTimeWithTimeZoneConstant(long integer) { + return new PrestoTimeWithTimeZoneConstant(integer); + } + + public static Node createTimestampWithTimeZoneConstant(long integer) { + return new PrestoTimestampWithTimezoneConstant(integer); + } + + public static Node createIntervalDayToSecond(long integer) { + return new PrestoIntervalDayToSecondConstant(); + } + + public static Node createIntervalYearToMonth(long integer) { + return new PrestoIntervalYearToMonthConstant(); + } + + public static Node createTimestampConstant(long integer) { + return new PrestoTimestampConstant(integer); + } + + public static Node createVarbinaryConstant(String string) { + return new PrestoVarbinaryConstant(string); + } + + public static Node createTimezoneConstant() { + String string = Randomly.fromOptions(TIME_ZONES); + return new PrestoTextConstant(string); + } + + public static Node createArrayConstant(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); + long size = Randomly.getNotCachedInteger(0, 10); + + List> elements = new ArrayList<>(); + for (int i = 0; i <= size; i++) { + if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + elements.add(createArrayConstant(elementType)); + } else { + elements.add(generateConstant(elementType, false)); + } + } + return new PrestoArrayConstant(elements); + } + + public static Node createMapConstant(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); + long size = Randomly.getNotCachedInteger(0, 10); + + List> elements = new ArrayList<>(); + for (int i = 0; i <= size; i++) { + if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + elements.add(createArrayConstant(elementType)); + } else { + elements.add(generateConstant(elementType, false)); + } + } + return new PrestoArrayConstant(elements); + } + + public static Node generateConstant(PrestoSchema.PrestoCompositeDataType type, + boolean castInteger) { + Randomly randomly = new Randomly(); + switch (type.getPrimitiveDataType()) { + case ARRAY: + return PrestoConstant.createArrayConstant(type); + case NULL: + return PrestoConstant.createNullConstant(); + case CHAR: + return PrestoConstant.PrestoTextConstant.createStringConstant(randomly.getAlphabeticChar(), type.getSize()); + case VARCHAR: + return PrestoConstant.PrestoTextConstant.createStringConstant(randomly.getString(), type.getSize()); + case VARBINARY: + return PrestoConstant.createVarbinaryConstant(randomly.getString()); + case JSON: + return PrestoConstant.PrestoJsonConstant.createJsonConstant(); + case TIME: + return PrestoConstant.createTimeConstant(randomly.getLong(0, System.currentTimeMillis())); + case TIME_WITH_TIME_ZONE: + return PrestoConstant.createTimeWithTimeZoneConstant(randomly.getLong(0, System.currentTimeMillis())); + case TIMESTAMP: + return PrestoConstant.createTimestampConstant(randomly.getLong(0, System.currentTimeMillis())); + case TIMESTAMP_WITH_TIME_ZONE: + return PrestoConstant.createTimestampWithTimeZoneConstant(randomly.getLong(0, System.currentTimeMillis())); + case INTERVAL_YEAR_TO_MONTH: + return PrestoConstant.createIntervalYearToMonth(randomly.getLong(0, System.currentTimeMillis())); + case INTERVAL_DAY_TO_SECOND: + return PrestoConstant.createIntervalDayToSecond(randomly.getLong(0, System.currentTimeMillis())); + case INT: + return PrestoConstant.PrestoIntConstant.createIntConstant(type, Randomly.getNonCachedInteger(), + castInteger); + case FLOAT: + return PrestoConstant.PrestoFloatConstant.createFloatConstant(randomly.getDouble()); + case BOOLEAN: + return PrestoConstant.PrestoBooleanConstant.createBooleanConstant(Randomly.getBoolean()); + case DATE: + return PrestoConstant.createDateConstant(randomly.getLong(0, System.currentTimeMillis())); + case DECIMAL: + return PrestoConstant.createDecimalConstant(type, randomly.getLong(0, System.currentTimeMillis())); + default: + throw new AssertionError("Unknown type: " + type); + } + } + + public boolean isNull() { + return false; + } + + public boolean isInt() { + return false; + } + + public boolean isBoolean() { + return false; + } + + public boolean isArray() { + return false; + } + + public boolean isString() { + return false; + } + + public boolean isFloat() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asFloat() { + throw new UnsupportedOperationException(this.toString()); + } + + public static class PrestoNullConstant extends PrestoConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + } + + public static class PrestoIntConstant extends PrestoConstant { + + private final long value; + + public PrestoIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + @Override + public boolean isInt() { + return true; + } + + } + + public static class PrestoFloatConstant extends PrestoConstant { + + private final double value; + + public PrestoFloatConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "infinity()"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-infinity()"; + } + return String.valueOf(value); + } + + @Override + public boolean isFloat() { + return true; + } + + @Override + public double asFloat() { + return value; + } + + } + + public static class PrestoDecimalConstant extends PrestoConstant { + + private static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("###0.0000"); + + private final double value; + + public PrestoDecimalConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return DECIMAL_FORMAT.format(value); + } + + @Override + public double asFloat() { + return value; + } + + } + + public static class PrestoTextConstant extends PrestoConstant { + + private final String value; + + public PrestoTextConstant(String value) { + this.value = value; + } + + public PrestoTextConstant(String value, int size) { + this.value = value.substring(0, Math.min(value.length(), size)); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static class PrestoVarbinaryConstant extends PrestoConstant { + + private final String value; + + public PrestoVarbinaryConstant(String value) { + this.value = value.replace("'", ""); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return String.format("CAST ('%s' AS VARBINARY)", value); + } + + } + + public static class PrestoJsonConstant extends PrestoConstant { + + private final String value; + + public PrestoJsonConstant() { + Randomly rand = new Randomly(); + JsonValueType jvt = Randomly.fromOptions(JsonValueType.values()); + String val; + switch (jvt) { + case NULL: + val = "null"; + value = "{\"val\":" + val + "}"; + break; + case FALSE: + val = FALSE; + value = "{\"val\":" + val + "}"; + break; + case TRUE: + val = TRUE; + value = "{\"val\":" + val + "}"; + break; + case STRING: + String randString = rand.getString(); + String string = randString.substring(0, Math.min(randString.length(), 250)); + string = string.replace("'", ""); + // https://www.rfc-editor.org/rfc/rfc8259#page-8 + string = PrestoConstantUtils.removeAllControlChars(string); + string = string.replace("\\", "\\\\"); + + value = "{\"val\": \"" + string + "\"}"; + break; + case NUMBER: + if (Randomly.getBoolean()) { + int no = (int) rand.getInteger(); + val = String.valueOf(no); + } else { + double no = rand.getDouble(); + val = String.valueOf(no); + } + value = "{\"val\": " + val + "}"; + break; + case ARRAY: + value = "{\"employees\":[\"John\", \"Anna\", \"Peter\"]}"; + break; + case OBJECT: + value = "{\"employee\":{\"name\":\"John\", \"age\":30, \"city\":\"New York\"}}"; + break; + default: + value = "{}"; + } + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "JSON '" + value + "'"; + } + + private enum JsonValueType { + OBJECT, ARRAY, NUMBER, STRING, TRUE, FALSE, NULL + } + + } + + public static class PrestoDateConstant extends PrestoConstant { + + private final String textRepresentation; + + public PrestoDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepresentation); + } + + } + + public static class PrestoTimeConstant extends PrestoConstant { + + public final String textRepresentation; + + public PrestoTimeConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS"); + textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIME '%s'", textRepresentation); + } + + } + + public static class PrestoTimeWithTimeZoneConstant extends PrestoConstant { + + private final String textRepresentation; + private final String timeZone; + + public PrestoTimeWithTimeZoneConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS"); + textRepresentation = dateFormat.format(timestamp); + this.timeZone = Randomly.fromOptions(TIME_ZONES); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIME '%s %s'", textRepresentation, timeZone); + } + + } + + public static class PrestoTimestampConstant extends PrestoConstant { + + private final String textRepresentation; + + public PrestoTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + this.textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepresentation); + } + + } + + public static class PrestoTimestampWithTimezoneConstant extends PrestoConstant { + + private final String textRepresentation; + private final String timeZone; + + public PrestoTimestampWithTimezoneConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + this.textRepresentation = dateFormat.format(timestamp); + this.timeZone = Randomly.fromOptions(TIME_ZONES); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s %s'", textRepresentation, timeZone); + } + + } + + public static class PrestoIntervalDayToSecondConstant extends PrestoConstant { + + private final String textRepresentation; + private final Interval fromInterval; + + public PrestoIntervalDayToSecondConstant() { + this.fromInterval = Randomly.fromOptions(Interval.values()); + SimpleDateFormat dateFormat = new SimpleDateFormat("dd HH:mm:ss"); + switch (fromInterval) { + case DAY: + dateFormat = new SimpleDateFormat("dd"); + break; + case HOUR: + dateFormat = new SimpleDateFormat("HH"); + break; + case MINUTE: + dateFormat = new SimpleDateFormat("mm"); + break; + case SECOND: + dateFormat = new SimpleDateFormat("ss"); + break; + default: + break; + } + + Randomly rand = new Randomly(); + + Timestamp timestamp = new Timestamp(rand.getLong(0, System.currentTimeMillis())); + this.textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + // if (toInterval == null) { + return String.format("INTERVAL '%s' %s", textRepresentation, fromInterval.name()); + // } else { + // return String.format("INTERVAL '%s' %s TO %s", textRepresentation, fromInterval, toInterval); + // } + } + + private enum Interval { + DAY, HOUR, MINUTE, SECOND + } + + } + + public static class PrestoIntervalYearToMonthConstant extends PrestoConstant { + + public String textRepresentation; + private final Interval fromInterval; + + public PrestoIntervalYearToMonthConstant() { + fromInterval = Randomly.fromOptions(Interval.values()); + SimpleDateFormat dateFormat; + switch (fromInterval) { + case YEAR: + dateFormat = new SimpleDateFormat("yyyy"); + break; + case MONTH: + dateFormat = new SimpleDateFormat("MM"); + break; + default: + dateFormat = new SimpleDateFormat("yyyy-MM"); + } + + Randomly rand = new Randomly(); + + Timestamp timestamp = new Timestamp(rand.getLong(0, System.currentTimeMillis())); + textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("INTERVAL '%s' %s", textRepresentation, fromInterval.name()); + } + + private enum Interval { + YEAR, MONTH + } + + } + + public static class PrestoBooleanConstant extends PrestoConstant { + + private final boolean value; + + public PrestoBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + } + + public static class PrestoArrayConstant extends PrestoConstant { + + private final List> elements; + + public PrestoArrayConstant(List> elements) { + this.elements = new ArrayList<>(elements); + } + + @Override + public boolean isArray() { + return true; + } + + @Override + public String toString() { + return "ARRAY[" + elements.stream().map(Object::toString).collect(Collectors.joining(", ")) + "]"; + } + + } + +} diff --git a/src/sqlancer/presto/ast/PrestoDateFunction.java b/src/sqlancer/presto/ast/PrestoDateFunction.java new file mode 100644 index 000000000..a8e00a3c0 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoDateFunction.java @@ -0,0 +1,523 @@ +package sqlancer.presto.ast; + +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; + +public enum PrestoDateFunction implements PrestoFunction { + + // Date and Time Functions# + // Returns the current date as of the start of the query. + CURRENT_DATE("current_date", PrestoDataType.DATE), + + // Returns the current time as of the start of the query. + CURRENT_TIME("current_time", PrestoDataType.TIME_WITH_TIME_ZONE), + + // Returns the current timestamp as of the start of the query. + CURRENT_TIMESTAMP("current_timestamp", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // Returns the current time zone in the format defined by IANA (e.g., America/Los_Angeles) or as fixed offset from + // UTC (e.g., +08:35) + CURRENT_TIMEZONE("current_timezone", PrestoDataType.VARCHAR), + + // This is an alias for CAST(x AS date). + DATE("date", PrestoDataType.DATE, PrestoDataType.DATE, PrestoDataType.INT, PrestoDataType.VARCHAR), + + // Returns the last day of the month. + LAST_DAY_OF_MONTH("last_day_of_month", PrestoDataType.DATE, PrestoDataType.DATE), + + // Parses the ISO 8601 formatted string into a timestamp with time zone. + FROM_ISO8601_TIMESTAMP("from_iso8601_timestamp", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.VARCHAR), + + // Parses the ISO 8601 formatted string into a date. + FROM_ISO8601_DATE("from_iso8601_date", PrestoDataType.DATE, PrestoDataType.VARCHAR), + + // Returns the UNIX timestamp unixtime as a timestamp. + FROM_UNIXTIME("from_unixtime", PrestoDataType.TIMESTAMP, PrestoDataType.INT), + + // Returns the UNIX timestamp unixtime as a timestamp with time zone using string for the time zone. + FROM_UNIXTIME_TIMEZONE("from_unixtime", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.INT, + PrestoDataType.VARCHAR) { + @Override + public boolean shouldPreserveOrderOfArguments() { + return true; + } + }, + + // Returns the UNIX timestamp unixtime as a timestamp with time zone using hours and minutes for the time zone + // offset. + FROM_UNIXTIME_HOURS_MINUTES("from_unixtime", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.INT, + PrestoDataType.INT) { + @Override + public boolean shouldPreserveOrderOfArguments() { + return true; + } + }, + + // Returns the current time as of the start of the query. -> time + LOCALTIME("localtime", PrestoDataType.TIME), + + // Returns the current timestamp as of the start of the query. -> timestamp + LOCALTIMESTAMP("localtimestamp", PrestoDataType.TIMESTAMP), + + // This is an alias for current_timestamp. → timestamp with time zone# + NOW("now", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // Formats x as an ISO 8601 string. x can be date, timestamp, or timestamp with time zone. → varchar# + TO_ISO8601("to_iso8601", PrestoDataType.VARCHAR, PrestoDataType.DATE, PrestoDataType.TIMESTAMP, + PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // Returns the day-to-second interval as milliseconds. → bigint# + TO_MILLISECONDS("to_milliseconds", PrestoDataType.INT, PrestoDataType.INTERVAL_DAY_TO_SECOND), + TO_MILLISECONDS_2("to_milliseconds", PrestoDataType.INT, PrestoDataType.INTERVAL_YEAR_TO_MONTH), + + // Returns timestamp as a UNIX timestamp. → double# + TO_UNIXTIME("to_unixtime", PrestoDataType.FLOAT, PrestoDataType.TIMESTAMP), + TO_UNIXTIME_2("to_unixtime", PrestoDataType.FLOAT, PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // The following SQL-standard functions do not use parenthesis: + CURRENT_DATE_NA("current_date", PrestoDataType.DATE) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + CURRENT_TIME_NA("current_time", PrestoDataType.TIME) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + CURRENT_TIMESTAMP_NA("current_timestamp", PrestoDataType.TIMESTAMP) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + LOCALTIME_NA("localtime", PrestoDataType.TIME) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + LOCALTIMESTAMP_NA("localtimestamp", PrestoDataType.TIMESTAMP) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + // Truncation Function + // date_trunc(unit, x) → [same as input] + DATE_TRUNC_1("date_trunc", PrestoDataType.TIMESTAMP, PrestoDataType.VARCHAR, PrestoDataType.TIMESTAMP), + DATE_TRUNC_2("date_trunc", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.VARCHAR, + PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + DATE_TRUNC_3("date_trunc", PrestoDataType.DATE, PrestoDataType.VARCHAR, PrestoDataType.DATE), + DATE_TRUNC_4("date_trunc", PrestoDataType.TIME, PrestoDataType.VARCHAR, PrestoDataType.TIME); + + /* + * + * Interval Functions# The functions in this section support the following interval units: + * + * Unit + * + * Description + * + * millisecond + * + * Milliseconds + * + * second + * + * Seconds + * + * minute + * + * Minutes + * + * hour + * + * Hours + * + * day + * + * Days + * + * week + * + * Weeks + * + * month + * + * Months + * + * quarter + * + * Quarters of a year + * + * year + * + * Years + * + * date_add(unit, value, timestamp) → [same as input]# Adds an interval value of type unit to timestamp. Subtraction + * can be performed by using a negative value. + * + * date_diff(unit, timestamp1, timestamp2) → bigint# Returns timestamp2 - timestamp1 expressed in terms of unit. + * + * Duration Function# The parse_duration function supports the following units: + * + * Unit + * + * Description + * + * ns + * + * Nanoseconds + * + * us + * + * Microseconds + * + * ms + * + * Milliseconds + * + * s + * + * Seconds + * + * m + * + * Minutes + * + * h + * + * Hours + * + * d + * + * Days + * + * parse_duration(string) → interval# Parses string of format value unit into an interval, where value is fractional + * number of unit values: + * + * SELECT parse_duration('42.8ms'); -- 0 00:00:00.043 SELECT parse_duration('3.81 d'); -- 3 19:26:24.000 SELECT + * parse_duration('5m'); -- 0 00:05:00.000 MySQL Date Functions# The functions in this section use a format string + * that is compatible with the MySQL date_parse and str_to_date functions. The following table, based on the MySQL + * manual, describes the format specifiers: + * + * Specifier + * + * Description + * + * %a + * + * Abbreviated weekday name (Sun .. Sat) + * + * %b + * + * Abbreviated month name (Jan .. Dec) + * + * %c + * + * Month, numeric (1 .. 12) 4 + * + * %D + * + * Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) + * + * %d + * + * Day of the month, numeric (01 .. 31) 4 + * + * %e + * + * Day of the month, numeric (1 .. 31) 4 + * + * %f + * + * Fraction of second (6 digits for printing: 000000 .. 999000; 1 - 9 digits for parsing: 0 .. 999999999) 1 + * + * %H + * + * Hour (00 .. 23) + * + * %h + * + * Hour (01 .. 12) + * + * %I + * + * Hour (01 .. 12) + * + * %i + * + * Minutes, numeric (00 .. 59) + * + * %j + * + * Day of year (001 .. 366) + * + * %k + * + * Hour (0 .. 23) + * + * %l + * + * Hour (1 .. 12) + * + * %M + * + * Month name (January .. December) + * + * %m + * + * Month, numeric (01 .. 12) 4 + * + * %p + * + * AM or PM + * + * %r + * + * Time, 12-hour (hh:mm:ss followed by AM or PM) + * + * %S + * + * Seconds (00 .. 59) + * + * %s + * + * Seconds (00 .. 59) + * + * %T + * + * Time, 24-hour (hh:mm:ss) + * + * %U + * + * Week (00 .. 53), where Sunday is the first day of the week + * + * %u + * + * Week (00 .. 53), where Monday is the first day of the week + * + * %V + * + * Week (01 .. 53), where Sunday is the first day of the week; used with %X + * + * %v + * + * Week (01 .. 53), where Monday is the first day of the week; used with %x + * + * %W + * + * Weekday name (Sunday .. Saturday) + * + * %w + * + * Day of the week (0 .. 6), where Sunday is the first day of the week 3 + * + * %X + * + * Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V + * + * %x + * + * Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v + * + * %Y + * + * Year, numeric, four digits + * + * %y + * + * Year, numeric (two digits) 2 + * + * %% + * + * A literal % character + * + * %x + * + * x, for any x not listed above + * + * 1 Timestamp is truncated to milliseconds. + * + * 2 When parsing, two-digit year format assumes range 1970 ... 2069, so “70” will result in year 1970 but “69” will + * produce 2069. + * + * 3 This specifier is not supported yet. Consider using day_of_week() (it uses 1-7 instead of 0-6). + * + * 4(1,2,3,4) This specifier does not support 0 as a month or day. + * + * Warning + * + * The following specifiers are not currently supported: %D %U %u %V %w %X + * + * date_format(timestamp, format) → varchar# Formats timestamp as a string using format. + * + * date_parse(string, format) → timestamp# Parses string into a timestamp using format. + * + * Java Date Functions# The functions in this section use a format string that is compatible with JodaTime’s + * DateTimeFormat pattern format. + * + * format_datetime(timestamp, format) → varchar# Formats timestamp as a string using format. + * + * parse_datetime(string, format) → timestamp with time zone# Parses string into a timestamp with time zone using + * format. + * + * Extraction Function# The extract function supports the following fields: + * + * Field + * + * Description + * + * YEAR + * + * year() + * + * QUARTER + * + * quarter() + * + * MONTH + * + * month() + * + * WEEK + * + * week() + * + * DAY + * + * day() + * + * DAY_OF_MONTH + * + * day() + * + * DAY_OF_WEEK + * + * day_of_week() + * + * DOW + * + * day_of_week() + * + * DAY_OF_YEAR + * + * day_of_year() + * + * DOY + * + * day_of_year() + * + * YEAR_OF_WEEK + * + * year_of_week() + * + * YOW + * + * year_of_week() + * + * HOUR + * + * hour() + * + * MINUTE + * + * minute() + * + * SECOND + * + * second() + * + * TIMEZONE_HOUR + * + * timezone_hour() + * + * TIMEZONE_MINUTE + * + * timezone_minute() + * + * The types supported by the extract function vary depending on the field to be extracted. Most fields support all + * date and time types. + * + * extract(field FROM x) → bigint# Returns field from x. + * + * Note + * + * This SQL-standard function uses special syntax for specifying the arguments. + * + * Convenience Extraction Functions# day(x) → bigint# Returns the day of the month from x. + * + * day_of_month(x) → bigint# This is an alias for day(). + * + * day_of_week(x) → bigint# Returns the ISO day of the week from x. The value ranges from 1 (Monday) to 7 (Sunday). + * + * day_of_year(x) → bigint# Returns the day of the year from x. The value ranges from 1 to 366. + * + * dow(x) → bigint# This is an alias for day_of_week(). + * + * doy(x) → bigint# This is an alias for day_of_year(). + * + * hour(x) → bigint# Returns the hour of the day from x. The value ranges from 0 to 23. + * + * millisecond(x) → bigint# Returns the millisecond of the second from x. + * + * minute(x) → bigint# Returns the minute of the hour from x. + * + * month(x) → bigint# Returns the month of the year from x. + * + * quarter(x) → bigint# Returns the quarter of the year from x. The value ranges from 1 to 4. + * + * second(x) → bigint# Returns the second of the minute from x. + * + * timezone_hour(timestamp) → bigint# Returns the hour of the time zone offset from timestamp. + * + * timezone_minute(timestamp) → bigint# Returns the minute of the time zone offset from timestamp. + * + * week(x) → bigint# Returns the ISO week of the year from x. The value ranges from 1 to 53. + * + * week_of_year(x) → bigint# This is an alias for week(). + * + * year(x) → bigint# Returns the year from x. + * + * year_of_week(x) → bigint# Returns the year of the ISO week from x. + * + * yow(x) → bigint# This is an alias for year_of_week(). + * + * + * + */ + + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoDateFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoDefaultFunction.java b/src/sqlancer/presto/ast/PrestoDefaultFunction.java new file mode 100644 index 000000000..95b89091e --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoDefaultFunction.java @@ -0,0 +1,233 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public enum PrestoDefaultFunction implements PrestoFunction { + + // Conditional functions + IF_TRUE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { PrestoSchema.PrestoDataType.BOOLEAN, + returnType.getPrimitiveDataType() }; + } + }, + + IF_TRUE_FALSE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { PrestoSchema.PrestoDataType.BOOLEAN, + returnType.getPrimitiveDataType(), returnType.getPrimitiveDataType() }; + } + }, + + NULLIF("nullif", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), returnType.getPrimitiveDataType() }; + } + }, + + COALESCE("coalesce", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public int getNumberOfArguments() { + return UNLIMITED_NO_OF_ARGUMENTS; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + ArrayList prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { + return super.getArgumentsForReturnType(gen, depth, argumentTypes, returnType); + } + }, + + // comparison + + // Returns the largest of the provided values. → [same as input] + GREATEST("greatest", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return UNLIMITED_NO_OF_ARGUMENTS; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + }, + // Returns the smallest of the provided values. → [same as input] + LEAST("least", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return UNLIMITED_NO_OF_ARGUMENTS; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + ArrayList prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + }; + + private static final int UNLIMITED_NO_OF_ARGUMENTS = -1; + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoDefaultFunction(String functionName, PrestoDataType returnType) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = new PrestoDataType[0]; + } + + PrestoDefaultFunction(PrestoDataType returnType) { + this.returnType = returnType; + this.argumentTypes = new PrestoDataType[0]; + this.functionName = toString(); + } + + PrestoDefaultFunction(PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + this.functionName = toString(); + } + + PrestoDefaultFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + public static List getFunctionsCompatibleWith(PrestoCompositeDataType returnType) { + return Stream.of(values()).filter(f -> f.isCompatibleWithReturnType(returnType)).collect(Collectors.toList()); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public int getNumberOfArguments() { + return argumentTypes == null ? 0 : argumentTypes.length; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { + List> arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to the same element type. + PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + + if (getNumberOfArguments() == UNLIMITED_NO_OF_ARGUMENTS) { + PrestoDataType dataType = getArgumentTypes(returnType)[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoCompositeDataType type; + + if (dataType == PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + type = savedArrayType; + } else { + type = PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(gen.generateExpression(type, depth + 1)); + } + } else { + for (PrestoDataType arg : argumentTypes) { + PrestoCompositeDataType type; + if (arg == PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = arg.get(); + } + type = savedArrayType; + } else { + type = PrestoCompositeDataType.fromDataType(arg); + } + arguments.add(gen.generateExpression(type, depth + 1)); + + } + } + return arguments; + } + + @Override + public String toString() { + if (functionName != null) { + return functionName; + } + return super.toString(); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoExpression.java b/src/sqlancer/presto/ast/PrestoExpression.java new file mode 100644 index 000000000..143eb6823 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoExpression.java @@ -0,0 +1,5 @@ +package sqlancer.presto.ast; + +public interface PrestoExpression { + +} diff --git a/src/sqlancer/presto/ast/PrestoFunction.java b/src/sqlancer/presto/ast/PrestoFunction.java new file mode 100644 index 000000000..c91759e2f --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoFunction.java @@ -0,0 +1,127 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public interface PrestoFunction { + + String getFunctionName(); + + boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType); + + PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType); + + default List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoSchema.PrestoDataType[] argumentTypes, PrestoSchema.PrestoCompositeDataType returnType) { + + List> arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to + // the same element type. + PrestoSchema.PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + // -1 - unlimited number of arguments + if (getNumberOfArguments() == -1) { + PrestoSchema.PrestoDataType dataType = argumentTypes[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoSchema.PrestoCompositeDataType type; + + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + type = savedArrayType; + } else { + type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(gen.generateExpression(type, depth + 1)); + } + } else { + for (PrestoSchema.PrestoDataType arg : argumentTypes) { + PrestoSchema.PrestoCompositeDataType dataType; + if (arg == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = arg.get(); + } + dataType = savedArrayType; + } else { + dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); + } + Node expression = gen.generateExpression(dataType, depth + 1); + arguments.add(expression); + } + } + return arguments; + } + + default List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoSchema.PrestoCompositeDataType returnType, boolean orderable) { + + List> arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to + // the same element type. + PrestoSchema.PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + if (getNumberOfArguments() == -1) { + PrestoSchema.PrestoDataType dataType = getArgumentTypes(returnType)[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoSchema.PrestoCompositeDataType compositeDataType; + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + compositeDataType = savedArrayType; + } else { + compositeDataType = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(gen.generateExpression(compositeDataType, depth + 1)); + } + } else { + for (PrestoSchema.PrestoDataType dataType : getArgumentTypes(returnType)) { + PrestoSchema.PrestoCompositeDataType compositeDataType; + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + PrestoSchema.PrestoCompositeDataType arrayType; + do { + arrayType = dataType.get(); + } while (!arrayType.getElementType().isOrderable()); + savedArrayType = arrayType; + } + compositeDataType = savedArrayType; + } else { + compositeDataType = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + Node expression = gen.generateExpression(compositeDataType, depth + 1); + arguments.add(expression); + } + } + return arguments; + } + + default int getNumberOfArguments() { + return getArgumentTypes(null).length; + } + + default boolean shouldPreserveOrderOfArguments() { + return false; + } + + default boolean isStandardFunction() { + return true; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java new file mode 100644 index 000000000..5b184671b --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java @@ -0,0 +1,24 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; + +public class PrestoFunctionWithoutParenthesis implements Node { + + private final PrestoSchema.PrestoCompositeDataType type; + private final String expr; + + public PrestoFunctionWithoutParenthesis(String expr, PrestoSchema.PrestoCompositeDataType type) { + this.expr = expr; + this.type = type; + } + + public String getExpr() { + return expr; + } + + public PrestoSchema.PrestoCompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoJoin.java b/src/sqlancer/presto/ast/PrestoJoin.java new file mode 100644 index 000000000..79bf7e1fc --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoJoin.java @@ -0,0 +1,118 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public class PrestoJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + private OuterType outerType; + + public PrestoJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public static List> getJoins( + List> tableList, PrestoGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + PrestoTypedExpressionGenerator joinGen = new PrestoTypedExpressionGenerator(globalState) + .setColumns(columns); + switch (JoinType.getRandom()) { + case INNER: + joinExpressions.add(PrestoJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN)))); + break; + case LEFT: + joinExpressions.add(PrestoJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN)))); + break; + case RIGHT: + joinExpressions.add(PrestoJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN)))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static PrestoJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new PrestoJoin(left, right, JoinType.RIGHT, predicate); + } + + public static PrestoJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new PrestoJoin(left, right, JoinType.LEFT, predicate); + } + + public static PrestoJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new PrestoJoin(left, right, JoinType.INNER, predicate); + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + public OuterType getOuterType() { + return outerType; + } + + @SuppressWarnings("unused") + private void setOuterType(OuterType outerType) { + this.outerType = outerType; + } + + public enum JoinType { + INNER, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum OuterType { + FULL, LEFT, RIGHT; + + public static OuterType getRandom() { + return Randomly.fromOptions(values()); + } + } + +} diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java new file mode 100644 index 000000000..47fc209fb --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java @@ -0,0 +1,39 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.ast.newast.Node; + +public class PrestoMultiValuedComparison implements Node { + + private final Node left; + private final List> right; + private final PrestoMultiValuedComparisonType type; + private final PrestoMultiValuedComparisonOperator op; + + public PrestoMultiValuedComparison(Node left, List> right, + PrestoMultiValuedComparisonType type, PrestoMultiValuedComparisonOperator op) { + this.left = left; + this.right = new ArrayList<>(right); + this.type = type; + this.op = op; + } + + public Node getLeft() { + return left; + } + + public PrestoMultiValuedComparisonOperator getOp() { + return op; + } + + public List> getRight() { + return new ArrayList<>(right); + } + + public PrestoMultiValuedComparisonType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java new file mode 100644 index 000000000..829f554f4 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java @@ -0,0 +1,43 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema; + +public enum PrestoMultiValuedComparisonOperator { + EQUALS("="), NOT_EQUALS("<>"), NOT_EQUALS_ALT("!="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), + SMALLER_EQUALS("<="); + + private final String stringRepresentation; + + PrestoMultiValuedComparisonOperator(String stringRepresentation) { + this.stringRepresentation = stringRepresentation; + } + + public static PrestoMultiValuedComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public static PrestoMultiValuedComparisonOperator getRandomForType(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoDataType dataType = type.getPrimitiveDataType(); + + switch (dataType) { + case BOOLEAN: + case INT: + case FLOAT: + case DECIMAL: + case DATE: + case TIME: + case TIMESTAMP: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return getRandom(); + default: + return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT); + } + } + + public String getStringRepresentation() { + return stringRepresentation; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java new file mode 100644 index 000000000..d1ecdd382 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; + +public enum PrestoMultiValuedComparisonType { + ANY, SOME, ALL; + + public static PrestoMultiValuedComparisonType getRandom() { + return Randomly.fromOptions(values()); + } +} diff --git a/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java new file mode 100644 index 000000000..d51539645 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java @@ -0,0 +1,36 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.Node; + +public class PrestoQuantifiedComparison implements Node { + + private final Node left; + private final PrestoSelect right; + private final PrestoMultiValuedComparisonType type; + private final PrestoMultiValuedComparisonOperator op; + + public PrestoQuantifiedComparison(Node left, PrestoSelect right, + PrestoMultiValuedComparisonType type, PrestoMultiValuedComparisonOperator op) { + this.left = left; + this.right = right; + this.type = type; + this.op = op; + } + + public Node getLeft() { + return left; + } + + public PrestoMultiValuedComparisonOperator getOp() { + return op; + } + + public Node getRight() { + return right; + } + + public PrestoMultiValuedComparisonType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoSelect.java b/src/sqlancer/presto/ast/PrestoSelect.java new file mode 100644 index 000000000..c99c5c7d0 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoSelect.java @@ -0,0 +1,18 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class PrestoSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public boolean isDistinct() { + return isDistinct; + } + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoTableReference.java b/src/sqlancer/presto/ast/PrestoTableReference.java new file mode 100644 index 000000000..4c0e0d789 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.presto.PrestoSchema; + +public class PrestoTableReference extends TableReferenceNode { + + public PrestoTableReference(PrestoSchema.PrestoTable table) { + super(table); + } +} diff --git a/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java new file mode 100644 index 000000000..2bdc142e6 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java @@ -0,0 +1,52 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; + +public class PrestoUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { + + public PrestoUnaryPostfixOperation(Node expr, PrestoUnaryPostfixOperator op) { + super(expr, op); + } + + public Node getExpression() { + return getExpr(); + } + + public enum PrestoUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public PrestoSchema.PrestoDataType[] getInputDataTypes() { + return PrestoSchema.PrestoDataType.values(); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public PrestoSchema.PrestoDataType[] getInputDataTypes() { + return PrestoSchema.PrestoDataType.values(); + } + }; + + private final String textRepresentations; + + PrestoUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static PrestoUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract PrestoSchema.PrestoDataType[] getInputDataTypes(); + + } + +} diff --git a/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java new file mode 100644 index 000000000..cb8313622 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java @@ -0,0 +1,62 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; + +public class PrestoUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { + + public PrestoUnaryPrefixOperation(PrestoUnaryPrefixOperator operation, Node expression) { + super(expression, operation); + } + + public enum PrestoUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", PrestoSchema.PrestoDataType.BOOLEAN) { + @Override + public PrestoSchema.PrestoDataType getExpressionType() { + return PrestoSchema.PrestoDataType.BOOLEAN; + } + }, + + UNARY_PLUS("+", PrestoSchema.PrestoDataType.INT, PrestoSchema.PrestoDataType.FLOAT, + PrestoSchema.PrestoDataType.DECIMAL) { + @Override + public PrestoSchema.PrestoDataType getExpressionType() { + return PrestoSchema.PrestoDataType.INT; + } + }, + UNARY_MINUS("-", PrestoSchema.PrestoDataType.INT, PrestoSchema.PrestoDataType.FLOAT, + PrestoSchema.PrestoDataType.DECIMAL) { + @Override + public PrestoSchema.PrestoDataType getExpressionType() { + return PrestoSchema.PrestoDataType.INT; + } + }; + + private final String textRepresentation; + private final PrestoSchema.PrestoDataType[] dataTypes; + + PrestoUnaryPrefixOperator(String textRepresentation, PrestoSchema.PrestoDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public PrestoSchema.PrestoDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); + } + + public abstract PrestoSchema.PrestoDataType getExpressionType(); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + + public PrestoSchema.PrestoDataType getExpressionType(PrestoSchema.PrestoDataType type) { + return type; + } + } + +} diff --git a/src/sqlancer/presto/gen/PrestoAlterTableGenerator.java b/src/sqlancer/presto/gen/PrestoAlterTableGenerator.java new file mode 100644 index 000000000..f3d5f9d55 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoAlterTableGenerator.java @@ -0,0 +1,67 @@ +package sqlancer.presto.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoTable; + +public final class PrestoAlterTableGenerator { + + private PrestoAlterTableGenerator() { + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + // PrestoTypedExpressionGenerator gen = new + // PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(PrestoCompositeDataType.getRandomWithoutNull()); + break; + case ALTER_COLUMN: + sb.append("ALTER COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" SET DATA TYPE "); + sb.append(PrestoCompositeDataType.getRandomWithoutNull()); + // if (Randomly.getBoolean()) { + // sb.append(" USING "); + // PrestoErrors.addExpressionErrors(errors); + // sb.append(PrestoToStringVisitor.asString(gen.generateExpression())); + // } + errors.add("Cannot change the type of this column: an index depends on it!"); + errors.add("Cannot change the type of a column that has a UNIQUE or PRIMARY KEY constraint specified"); + errors.add("Unimplemented type for cast"); + errors.add("Conversion:"); + errors.add("Cannot change the type of a column that has a CHECK constraint specified"); + break; + case DROP_COLUMN: + sb.append("DROP COLUMN "); + sb.append(table.getRandomColumn().getName()); + errors.add("named in key does not exist"); // TODO + errors.add("Cannot drop this column:"); + errors.add("Cannot drop column: table only has one column remaining!"); + errors.add("because there is a CHECK constraint that depends on it"); + errors.add("because there is a UNIQUE constraint that depends on it"); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN + } + +} diff --git a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java new file mode 100644 index 000000000..9f869c241 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java @@ -0,0 +1,32 @@ +package sqlancer.presto.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; + +public final class PrestoDeleteGenerator { + + private PrestoDeleteGenerator() { + } + + public static SQLQueryAdapter generate(PrestoGlobalState globalState) { + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(PrestoToStringVisitor + .asString(new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()))); + } + PrestoErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java new file mode 100644 index 000000000..7a467e37d --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -0,0 +1,57 @@ +package sqlancer.presto.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public final class PrestoIndexGenerator { + + private PrestoIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + if (Randomly.getBoolean()) { + errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + sb.append("UNIQUE "); + } + sb.append("INDEX "); + sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information + sb.append(" ON "); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append("("); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + } + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + Node expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + sb.append(PrestoToStringVisitor.asString(expr)); + } + errors.add("already exists!"); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java new file mode 100644 index 000000000..102d2f1be --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -0,0 +1,52 @@ +package sqlancer.presto.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoInsertGenerator extends AbstractInsertGenerator { + + private final PrestoGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public PrestoInsertGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + return new PrestoInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + PrestoErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } + + @Override + protected void insertValue(PrestoColumn prestoColumn) { + Node constant = new PrestoTypedExpressionGenerator(globalState) + .generateInsertConstant(prestoColumn.getType()); + sb.append(PrestoToStringVisitor.asString(constant)); + + } + +} diff --git a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java new file mode 100644 index 000000000..167c1f222 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java @@ -0,0 +1,72 @@ +package sqlancer.presto.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.ast.PrestoConstant; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; + +public final class PrestoRandomQuerySynthesizer { + + private PrestoRandomQuerySynthesizer() { + } + + public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrColumns) { + PrestoTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); + PrestoSelect select = new PrestoSelect(); + // TODO: distinct + // select.setDistinct(Randomly.getBoolean()); + // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); + List> columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + // if (allowAggregates && Randomly.getBoolean()) { + Node expression = gen + .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + columns.add(expression); + // } else { + // columns.add(gen()); + // } + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = PrestoJoin.getJoins(tableList, globalState); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableList)); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull())); + } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + + if (Randomly.getBoolean()) { + select.setLimitClause(PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + // if (Randomly.getBoolean()) { + // select.setOffsetClause( + // PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + // } + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + return select; + } + +} diff --git a/src/sqlancer/presto/gen/PrestoTableGenerator.java b/src/sqlancer/presto/gen/PrestoTableGenerator.java new file mode 100644 index 000000000..1d7df2ee6 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoTableGenerator.java @@ -0,0 +1,69 @@ +package sqlancer.presto.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; + +public class PrestoTableGenerator { + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + PrestoCompositeDataType columnType = PrestoCompositeDataType.getRandomWithoutNull(); + columns.add(new PrestoColumn(columnName, columnType, false, false)); + } + return columns; + } + + public SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + String catalog = globalState.getDbmsSpecificOptions().catalog; + String schema = globalState.getDatabaseName(); + + sb.append(catalog).append("."); + sb.append(schema).append("."); + + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(); + // TypedExpressionGenerator, PrestoColumn, PrestoCompositeDataType> + // typedExpressionGenerator = new PrestoTypedExpressionGenerator(globalState).setColumns(columns); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + PrestoColumn column = columns.get(i); + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + // sb.append(" UNIQUE"); + // } + // if (globalState.getDbmsSpecificOptions().testNotNullConstraints + // && Randomly.getBooleanWithRatherLowProbability()) { + // sb.append(" NOT NULL"); + // } + } + // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { + // errors.add("Invalid type for index"); + // List primaryKeyColumns = Randomly.nonEmptySubset(columns); + // sb.append(", PRIMARY KEY("); + // sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + // sb.append(")"); + // } + sb.append(")"); + + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java new file mode 100644 index 000000000..7afb8da29 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -0,0 +1,798 @@ +package sqlancer.presto.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewCaseOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewTernaryNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.ast.PrestoAggregateFunction; +import sqlancer.presto.ast.PrestoAtTimeZoneOperator; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoColumnReference; +import sqlancer.presto.ast.PrestoConstant; +import sqlancer.presto.ast.PrestoDefaultFunction; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoMultiValuedComparison; +import sqlancer.presto.ast.PrestoMultiValuedComparisonOperator; +import sqlancer.presto.ast.PrestoMultiValuedComparisonType; +import sqlancer.presto.ast.PrestoQuantifiedComparison; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoUnaryPostfixOperation; +import sqlancer.presto.ast.PrestoUnaryPrefixOperation; + +public final class PrestoTypedExpressionGenerator extends + TypedExpressionGenerator, PrestoSchema.PrestoColumn, PrestoSchema.PrestoCompositeDataType> { + + private final Randomly randomly; + private final PrestoGlobalState globalState; + private final int maxDepth; + + public PrestoTypedExpressionGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.randomly = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + } + + @Override + public Node generatePredicate() { + return generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), + randomly.getInteger(0, maxDepth)); + } + + @Override + public Node negatePredicate(Node predicate) { + return new PrestoUnaryPrefixOperation(PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT, predicate); + } + + @Override + public Node isNull(Node expr) { + return new PrestoUnaryPostfixOperation(expr, PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); + } + + @Override + public Node generateConstant(PrestoSchema.PrestoCompositeDataType type) { + if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { + return PrestoConstant.createArrayConstant(type); + // case MAP: + // return PrestoConstant.createMapConstant(type); + } + return PrestoConstant.generateConstant(type, false); + } + + public Node generateInsertConstant(PrestoSchema.PrestoCompositeDataType type) { + if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { + return PrestoConstant.createArrayConstant(type); + // case MAP: + // return PrestoConstant.createMapConstant(type); + } + return PrestoConstant.generateConstant(type, true); + } + + @Override + public Node generateExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (allowAggregates && Randomly.getBoolean()) { + return generateAggregate(type); + } + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } else { + // TODO: functions + List applicableFunctions = PrestoDefaultFunction.getFunctionsCompatibleWith(type); + if (Randomly.getBooleanWithRatherLowProbability() && !applicableFunctions.isEmpty()) { + PrestoDefaultFunction function = Randomly.fromList(applicableFunctions); + return generateFunction(type, depth, function); + } + // TODO: try + // if (Randomly.getBooleanWithRatherLowProbability()) { + // return generateTry(type, depth); + // } + + // TODO: cast + // + // if (Randomly.getBooleanWithRatherLowProbability()) { + // Node expressionNode = generateCast(type, depth); + // } + if (Randomly.getBooleanWithRatherLowProbability()) { + return getCase(type, depth); + } + switch (type.getPrimitiveDataType()) { + case BOOLEAN: + return generateBooleanExpression(depth); + case VARCHAR: + case CHAR: + return generateStringExpression(type, depth); + case INT: + case DECIMAL: + case FLOAT: + return generateNumericExpression(depth); + case DATE: + case TIME: + case TIMESTAMP: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return generateTemporalExpression(type, depth); + case INTERVAL_YEAR_TO_MONTH: + case INTERVAL_DAY_TO_SECOND: + return generateIntervalExpression(type, depth); + case JSON: + return generateJsonExpression(type); + case VARBINARY: + case ARRAY: + // case MAP: + return generateLeafNode(type); // TODO + default: + throw new AssertionError(type); + } + } + } + + private Node generateJsonExpression(PrestoSchema.PrestoCompositeDataType type) { + return generateLeafNode(type); + } + + private Node generateCast(PrestoSchema.PrestoCompositeDataType type, int depth) { + // check can cast + Node expressionNode = generateExpression(getRandomType(), depth + 1); + return new PrestoCastFunction(expressionNode, type); + } + + @SuppressWarnings("unused") + private Node generateTry(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (type.getPrimitiveDataType().isNumeric() && Randomly.getBooleanWithRatherLowProbability()) { + Node expression = generateExpression(type); + return new NewFunctionNode<>(List.of(expression), "try"); + } + + List applicableFunctions = PrestoDefaultFunction.getFunctionsCompatibleWith(type); + if (Randomly.getBooleanWithRatherLowProbability() && !applicableFunctions.isEmpty()) { + PrestoDefaultFunction function = Randomly.fromList(applicableFunctions); + Node expression = generateFunction(type, depth, function); + return new NewFunctionNode<>(List.of(expression), "try"); + } + return new NewFunctionNode<>(List.of(generateCast(type, depth)), "try"); + } + + private NewCaseOperatorNode getCase(PrestoSchema.PrestoCompositeDataType type, int depth) { + List> conditions = new ArrayList<>(); + List> cases = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + conditions.add(generateExpression(type, depth + 1)); + cases.add(generateExpression(type, depth + 1)); + } + Node elseExpr = null; + if (Randomly.getBoolean()) { + elseExpr = generateExpression(type, depth + 1); + } + Node expression = generateExpression(type); + return new NewCaseOperatorNode<>(expression, conditions, cases, elseExpr); + } + + private Node generateFunction(PrestoSchema.PrestoCompositeDataType returnType, int depth, + PrestoDefaultFunction function) { + + PrestoSchema.PrestoDataType[] argumentTypes = function.getArgumentTypes(returnType); + List> arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to + // the same element type. + PrestoSchema.PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + if (function.getNumberOfArguments() == -1) { + PrestoSchema.PrestoDataType dataType = argumentTypes[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoSchema.PrestoCompositeDataType type; + + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + type = savedArrayType; + } else { + type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(generateExpression(type, depth + 1)); + } + } else { + for (PrestoSchema.PrestoDataType arg : argumentTypes) { + PrestoSchema.PrestoCompositeDataType dataType; + if (arg == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = arg.get(); + } + dataType = savedArrayType; + } else { + dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); + } + Node expression = generateExpression(dataType, depth + 1); + arguments.add(expression); + } + } + return new NewFunctionNode<>(arguments, function); + } + + private Node generateStringExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } + return getStringOperation(depth); + } + + private NewBinaryOperatorNode getStringOperation(int depth) { + StringExpression exprType = Randomly.fromOptions(StringExpression.values()); + if (Objects.requireNonNull(exprType) == StringExpression.CONCAT) { + Node left = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + Node right = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + PrestBinaryStringOperator operator = PrestBinaryStringOperator.CONCAT; + return new NewBinaryOperatorNode<>(left, right, operator); + } + throw new AssertionError(exprType); + } + + private Node generateBooleanExpression(int depth) { + List booleanExpressions = Arrays.stream(BooleanExpression.values()) + .collect(Collectors.toList()); + if (!globalState.getDbmsSpecificOptions().testBetween) { + booleanExpressions.remove(BooleanExpression.BETWEEN); + } + + booleanExpressions.remove(BooleanExpression.REGEX); + + BooleanExpression exprType = Randomly.fromList(booleanExpressions); + switch (exprType) { + case NOT: + return generateNOT(depth + 1); + case BINARY_COMPARISON: + return getBinaryComparison(depth); + case BINARY_LOGICAL: + return getBinaryLogical(depth); + case AND_OR_CHAIN: + return getAndOrChain(depth); + case REGEX: + return getRegex(depth); + case IS_NULL: + return new PrestoUnaryPostfixOperation(generateExpression(getRandomType(), depth + 1), + Randomly.fromOptions(PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL, + PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NOT_NULL)); + case IN: + return getInOperation(depth); + case BETWEEN: + return getBetween(depth); + case LIKE: + return getLike(depth); + case MULTI_VALUED_COMPARISON: // TODO other operators + return getMultiValuedComparison(depth); + default: + throw new AssertionError(exprType); + } + } + + private Node getMultiValuedComparison(int depth) { + + PrestoSchema.PrestoCompositeDataType type; + do { + type = PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getOrderableTypes())); + } while (type.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY + && !type.getElementType().getPrimitiveDataType().isOrderable()); + + PrestoMultiValuedComparisonType comparisonType = PrestoMultiValuedComparisonType.getRandom(); + PrestoMultiValuedComparisonOperator comparisonOperator = PrestoMultiValuedComparisonOperator + .getRandomForType(type); + Node left = generateExpression(type, depth + 1); + // sub-query + PrestoSchema.PrestoCompositeDataType finalType = type; + List columnsOfType = columns.stream().filter(c -> c.getType() == finalType) + .collect(Collectors.toList()); + if (Randomly.getBooleanWithRatherLowProbability() && !columnsOfType.isEmpty()) { + PrestoSchema.PrestoColumn column = Randomly.fromList(columnsOfType); + PrestoSelect subquery = generateSubquery(List.of(column)); + return new PrestoQuantifiedComparison(left, subquery, comparisonType, comparisonOperator); + } + int nr = Randomly.smallNumber() + 2; + List> rightList = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + rightList.add(generateConstant(type)); + } + return new PrestoMultiValuedComparison(left, rightList, comparisonType, comparisonOperator); + } + + private PrestoSelect generateSubquery(List columns) { + PrestoSelect select = new PrestoSelect(); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + List tables = columns.stream().map(AbstractTableColumn::getTable) + .collect(Collectors.toList()); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).distinct() + .collect(Collectors.toList()); + List> tableNodeList = tables.stream() + .map(t -> new TableReferenceNode(t)) + .collect(Collectors.toList()); + select.setFromList(tableNodeList); + TypedExpressionGenerator, PrestoSchema.PrestoColumn, PrestoSchema.PrestoCompositeDataType> typedExpressionGenerator = new PrestoTypedExpressionGenerator( + globalState).setColumns(columns); + Node predicate = typedExpressionGenerator.generatePredicate(); + select.setWhereClause(predicate); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(typedExpressionGenerator.generateOrderBys()); + } + List> joins = PrestoJoin.getJoins(tableList, globalState); + select.setJoinList(joins); + return select; + } + + private Node generateNumericExpression(int depth) { + PrestoSchema.PrestoDataType dataType = Randomly.fromList(PrestoSchema.PrestoDataType.getNumberTypes()); + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + if (Randomly.getBoolean()) { + BinaryOperatorNode.Operator operator = PrestoBinaryArithmeticOperator.getRandom(); + Node left = generateExpression(type, depth); + Node right = generateExpression(type, depth); + return new NewBinaryOperatorNode<>(left, right, operator); + } else { + BinaryOperatorNode.Operator operator = PrestoUnaryArithmeticOperator.MINUS; + Node left = generateExpression(type, depth); + return new NewUnaryPrefixOperatorNode<>(left, operator); + } + } + + private Node generateTemporalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (Randomly.getBooleanWithSmallProbability()) { + Node left = generateExpression(type, depth); + Node right = generateExpression(PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getIntervalTypes())), depth); + BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); + return new NewBinaryOperatorNode<>(left, right, operator); + } + + // timestamp at time zone + if (Randomly.getBooleanWithSmallProbability() + && (type.getPrimitiveDataType() == PrestoSchema.PrestoDataType.TIMESTAMP + || type.getPrimitiveDataType() == PrestoSchema.PrestoDataType.TIMESTAMP_WITH_TIME_ZONE)) { + return new PrestoAtTimeZoneOperator(generateExpression(type, depth + 1), + PrestoConstant.createTimezoneConstant()); + } + return generateLeafNode(type); + } + + private Node generateIntervalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (Randomly.getBooleanWithSmallProbability()) { + Node left = generateExpression(type, depth); + + Node right; + if (Randomly.getBoolean()) { + right = generateExpression(PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getTemporalTypes())), depth); + } else { + right = generateExpression(type, depth); + } + BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); + if (Randomly.getBoolean()) { + return new NewBinaryOperatorNode<>(left, right, operator); + } else { + return new NewBinaryOperatorNode<>(right, left, operator); + } + } + return generateLeafNode(type); + + // functions + + // timestamp at time zone + } + + private Node getLike(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(PrestoSchema.PrestoDataType.VARCHAR); + Node expression = generateExpression(type, depth + 1); + Node pattern = generateExpression(type, depth + 1); + if (Randomly.getBoolean()) { + return new NewBinaryOperatorNode<>(expression, pattern, PrestoLikeOperator.getRandom()); + } else { + String randomlyString = randomly.getString(); + String randomlyChar = randomly.getChar(); + Node escape = new PrestoConstant.PrestoTextConstant(randomlyChar, 1); + int index = randomlyString.indexOf(randomlyChar); + while (index > -1) { + String wildcard = Randomly.fromOptions("%", "_"); + randomlyString = randomlyString.substring(0, index + 1) + wildcard + + randomlyString.substring(index + 1); + index = randomlyString.indexOf(randomlyChar, index + 1); + } + PrestoConstant.PrestoTextConstant patternString = new PrestoConstant.PrestoTextConstant(randomlyString); + return new NewTernaryNode<>(expression, patternString, escape, "LIKE", "ESCAPE"); + } + } + + private NewBinaryOperatorNode getRegex(int depth) { + Node left = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + Node right = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + return new NewBinaryOperatorNode<>(left, right, PrestoBinaryLogicalOperator.getRandom()); + } + + private NewBinaryOperatorNode getBinaryLogical(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(PrestoSchema.PrestoDataType.BOOLEAN); + Node left = generateExpression(type, depth + 1); + Node right = generateExpression(type, depth + 1); + BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode<>(left, right, operator); + } + + private Node getBetween(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getNumericTypes())); + Node expression = generateExpression(type, depth + 1); + Node left = generateExpression(type, depth + 1); + Node right = generateExpression(type, depth + 1); + return new NewBetweenOperatorNode<>(expression, left, right, Randomly.getBoolean()); + } + + private Node getInOperation(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(PrestoSchema.PrestoDataType.getRandomWithoutNull()); + Node left = generateExpression(type, depth + 1); + List> inList = generateExpressions(type, Randomly.smallNumber() + 1, depth + 1); + boolean isNegated = Randomly.getBoolean(); + return new NewInOperatorNode<>(left, inList, isNegated); + } + + private Node getAndOrChain(int depth) { + Node left = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + Node right = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); + BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); + left = new NewBinaryOperatorNode<>(left, right, operator); + } + return left; + } + + private Node getBinaryComparison(int depth) { + PrestoSchema.PrestoCompositeDataType type = getRandomType(); + BinaryOperatorNode.Operator op = PrestoBinaryComparisonOperator.getRandomForType(type); + Node left = generateExpression(type, depth + 1); + Node right = generateExpression(type, depth + 1); + return new NewBinaryOperatorNode<>(left, right, op); + } + + private Node generateNOT(int depth) { + PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator operator = PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT; + return new PrestoUnaryPrefixOperation(operator, generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth)); + } + + @Override + protected Node generateColumn(PrestoSchema.PrestoCompositeDataType type) { + List columnList = columns.stream() + .filter(c -> c.getType().getPrimitiveDataType() == type.getPrimitiveDataType()) + .collect(Collectors.toList()); + PrestoSchema.PrestoColumn column = Randomly.fromList(columnList); + return new PrestoColumnReference(column); + } + + @Override + public Node generateLeafNode(PrestoSchema.PrestoCompositeDataType type) { + if (Randomly.getBoolean()) { + return generateConstant(type); + } else { + List columnList = filterColumns(type.getPrimitiveDataType()); + if (columnList.isEmpty()) { + return generateConstant(type); + } else { + return generateColumn(type); + } + } + } + + private List filterColumns(PrestoSchema.PrestoDataType dataType) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType().getPrimitiveDataType() == dataType) + .collect(Collectors.toList()); + } + } + + @Override + protected PrestoSchema.PrestoCompositeDataType getRandomType() { + return PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull(); + } + + @Override + protected boolean canGenerateColumnOfType(PrestoSchema.PrestoCompositeDataType type) { + return columns.stream().anyMatch(c -> c.getType() == type); + } + + public Node generateAggregate() { + PrestoAggregateFunction aggregateFunction = PrestoAggregateFunction.getRandom(); + List> argsForAggregate = generateArgsForAggregate(aggregateFunction); + return new NewFunctionNode<>(argsForAggregate, aggregateFunction); + } + + public List> generateArgsForAggregate(PrestoAggregateFunction aggregateFunction) { + PrestoSchema.PrestoCompositeDataType returnType; + do { + returnType = aggregateFunction.getCompositeReturnType(); + } while (!aggregateFunction.isCompatibleWithReturnType(returnType)); + return aggregateFunction.getArgumentsForReturnType(this, this.maxDepth - 1, returnType, false); + } + + private Node generateAggregate(PrestoSchema.PrestoCompositeDataType type) { + PrestoAggregateFunction aggregateFunction = Randomly + .fromList(PrestoAggregateFunction.getFunctionsCompatibleWith(type)); + List> argsForAggregate = generateArgsForAggregate(type, aggregateFunction); + return new NewFunctionNode<>(argsForAggregate, aggregateFunction); + } + + public List> generateArgsForAggregate(PrestoSchema.PrestoCompositeDataType type, + PrestoAggregateFunction aggregateFunction) { + List returnTypes = aggregateFunction.getReturnTypes(type.getPrimitiveDataType()); + List> arguments = new ArrayList<>(); + allowAggregates = false; // + for (PrestoSchema.PrestoDataType argumentType : returnTypes) { + arguments.add(generateExpression(PrestoSchema.PrestoCompositeDataType.fromDataType(argumentType))); + } + // return new NewFunctionNode<>(arguments, aggregateFunction); + return arguments; + } + + @Override + public List> generateOrderBys() { + List> expressions = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + ArrayList prestoColumns = new ArrayList<>(columns); + prestoColumns.removeIf(c -> !c.isOrderable()); + for (int i = 0; i < nr && !prestoColumns.isEmpty(); i++) { + PrestoSchema.PrestoColumn randomColumn = Randomly.fromList(prestoColumns); + PrestoColumnReference columnReference = new PrestoColumnReference(randomColumn); + prestoColumns.remove(randomColumn); + expressions.add(columnReference); + } + return expressions; + } + + public Node generateHavingClause() { + allowAggregates = true; + Node expr = generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + allowAggregates = false; + return expr; + } + + public Node generateExpressionWithColumns(List columns, + int remainingDepth) { + if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + } + PrestoSchema.PrestoColumn column = Randomly.fromList(columns); + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + return new PrestoColumnReference(column); + } + List possibleOptions = new ArrayList<>( + Arrays.asList(PrestoTypedExpressionGenerator.Expression.values())); + PrestoTypedExpressionGenerator.Expression expr = Randomly.fromList(possibleOptions); + BinaryOperatorNode.Operator op; + switch (expr) { + case BINARY_LOGICAL: + case BINARY_ARITHMETIC: + op = PrestoTypedExpressionGenerator.PrestoBinaryLogicalOperator.getRandom(); + break; + case BINARY_COMPARISON: + op = PrestoBinaryComparisonOperator.getRandom(); + break; + default: + throw new AssertionError(); + } + return new NewBinaryOperatorNode<>(generateExpression(column.getType(), remainingDepth - 1), + generateExpression(column.getType(), remainingDepth - 1), op); + } + + private enum StringExpression { + CONCAT + } + + public enum PrestBinaryStringOperator implements BinaryOperatorNode.Operator { + CONCAT("||"); + + private final String textRepresentation; + + PrestBinaryStringOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoBinaryTemporalOperator implements BinaryOperatorNode.Operator { + ADD("+"), SUB("-"); + + private final String textRepresentation; + + PrestoBinaryTemporalOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + private enum BooleanExpression { + NOT, BINARY_COMPARISON, BINARY_LOGICAL, AND_OR_CHAIN, REGEX, IS_NULL, IN, BETWEEN, LIKE, MULTI_VALUED_COMPARISON + } + + public enum PrestoBinaryLogicalOperator implements BinaryOperatorNode.Operator { + + AND, OR; + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + + public enum PrestoLikeOperator implements BinaryOperatorNode.Operator { + LIKE("LIKE"), // + NOT_LIKE("NOT LIKE"); + + private final String textRepresentation; + + PrestoLikeOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static PrestoLikeOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("="), NOT_EQUALS("<>"), NOT_EQUALS_ALT("!="), IS_DISTINCT_FROM("IS DISTINCT FROM"), + IS_NOT_DISTINCT_FROM("IS NOT DISTINCT FROM"), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), + SMALLER_EQUALS("<="); + + private final String textRepresentation; + + PrestoBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + public static BinaryOperatorNode.Operator getRandomStringOperator() { + return Randomly.fromOptions(EQUALS, NOT_EQUALS, IS_DISTINCT_FROM, IS_NOT_DISTINCT_FROM); + } + + public static BinaryOperatorNode.Operator getRandomForType(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoDataType dataType = type.getPrimitiveDataType(); + + switch (dataType) { + case BOOLEAN: + case INT: + case FLOAT: + case DECIMAL: + case DATE: + case TIME: + case TIMESTAMP: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return getRandom(); + case VARCHAR: + case CHAR: + case VARBINARY: + case JSON: + case ARRAY: + case INTERVAL_YEAR_TO_MONTH: + case INTERVAL_DAY_TO_SECOND: + // return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, IS_DISTINCT_FROM, + // IS_NOT_DISTINCT_FROM); + default: + return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, IS_DISTINCT_FROM, IS_NOT_DISTINCT_FROM); + } + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"); + + private final String textRepresentation; + + PrestoBinaryArithmeticOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoUnaryArithmeticOperator implements BinaryOperatorNode.Operator { + MINUS("-"); + + private final String textRepresentation; + + PrestoUnaryArithmeticOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + private enum Expression { + BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC + } + +} diff --git a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java new file mode 100644 index 000000000..bdd8cf2a3 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.presto.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public final class PrestoUpdateGenerator extends AbstractUpdateGenerator { + + private final PrestoGlobalState globalState; + private PrestoTypedExpressionGenerator gen; + + private PrestoUpdateGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + return new PrestoUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + PrestoErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } + + @Override + protected void updateValue(PrestoColumn column) { + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(column.getType()); + PrestoErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(column.getType()); + } + sb.append(PrestoToStringVisitor.asString(expr)); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoViewGenerator.java b/src/sqlancer/presto/gen/PrestoViewGenerator.java new file mode 100644 index 000000000..72130cf82 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoViewGenerator.java @@ -0,0 +1,36 @@ +package sqlancer.presto.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoToStringVisitor; + +public final class PrestoViewGenerator { + + private PrestoViewGenerator() { + } + + public static SQLQueryAdapter generate(PrestoGlobalState globalState) { + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(PrestoToStringVisitor.asString(PrestoRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + PrestoErrors.addExpressionErrors(errors); + PrestoErrors.addGroupByErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + +} diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java new file mode 100644 index 000000000..a63d912b0 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -0,0 +1,138 @@ +package sqlancer.presto.test; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public class PrestoNoRECOracle extends NoRECBase implements TestOracle { + + private final PrestoSchema s; + + public PrestoNoRECOracle(PrestoGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + PrestoErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + PrestoTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + + List tables = randomTables.getTables(); + + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = PrestoJoin.getJoins(tableList, state); + + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(state).setColumns(columns); + Node randomWhereCondition = gen.generatePredicate(); + int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); + + int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getSecondQuery(List> tableList, Node randomWhereCondition, + List> joins) throws SQLException { + PrestoSelect select = new PrestoSelect(); + + Node asText = new NewPostfixTextNode<>( + + new PrestoCastFunction( + new NewPostfixTextNode<>(randomWhereCondition, + " IS NOT NULL AND " + PrestoToStringVisitor.asString(randomWhereCondition)), + new PrestoCompositeDataType(PrestoDataType.INT, 8, 0)), + "as count"); + + select.setFetchColumns(List.of(asText)); + select.setFromList(tableList); + select.setJoinList(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + PrestoToStringVisitor.asString(select) + ") as res"; + + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors, false, false); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, + List> joins) { + PrestoSelect select = new PrestoSelect(); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions( + new PrestoTypedExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + select.setJoinList(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = PrestoToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java new file mode 100644 index 000000000..527d5d648 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java @@ -0,0 +1,206 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoAggregateFunction; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoUnaryPostfixOperation; +import sqlancer.presto.ast.PrestoUnaryPrefixOperation; + +public class PrestoQueryPartitioningAggregateTester extends PrestoQueryPartitioningBase + implements TestOracle { + + private String firstResult; + private String firstResultType; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public PrestoQueryPartitioningAggregateTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + PrestoErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + PrestoAggregateFunction aggregateFunction = Randomly.fromOptions(PrestoAggregateFunction.MAX, + PrestoAggregateFunction.MIN, PrestoAggregateFunction.SUM, PrestoAggregateFunction.COUNT, + PrestoAggregateFunction.AVG/* , PrestoAggregateFunction.STDDEV_POP */); + List> aggregateArgs = gen.generateArgsForAggregate(aggregateFunction); + NewFunctionNode aggregate = new NewFunctionNode<>(aggregateArgs, + aggregateFunction); + select.setFetchColumns(List.of(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + originalQuery = PrestoToStringVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + firstResultType = getAggregateResultType(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResultType.equals("VARBINARY") || firstResultType.equals("ARRAY(VARBINARY)") + || firstResultType.equals("ARRAY(ARRAY(VARBINARY))")) { + throw new IgnoreMeException(); + } + if (firstResult == null && secondResult != null) { + if (secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + throw new AssertionError(); + } else if (firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + throw new AssertionError(); + } + + } + + private String createMetamorphicUnionQuery(PrestoSelect select, + NewFunctionNode aggregate, List> from) { + String metamorphicQuery; + Node whereClause = gen.generatePredicate(); + Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT); + Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); + List> mappedAggregate = mapped(aggregate); + PrestoSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + PrestoSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + PrestoSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += PrestoToStringVisitor.asString(leftSelect) + " UNION ALL " + + PrestoToStringVisitor.asString(middleSelect) + " UNION ALL " + + PrestoToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private String getAggregateResultType(String queryString) { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getType(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List> mapped(NewFunctionNode aggregate) { + PrestoCastFunction count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(List.of(aggregate)); + case AVG: + NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + PrestoAggregateFunction.SUM); + count = new PrestoCastFunction(new NewFunctionNode<>(aggregate.getArgs(), PrestoAggregateFunction.COUNT), + new PrestoCompositeDataType(PrestoDataType.FLOAT, 8, 0)); + return aliasArgs(Arrays.asList(sum, count)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List> aliasArgs(List> originalAggregateArgs) { + List> args = new ArrayList<>(); + int i = 0; + for (Node expr : originalAggregateArgs) { + args.add(new NewAliasNode<>(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(NewFunctionNode aggregate) { + switch (aggregate.getFunc()) { + case AVG: + return "SUM(CAST(agg0 AS DOUBLE))/CAST(SUM(agg1) AS DOUBLE)"; + case COUNT: + return PrestoAggregateFunction.SUM + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private PrestoSelect getSelect(List> aggregates, List> from, + Node whereClause, List> joinList) { + PrestoSelect leftSelect = new PrestoSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java new file mode 100644 index 000000000..5ee56e1c8 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java @@ -0,0 +1,90 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public class PrestoQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, PrestoGlobalState> + implements TestOracle { + + PrestoSchema s; + PrestoTables targetTables; + PrestoTypedExpressionGenerator gen; + PrestoSelect select; + + public PrestoQueryPartitioningBase(PrestoGlobalState state) { + super(state); + PrestoErrors.addExpressionErrors(errors); + } + + public static String canonicalizeResultValue(String value) { + if (value == null) { + return null; + } + + // TODO: check this + switch (value) { + case "-0.0": + return "0.0"; + case "-0": + return "0"; + default: + } + + return value; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new PrestoTypedExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new PrestoSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = PrestoJoin.getJoins(tableList, state); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableList)); + select.setWhereClause(null); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new PrestoColumn("*", null, false, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..008e8f4db --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java @@ -0,0 +1,44 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoToStringVisitor; + +public class PrestoQueryPartitioningDistinctTester extends PrestoQueryPartitioningBase { + + public PrestoQueryPartitioningDistinctTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + if (Randomly.getBoolean()) { + select.setDistinct(false); + } + select.setWhereClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..5d5e2343f --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java @@ -0,0 +1,53 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoQueryPartitioningGroupByTester extends PrestoQueryPartitioningBase { + + public PrestoQueryPartitioningGroupByTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setGroupByExpressions(select.getFetchColumns()); + select.setWhereClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + List> generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java new file mode 100644 index 000000000..b3ff3beaa --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java @@ -0,0 +1,64 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoQueryPartitioningHavingTester extends PrestoQueryPartitioningBase + implements TestOracle { + + public PrestoQueryPartitioningHavingTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull())); + } + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setHavingClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + protected Node generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List> generateFetchColumns() { + return Collections.singletonList(gen.generateHavingClause()); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java new file mode 100644 index 000000000..093c27108 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java @@ -0,0 +1,46 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoToStringVisitor; + +public class PrestoQueryPartitioningWhereTester extends PrestoQueryPartitioningBase { + + public PrestoQueryPartitioningWhereTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + PrestoErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setWhereClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + +} From 70d11138eb01a5253ada440ee9a469bb8784e871 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 5 Sep 2023 10:26:23 +0800 Subject: [PATCH 0819/1284] feat: add support of 8.0 version (#886) * feat: add support of 8.0 version * feat: add expected errors for 8.0 version * fix: change --test-8.0-version to true * feat: add RENAME COLUMN support when altering table * fix: test error * feat: add AS support when inserting * feat: add AS support when deleting --- src/sqlancer/stonedb/StoneDBErrors.java | 9 ++++++++- src/sqlancer/stonedb/StoneDBOptions.java | 3 +++ src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java | 6 ++++++ .../stonedb/gen/StoneDBTableDeleteGenerator.java | 4 ++++ .../stonedb/gen/StoneDBTableInsertGenerator.java | 9 +++++++++ src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java | 2 +- 8 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java index 1e72e5685..152c5bfc9 100644 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -1,12 +1,13 @@ package sqlancer.stonedb; import sqlancer.common.query.ExpectedErrors; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; public final class StoneDBErrors { private StoneDBErrors() { } - public static void addExpectedExpressionErrors(ExpectedErrors errors) { + public static void addExpectedExpressionErrors(StoneDBGlobalState globalState, ExpectedErrors errors) { // java.sql.SQLException: Incorrect DATE value: '292269055-12-02' errors.add("Incorrect DATE value: "); // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 @@ -31,5 +32,11 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Numeric result of an expression is too large and cannot be handled by tianmu."); // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order clause' errors.add("Unknown column "); + if (globalState.getDbmsSpecificOptions().test80Version) { + // Caused by: java.sql.SQLException: Incorrect DATETIME value: '292269055-12-02 16:47:04' + errors.add("Incorrect DATETIME value: "); + // Caused by: java.sql.SQLException: Incorrect TIMESTAMP value: '292269055-12-02 16:47:04' + errors.add("Incorrect TIMESTAMP value: "); + } } } diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java index a9728323f..0c9d8816a 100644 --- a/src/sqlancer/stonedb/StoneDBOptions.java +++ b/src/sqlancer/stonedb/StoneDBOptions.java @@ -27,6 +27,9 @@ public class StoneDBOptions implements DBMSSpecificOptions public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; + @Parameter(names = "--test-8-version", description = "Let SQLancer test the StoneDB 8.0, otherwise, SQLancer will test the StoneDB 5.7", arity = 1) + public boolean test80Version = true; + @Parameter(names = "--oracle") public List oracles = List.of(StoneDBOracleFactory.NOREC); diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index e9c283cc9..c37024669 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -66,6 +66,12 @@ private void addExpectedErrors() { private void appendAlterOption(Action action) { StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) .setColumns(table.getColumns()); + if (globalState.getDbmsSpecificOptions().test80Version && Randomly.getBooleanWithSmallProbability()) { + sb.append("RENAME COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" TO "); + sb.append(table.getFreeColumnName()); + } switch (action) { case ADD_COLUMN: sb.append("ADD COLUMN "); diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index 16b9c1061..0d8446d4b 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -40,6 +40,10 @@ public SQLQueryAdapter getQuery() { } sb.append(" FROM "); sb.append(randomTable.getName()); + if (globalState.getDbmsSpecificOptions().test80Version) { + sb.append(" AS "); + sb.append(globalState.getSchema().getFreeTableName()); + } if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index a9823be6d..0a70c8467 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -48,6 +48,7 @@ private SQLQueryAdapter getQuery() { sb.append(table.getName()); appendPartition(); appendColumnsAndValues(columns); + appendAS(); appendOnDuplicateUpdate(); addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); @@ -84,6 +85,14 @@ private void appendColumnsAndValues(List columns) { appendValues(); } + private void appendAS() { + if (!globalState.getDbmsSpecificOptions().test80Version) { + return; + } + sb.append(" AS "); + sb.append("r").append(table.getNrRows(globalState)); + } + private void appendOnDuplicateUpdate() { sb.append("on duplicate key update "); StoneDBColumn randomColumn = table.getRandomColumn(); diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java index 27ba91013..2465494ef 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -31,7 +31,7 @@ public class StoneDBAggregateOracle extends StoneDBQueryPartitioningBase { public StoneDBAggregateOracle(StoneDBGlobalState state) { super(state); - StoneDBErrors.addExpectedExpressionErrors(errors); + StoneDBErrors.addExpectedExpressionErrors(state, errors); } @Override diff --git a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java index 7e4d6a622..6cd5b3bc9 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java @@ -31,7 +31,7 @@ public class StoneDBFuzzOracle implements TestOracle { public StoneDBFuzzOracle(StoneDBGlobalState globalState) { this.globalState = globalState; this.schema = globalState.getSchema(); - StoneDBErrors.addExpectedExpressionErrors(errors); + StoneDBErrors.addExpectedExpressionErrors(globalState, errors); } @Override diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index e042cd44a..744d52f55 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -43,7 +43,7 @@ public class StoneDBNoRECOracle extends NoRECBase implements public StoneDBNoRECOracle(StoneDBGlobalState globalState) { super(globalState); this.schema = globalState.getSchema(); - StoneDBErrors.addExpectedExpressionErrors(errors); + StoneDBErrors.addExpectedExpressionErrors(globalState, errors); } @Override From 952c7cd37d065950c9c0fb76721c3aac7684983f Mon Sep 17 00:00:00 2001 From: Nathee Jaywaree <73115539+eltfshr@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:56:12 +0700 Subject: [PATCH 0820/1284] Add Fuzzer oracle for PostgreSQL and MySQL (#893) * Add random query synthesizer for MySQL * Add Fuzzer oracle for MySQL and Postgres --- src/sqlancer/mysql/MySQLOptions.java | 8 +++ .../gen/MySQLRandomQuerySynthesizer.java | 52 +++++++++++++++++++ src/sqlancer/mysql/oracle/MySQLFuzzer.java | 30 +++++++++++ src/sqlancer/postgres/PostgresOptions.java | 8 +++ .../postgres/oracle/PostgresFuzzer.java | 30 +++++++++++ src/sqlancer/sqlite3/SQLite3Options.java | 4 -- .../sqlite3/oracle/SQLite3Fuzzer.java | 6 +-- 7 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java create mode 100644 src/sqlancer/mysql/oracle/MySQLFuzzer.java create mode 100644 src/sqlancer/postgres/oracle/PostgresFuzzer.java diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 2d9e80bc0..4e42e8b00 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -12,6 +12,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; import sqlancer.mysql.oracle.MySQLCERTOracle; +import sqlancer.mysql.oracle.MySQLFuzzer; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @@ -57,6 +58,13 @@ public TestOracle create(MySQLGlobalState globalState) throws public boolean requiresAllTablesToContainRows() { return true; } + }, + FUZZER { + @Override + public TestOracle create(MySQLGlobalState globalState) throws Exception { + return new MySQLFuzzer(globalState); + } + }; } diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java new file mode 100644 index 000000000..686e6091d --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -0,0 +1,52 @@ +package sqlancer.mysql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLTableReference; + +public final class MySQLRandomQuerySynthesizer { + + private MySQLRandomQuerySynthesizer() { + } + + public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) { + MySQLTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + MySQLSelect select = new MySQLSelect(); + List columns = new ArrayList<>(); + + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); + columns.addAll(gen.generateExpressions(nrColumns)); + select.setFetchColumns(columns); + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) + .collect(Collectors.toList()); + select.setFromList(tableList); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBoolean()) { + select.setLimitClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(gen.generateExpression()); + } + return select; + } + +} diff --git a/src/sqlancer/mysql/oracle/MySQLFuzzer.java b/src/sqlancer/mysql/oracle/MySQLFuzzer.java new file mode 100644 index 000000000..2e361c0b8 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLFuzzer.java @@ -0,0 +1,30 @@ +package sqlancer.mysql.oracle; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.gen.MySQLRandomQuerySynthesizer; + +public class MySQLFuzzer implements TestOracle { + + private final MySQLGlobalState globalState; + + public MySQLFuzzer(MySQLGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public void check() throws Exception { + String s = MySQLVisitor.asString(MySQLRandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + + ';'; + try { + globalState.executeStatement(new SQLQueryAdapter(s)); + globalState.getManager().incrementSelectQueryCount(); + } catch (Error e) { + + } + } + +} diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 948a539ce..bb76404de 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -13,6 +13,7 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; +import sqlancer.postgres.oracle.PostgresFuzzer; import sqlancer.postgres.oracle.PostgresNoRECOracle; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; @@ -76,6 +77,13 @@ public TestOracle create(PostgresGlobalState globalState) t oracles.add(new PostgresTLPAggregateOracle(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + FUZZER { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + return new PostgresFuzzer(globalState); + } + }; } diff --git a/src/sqlancer/postgres/oracle/PostgresFuzzer.java b/src/sqlancer/postgres/oracle/PostgresFuzzer.java new file mode 100644 index 000000000..a0b46bd17 --- /dev/null +++ b/src/sqlancer/postgres/oracle/PostgresFuzzer.java @@ -0,0 +1,30 @@ +package sqlancer.postgres.oracle; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.gen.PostgresRandomQueryGenerator; + +public class PostgresFuzzer implements TestOracle { + + private final PostgresGlobalState globalState; + + public PostgresFuzzer(PostgresGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public void check() throws Exception { + String s = PostgresVisitor.asString( + PostgresRandomQueryGenerator.createRandomQuery(Randomly.smallNumber() + 1, globalState)) + ';'; + try { + globalState.executeStatement(new SQLQueryAdapter(s)); + globalState.getManager().incrementSelectQueryCount(); + } catch (Error e) { + + } + } + +} diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index e740a5eea..e2c04ed46 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -86,10 +86,6 @@ public class SQLite3Options implements DBMSSpecificOptions "--generate-new-database" }, description = "Specifies whether new databases should be generated", arity = 1) public boolean generateDatabase = true; - @Parameter(names = { - "--execute-queries" }, description = "Specifies whether the query in the fuzzer should be executed", arity = 1) - public boolean executeQuery = true; - @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables/virtual tables/ rtree tables/ views that can be created") public int maxNumTables = 10; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index bf13ffc80..e98e5f032 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -20,10 +20,8 @@ public void check() throws Exception { String s = SQLite3Visitor .asString(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + ";"; try { - if (globalState.getDbmsSpecificOptions().executeQuery) { - globalState.executeStatement(new SQLQueryAdapter(s)); - globalState.getManager().incrementSelectQueryCount(); - } + globalState.executeStatement(new SQLQueryAdapter(s)); + globalState.getManager().incrementSelectQueryCount(); } catch (Error e) { } From cddff69308e9bcdbf0ed9abc5b77b57590431ff7 Mon Sep 17 00:00:00 2001 From: Nathee Jaywaree <73115539+eltfshr@users.noreply.github.com> Date: Fri, 15 Sep 2023 08:39:53 +0700 Subject: [PATCH 0821/1284] Change OFFSET and LIMIT clause in MySQLRandomQuerySynthesizer (#894) * Add random query synthesizer for MySQL * Add Fuzzer oracle for MySQL and Postgres * Change offset from MySQLRandomQuerySynthesizer --- src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java index 686e6091d..8ebaff3b6 100644 --- a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -7,6 +7,7 @@ import sqlancer.Randomly; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.ast.MySQLConstant; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLTableReference; @@ -41,10 +42,10 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) } } if (Randomly.getBoolean()) { - select.setLimitClause(gen.generateExpression()); - } - if (Randomly.getBoolean()) { - select.setOffsetClause(gen.generateExpression()); + select.setLimitClause(MySQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(MySQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } } return select; } From b9759b4ae8ed9cdee06cbac7a4ba6477be3b24d2 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 6 Nov 2023 18:32:31 +0800 Subject: [PATCH 0822/1284] Reducer updates. (#892) * use single `FileWriter` for reducer. * Trivial update. * Add warning when reducer is not available. * resolve comment. * Try to implement a reducer. * fix format * Bump `setup-java` to v3 to cache maven artifacts. * fix PMD. * Remove non-LTS Java versions. * Fix reducer log. * Resolve comments. * Update doc. * Extract reproducer to separate class. * Fix format issues. * Cleanup and update messages. --- .github/workflows/main.yml | 178 +++++++++--------- .github/workflows/release.yml | 10 +- docs/testCaseReduction.md | 23 ++- src/sqlancer/ASTBasedReducer.java | 3 +- src/sqlancer/Main.java | 54 ++++-- src/sqlancer/ProviderAdapter.java | 5 +- .../sqlite3/oracle/SQLite3NoRECOracle.java | 22 +-- .../tidb/oracle/TiDBTLPWhereOracle.java | 44 +++++ 8 files changed, 204 insertions(+), 135 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ec8698f03..612c6f571 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,9 +24,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests @@ -46,9 +48,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up Citus @@ -92,9 +96,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up ClickHouse @@ -119,9 +125,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up CockroachDB @@ -142,9 +150,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up CockroachDB @@ -174,9 +184,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -192,9 +204,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: DuckDB Tests @@ -208,9 +222,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -225,9 +241,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Install MariaDB @@ -257,9 +275,11 @@ jobs: # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -282,9 +302,11 @@ jobs: # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -300,9 +322,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up MySQL @@ -330,9 +354,11 @@ jobs: postgresql password: 'sqlancer' postgresql db: 'test' - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -347,9 +373,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests @@ -366,9 +394,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests for QPG @@ -393,9 +423,11 @@ jobs: docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" sleep 60s - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: StoneDB Tests @@ -410,9 +442,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up TiDB @@ -433,9 +467,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up TiDB @@ -456,9 +492,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up Yugabyte @@ -478,9 +516,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: install mysql client run: | sudo apt update @@ -509,58 +549,6 @@ jobs: - name: Run Tests run: DORIS_AVAILABLE=true mvn -Dtest=TestDoris test - - java13: - name: Java 13 Compatibility (DuckDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up JDK 13 - uses: actions/setup-java@v1.4.4 - with: - java-version: 13 - - name: Build - run: mvn -B package -DskipTests=true - - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - - java14: - name: Java 14 Compatibility (DuckDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up JDK 14 - uses: actions/setup-java@v1.4.4 - with: - java-version: 14 - - name: Build - run: mvn -B package -DskipTests=true - - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - - java15: - name: Java 15 EA Compatibility (DuckDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up JDK 15 - uses: actions/setup-java@v1.4.4 - with: - java-version: 15-ea - - name: Build - run: mvn -B package -DskipTests=true - - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - reducer: name: Reducer Tests runs-on: ubuntu-latest @@ -570,9 +558,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: Run Tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 342a358f3..6194c1529 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Maven Central Repository - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD @@ -29,9 +30,10 @@ jobs: - name: Check out the repo uses: actions/checkout@v2 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Push to Docker Hub diff --git a/docs/testCaseReduction.md b/docs/testCaseReduction.md index c95770688..ee317f791 100644 --- a/docs/testCaseReduction.md +++ b/docs/testCaseReduction.md @@ -11,7 +11,26 @@ The AST-based reducer can shorten a statement by applying AST level transformati The transformations are implemented by [JSQLParser](https://github.com/JSQLParser/JSqlParser), a RDBMS agnostic SQL statement parser that can translate SQL statements into a traversable hierarchy of Java classes. JSQLParser provides support for the SQL standard as well as major SQL dialects. The AST-based reducer works for any SQL dialects that can be parsed by this tool. -## Enable reducers +## Implementing reproducer +Determining whether a bug persists after reducing statements +is an undecidable task for general transformations. +In practice, reducers use the [reproducer](../src/sqlancer/Reproducer.java) to determine +if a bug remains after statements have been removed or modified. +The reducer's responsibility is to verify if the current state, +formed by the pared-down statements, +continues to yield incorrect results for specific queries. + +Different oracles have distinct logic for determination, +meaning a universal reproducer doesn't exist. +Each oracle type needs its own reproducer implementation. +If reproducer is not implemented for specific oracle, +test case reduction is not available while using the oracle. + +Oracles for which reproducers have currently been implemented include: +1. for [`SQLite3NoRECOracle`](../src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java) +2. for [`TiDBTLPWhereOracle`](../src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java) + +## Using reducers Test-case reduction is disabled by default. The statement reducer can be enabled by passing `--use-reducer` when starting SQLancer. If you wish to further shorten each statements, you need to additionally pass the `--reduce-ast` parameter so that the AST-based reduction is applied. Note: if `--reduce-ast` is set, `--use-reducer` option must be enabled first. @@ -28,4 +47,4 @@ There are also options to define timeout seconds and max steps of reduction for ## Reduction logs If test-case reduction is enabled, each time the reducer performs a reduction step successfully,it prints the reduced statements to the log file, overwriting the previous ones. -The log files will be stored in the following format: `logs//reduce/-reduce.log`. For instance, if the tested DBMS is SQLite3 and the current database is named database0, the log file will be located at `logs/sqlite3/reduce/database0-reduce.log`. \ No newline at end of file +The log files will be stored in the following format: `logs//reduce/-reduce.log`. For instance, if the tested DBMS is SQLite3 and the current database is named database0, the log file will be located at `logs/sqlite3/reduce/database0-reduce.log`. diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java index 36f16c2ac..876a2da12 100644 --- a/src/sqlancer/ASTBasedReducer.java +++ b/src/sqlancer/ASTBasedReducer.java @@ -103,7 +103,8 @@ public void reduce(G state, Reproducer reproducer, G newGlobalState) throws E }); if (!initFlag) { - System.out.println("Error when parsing the statement at transformer :" + t); + newGlobalState.getLogger() + .logReducer("warning: failed parsing the statement at transformer : " + t); continue; } t.apply(); diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 89fbafd38..ebb3f1213 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -81,6 +81,7 @@ public static final class StateLogger { private FileWriter logFileWriter; public FileWriter currentFileWriter; private FileWriter queryPlanFileWriter; + private FileWriter reduceFileWriter; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; @@ -201,14 +202,14 @@ public FileWriter getReduceFileWriter() { if (!useReducer) { throw new UnsupportedOperationException(); } - FileWriter fileWriter; - try { - fileWriter = new FileWriter(reduceFile, false); - } catch (IOException e) { - throw new AssertionError(e); + if (reduceFileWriter == null) { + try { + reduceFileWriter = new FileWriter(reduceFile, false); + } catch (IOException e) { + throw new AssertionError(e); + } } - - return fileWriter; + return reduceFileWriter; } public void writeCurrent(StateToReproduce state) { @@ -257,6 +258,26 @@ public void writeQueryPlan(String queryPlan) { } } + public void logReducer(String reducerLog) { + FileWriter reduceFileWriter = getReduceFileWriter(); + + StringBuilder sb = new StringBuilder(); + sb.append("[reducer log] "); + sb.append(reducerLog); + try { + reduceFileWriter.write(sb.toString()); + } catch (IOException e) { + throw new AssertionError(e); + } finally { + try { + reduceFileWriter.flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + public void logReduced(StateToReproduce state) { FileWriter reduceFileWriter = getReduceFileWriter(); @@ -272,7 +293,6 @@ public void logReduced(StateToReproduce state) { } finally { try { reduceFileWriter.flush(); - reduceFileWriter.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -293,7 +313,6 @@ public void logException(Throwable reduce, StateToReproduce state) { try { logFileWriter2.flush(); } catch (IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } @@ -449,9 +468,11 @@ public void run() throws Exception { if (options.reduceAST() && !options.useReducer()) { throw new AssertionError("To reduce AST, use-reducer option must be enabled first"); } - if (reproducer != null && options.useReducer()) { - System.out.println("EXPERIMENTAL: Trying to reduce queries using a simple reducer."); - // System.out.println("Reduced query will be output to stdout but not logs."); + if (options.useReducer()) { + if (reproducer == null) { + logger.getReduceFileWriter().write("current oracle does not support experimental reducer."); + throw new IgnoreMeException(); + } G newGlobalState = createGlobalState(); newGlobalState.setState(stateToRepro); newGlobalState.setRandomly(r); @@ -470,7 +491,14 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } - throw new AssertionError("Found a potential bug"); + try { + logger.getReduceFileWriter().close(); + logger.reduceFileWriter = null; + } catch (IOException e) { + throw new AssertionError(e); + } + + throw new AssertionError("Found a potential bug, please check reducer log for detail."); } } } diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 1972557d6..346567300 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -60,8 +60,7 @@ public Reproducer generateAndTestDatabase(G globalState) throws Exception { try { oracle.check(); globalState.getManager().incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - + } catch (IgnoreMeException ignored) { } catch (AssertionError e) { Reproducer reproducer = oracle.getLastReproducer(); if (reproducer != null) { @@ -69,7 +68,6 @@ public Reproducer generateAndTestDatabase(G globalState) throws Exception { } throw e; } - assert localState != null; localState.executedWithoutError(); } } @@ -141,7 +139,6 @@ public void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws E } catch (IgnoreMeException e) { } - assert localState != null; localState.executedWithoutError(); } // exit loop to mutate tables if no new query plans have been found after a while diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 3a5a5c9d0..174efcbb6 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Function; import sqlancer.IgnoreMeException; @@ -38,7 +39,7 @@ public class SQLite3NoRECOracle extends NoRECBase implements private SQLite3ExpressionGenerator gen; private Reproducer reproducer; - private class SQLite3NoRECReproducer implements Reproducer { + private static class SQLite3NoRECReproducer implements Reproducer { private final Function optimizedQuery; private final Function unoptimizedQuery; @@ -50,7 +51,7 @@ private class SQLite3NoRECReproducer implements Reproducer { @Override public boolean bugStillTriggers(SQLite3GlobalState globalState) { - return optimizedQuery.apply(globalState) != unoptimizedQuery.apply(globalState); + return !Objects.equals(optimizedQuery.apply(globalState), unoptimizedQuery.apply(globalState)); } } @@ -119,13 +120,7 @@ private Function getUnoptimizedQuery(SQLite3Select logger.writeCurrent(unoptimizedQueryString); } SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - return new Function() { - - @Override - public Integer apply(SQLite3GlobalState state) { - return extractCounts(q, state); - } - }; + return state -> extractCounts(q, state); } private Function getOptimizedQuery(SQLite3Select select, @@ -147,14 +142,7 @@ private Function getOptimizedQuery(SQLite3Select se logger.writeCurrent(optimizedQueryString); } SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - return new Function() { - - @Override - public Integer apply(SQLite3GlobalState state) { - return useAggregate ? extractCounts(q, state) : countRows(q, state); - } - - }; + return state -> useAggregate ? extractCounts(q, state) : countRows(q, state); } private int countRows(SQLQueryAdapter q, SQLite3GlobalState globalState) { diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java index e03b347f3..f3640800e 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java @@ -6,6 +6,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.visitor.TiDBVisitor; @@ -13,14 +14,50 @@ public class TiDBTLPWhereOracle extends TiDBTLPBase { private String generatedQueryString; + private Reproducer reproducer; public TiDBTLPWhereOracle(TiDBGlobalState state) { super(state); TiDBErrors.addExpressionErrors(errors); } + private class TiDBTLPWhereReproducer implements Reproducer { + final String firstQueryString; + final String secondQueryString; + final String thirdQueryString; + final String originalQueryString; + final List resultSet; + final boolean orderBy; + + TiDBTLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, + String originalQueryString, List resultSet, boolean orderBy) { + this.firstQueryString = firstQueryString; + this.secondQueryString = secondQueryString; + this.thirdQueryString = thirdQueryString; + this.originalQueryString = originalQueryString; + this.resultSet = resultSet; + this.orderBy = orderBy; + } + + @Override + public boolean bugStillTriggers(TiDBGlobalState globalState) { + try { + List combinedString1 = new ArrayList<>(); + List secondResultSet1 = ComparatorHelper.getCombinedResultSet(firstQueryString, + secondQueryString, thirdQueryString, combinedString1, !orderBy, globalState, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet1, originalQueryString, + combinedString1, globalState); + } catch (AssertionError triggeredError) { + return true; + } catch (SQLException ignored) { + } + return false; + } + } + @Override public void check() throws SQLException { + reproducer = null; super.check(); select.setWhereClause(null); String originalQueryString = TiDBVisitor.asString(select); @@ -42,6 +79,8 @@ public void check() throws SQLException { thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); + reproducer = new TiDBTLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, + originalQueryString, resultSet, orderBy); } @Override @@ -49,4 +88,9 @@ public String getLastQueryString() { return generatedQueryString; } + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + } From 18f118f7e8cb597bb27acaa07ca6bc8ae183d4a1 Mon Sep 17 00:00:00 2001 From: Akshat Jaimini Date: Wed, 8 Nov 2023 19:37:00 +0530 Subject: [PATCH 0823/1284] CERT Implementation for PostgreSQL (#896) * Added CERT to Postgres Oracles * Postgres CERT Oracle, added Column Reference * Removed unwanted changes in .settings * Removed .factorypath file * Rows extracted from query plan and query plan successfully extracted * JOIN mutation added * Removed unnecessary code * Restored jdt.core.prefs * Removed unnecessary files * Removed PostgresErrors file --- .../PostgresExpectedValueVisitor.java | 11 + src/sqlancer/postgres/PostgresOptions.java | 12 + .../postgres/PostgresToStringVisitor.java | 12 + src/sqlancer/postgres/PostgresVisitor.java | 10 + .../postgres/ast/PostgresColumnReference.java | 15 + src/sqlancer/postgres/ast/PostgresJoin.java | 73 ++++- .../postgres/ast/PostgresTableReference.java | 15 + src/sqlancer/postgres/gen/PostgresCommon.java | 1 + .../postgres/oracle/PostgresCERTOracle.java | 266 ++++++++++++++++++ 9 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/postgres/ast/PostgresColumnReference.java create mode 100644 src/sqlancer/postgres/ast/PostgresTableReference.java create mode 100644 src/sqlancer/postgres/oracle/PostgresCERTOracle.java diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index d75aa4a5e..efc1b7442 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -5,6 +5,7 @@ import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; +import sqlancer.postgres.ast.PostgresColumnReference; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; @@ -20,6 +21,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -75,6 +77,15 @@ public void visit(PostgresPrefixOperation op) { visit(op.getExpression()); } + @Override + public void visit(PostgresColumnReference column) { + print(column); + } + + @Override + public void visit(PostgresTableReference tb) { + } + @Override public void visit(PostgresSelect op) { visit(op.getWhereClause()); diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index bb76404de..ced65d5da 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -13,6 +13,7 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; +import sqlancer.postgres.oracle.PostgresCERTOracle; import sqlancer.postgres.oracle.PostgresFuzzer; import sqlancer.postgres.oracle.PostgresNoRECOracle; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; @@ -78,6 +79,17 @@ public TestOracle create(PostgresGlobalState globalState) t return new CompositeTestOracle(oracles, globalState); } }, + CERT { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, FUZZER { @Override public TestOracle create(PostgresGlobalState globalState) throws Exception { diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 1e95ae745..d1766e6c8 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -11,6 +11,7 @@ import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; +import sqlancer.postgres.ast.PostgresColumnReference; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; @@ -28,6 +29,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; public final class PostgresToStringVisitor extends ToStringVisitor implements PostgresVisitor { @@ -46,6 +48,11 @@ public String get() { return sb.toString(); } + @Override + public void visit(PostgresColumnReference column) { + sb.append(column.getColumn().getFullQualifiedName()); + } + @Override public void visit(PostgresPostfixOperation op) { sb.append("("); @@ -87,6 +94,11 @@ public void visit(PostgresSubquery subquery) { sb.append(subquery.getName()); } + @Override + public void visit(PostgresTableReference ref) { + sb.append(ref.getTable().getName()); + } + @Override public void visit(PostgresSelect s) { sb.append("SELECT "); diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index d66dd1271..d9c82232c 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -9,6 +9,7 @@ import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; +import sqlancer.postgres.ast.PostgresColumnReference; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; @@ -24,6 +25,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; import sqlancer.postgres.gen.PostgresExpressionGenerator; public interface PostgresVisitor { @@ -34,6 +36,10 @@ public interface PostgresVisitor { void visit(PostgresColumnValue c); + void visit(PostgresColumnReference c); + + void visit(PostgresTableReference tb); + void visit(PostgresPrefixOperation op); void visit(PostgresSelect op); @@ -103,6 +109,10 @@ default void visit(PostgresExpression expression) { visit((PostgresSubquery) expression); } else if (expression instanceof PostgresLikeOperation) { visit((PostgresLikeOperation) expression); + } else if (expression instanceof PostgresColumnReference) { + visit((PostgresColumnReference) expression); + } else if (expression instanceof PostgresTableReference) { + visit((PostgresTableReference) expression); } else { throw new AssertionError(expression); } diff --git a/src/sqlancer/postgres/ast/PostgresColumnReference.java b/src/sqlancer/postgres/ast/PostgresColumnReference.java new file mode 100644 index 000000000..0d835bf89 --- /dev/null +++ b/src/sqlancer/postgres/ast/PostgresColumnReference.java @@ -0,0 +1,15 @@ +package sqlancer.postgres.ast; + +import sqlancer.postgres.PostgresSchema.PostgresColumn; + +public class PostgresColumnReference implements PostgresExpression { + private final PostgresColumn c; + + public PostgresColumnReference(PostgresColumn c) { + this.c = c; + } + + public PostgresColumn getColumn() { + return c; + } +} diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index 57b92ddf3..d1a076bbe 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -1,7 +1,14 @@ package sqlancer.postgres.ast; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import sqlancer.Randomly; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.gen.PostgresExpressionGenerator; public class PostgresJoin implements PostgresExpression { @@ -12,22 +19,84 @@ public static PostgresJoinType getRandom() { return Randomly.fromOptions(values()); } + public static PostgresJoinType getRandomExcept(PostgresJoinType... exclude) { + PostgresJoinType[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(PostgresJoinType[]::new); + return Randomly.fromOptions(values); + } + } private final PostgresExpression tableReference; - private final PostgresExpression onClause; - private final PostgresJoinType type; + private PostgresExpression onClause; + private PostgresJoinType type; + private final PostgresExpression leftTable; + private final PostgresExpression rightTable; public PostgresJoin(PostgresExpression tableReference, PostgresExpression onClause, PostgresJoinType type) { this.tableReference = tableReference; this.onClause = onClause; this.type = type; + this.leftTable = null; + this.rightTable = null; + } + + public PostgresJoin(PostgresExpression leftTable, PostgresExpression rightTable, PostgresJoinType joinType, + PostgresExpression whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.type = joinType; + this.onClause = whereCondition; + this.tableReference = null; + } + + public static PostgresJoin createJoin(PostgresExpression left, PostgresExpression right, PostgresJoinType type, + PostgresExpression onClause) { + if (type == PostgresJoinType.CROSS) { + return new PostgresJoin(left, right, type, null); + } else { + return new PostgresJoin(left, right, type, onClause); + } + } + + public static List getJoins(List tableList, + PostgresGlobalState globalState) { + // Clone Table to prevent the original list from being manipulated + List tbl = new ArrayList<>(tableList); + List joinExpressions = new ArrayList<>(); + while (tbl.size() >= 2 && Randomly.getBoolean()) { + PostgresTableReference left = (PostgresTableReference) tbl.remove(0); + PostgresTableReference right = (PostgresTableReference) tbl.remove(0); + List columns = new ArrayList<>(); + columns.addAll(left.getTable().getColumns()); + columns.addAll(right.getTable().getColumns()); + PostgresExpressionGenerator joinGen = new PostgresExpressionGenerator(globalState).setColumns(columns); + joinExpressions.add(PostgresJoin.createJoin(left, right, PostgresJoinType.getRandom(), + joinGen.generateExpression(0, PostgresDataType.BOOLEAN))); + } + return joinExpressions; + } + + public void setOnClause(PostgresExpression clause) { + this.onClause = clause; + } + + public void setType(PostgresJoinType type) { + this.type = type; } public PostgresExpression getTableReference() { return tableReference; } + public PostgresExpression getLeftTable() { + return leftTable; + } + + public PostgresExpression getRightTable() { + return rightTable; + } + public PostgresExpression getOnClause() { return onClause; } diff --git a/src/sqlancer/postgres/ast/PostgresTableReference.java b/src/sqlancer/postgres/ast/PostgresTableReference.java new file mode 100644 index 000000000..2abf8a7d2 --- /dev/null +++ b/src/sqlancer/postgres/ast/PostgresTableReference.java @@ -0,0 +1,15 @@ +package sqlancer.postgres.ast; + +import sqlancer.postgres.PostgresSchema.PostgresTable; + +public class PostgresTableReference implements PostgresExpression { + private final PostgresTable table; + + public PostgresTableReference(PostgresTable table) { + this.table = table; + } + + public PostgresTable getTable() { + return table; + } +} diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 99bb7fa66..497ad6d0b 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -134,6 +134,7 @@ public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { errors.add("value too long for type character"); + errors.add("cannot insert a non-DEFAULT value into column"); errors.add("not found in view targetlist"); } diff --git a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java new file mode 100644 index 000000000..9b509834e --- /dev/null +++ b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java @@ -0,0 +1,266 @@ +package sqlancer.postgres.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CERTOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTables; +import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation.BinaryLogicalOperator; +import sqlancer.postgres.ast.PostgresColumnReference; +import sqlancer.postgres.ast.PostgresConstant; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; + +public class PostgresCERTOracle extends CERTOracleBase implements TestOracle { + private PostgresExpressionGenerator gen; + private PostgresSelect select; + + public PostgresCERTOracle(PostgresGlobalState globalState) { + super(globalState); + PostgresCommon.addCommonExpressionErrors(errors); + PostgresCommon.addCommonInsertUpdateErrors(errors); + PostgresCommon.addGroupingErrors(errors); + PostgresCommon.addCommonInsertUpdateErrors(errors); + PostgresCommon.addCommonRangeExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + queryPlan1Sequences = new ArrayList<>(); + queryPlan2Sequences = new ArrayList<>(); + + // Generate Random Query + PostgresTables tables = state.getSchema().getRandomTableNonEmptyTables(); + List tableList = tables.getTables().stream().map(t -> new PostgresTableReference(t)) + .collect(Collectors.toList()); + gen = new PostgresExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new PostgresColumnReference(c)).collect(Collectors.toList())); + + select = new PostgresSelect(); + select.setFetchColumns(fetchColumns); + select.setFromList(tableList); + List joins = PostgresJoin.getJoins(tableList, state); + select.setJoinList(joins); + + select.setSelectType(Randomly.fromOptions(PostgresSelect.SelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + } + + // First query row count + String queryString1 = PostgresVisitor.asString(select); + int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + // JOIN and LIMIT mutations not added + boolean increase = mutate(Mutator.LIMIT); + + // Second Query row count + String queryString2 = PostgresVisitor.asString(select); + int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + // Check query plan equivalence + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check results + if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { + throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 + + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + } + } + + @Override + protected boolean mutateJoin() { + if (select.getJoinList().isEmpty()) { + return false; + } + PostgresJoin join = (PostgresJoin) Randomly.fromList(select.getJoinList()); + + // Exclude CROSS for on condition + if (join.getType() == PostgresJoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((PostgresTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((PostgresTableReference) join.getRightTable()).getTable().getColumns()); + PostgresExpressionGenerator joinGen2 = new PostgresExpressionGenerator(state).setColumns(columns); + join.setOnClause(joinGen2.generateExpression(0, PostgresDataType.BOOLEAN)); + } + + PostgresJoinType newJoinType = PostgresJoinType.INNER; + if (join.getType() == PostgresJoinType.LEFT || join.getType() == PostgresJoinType.RIGHT) { + newJoinType = PostgresJoinType.getRandomExcept(PostgresJoinType.LEFT, PostgresJoinType.RIGHT); + } else { + newJoinType = PostgresJoinType.getRandomExcept(join.getType()); + } + boolean increase = join.getType().ordinal() < newJoinType.ordinal(); + join.setType(newJoinType); + if (newJoinType == PostgresJoinType.CROSS) { + join.setOnClause(null); + } + return increase; + } + + @Override + protected boolean mutateDistinct() { + PostgresSelect.SelectType selectType = select.getSelectOption(); + if (selectType != PostgresSelect.SelectType.ALL) { + select.setSelectType(PostgresSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(PostgresSelect.SelectType.DISTINCT); + return false; + } + } + + @Override + protected boolean mutateWhere() { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + return increase; + } + + @Override + protected boolean mutateGroupBy() { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + @Override + protected boolean mutateHaving() { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + @Override + protected boolean mutateAnd() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + @Override + protected boolean mutateOr() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + @Override + protected boolean mutateLimit() { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + Randomly r = new Randomly(); + select.setLimitClause(PostgresConstant.createIntConstant((int) Math.abs(r.getInteger()))); + } + return increase; + } + + private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + throws AssertionError, SQLException { + int row = -1; + String explainQuery = "EXPLAIN " + selectStr; + + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String content = rs.getString(1).trim(); + if (content.contains("rows=")) { + try { + int ind = content.indexOf("rows="); + int number = Integer.parseInt(content.substring(ind + 5).split(" ")[0]); + if (row == -1) { + row = number; + + } + } catch (Exception e) { + } + } + // Proper Formatting TBD + String[] planPart = content.split("-> "); + String plan = planPart[planPart.length - 1]; + queryPlanSequences.add(plan.split(" ")[0].trim()); + } + + } + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + if (row == -1) { + throw new IgnoreMeException(); + } + return row; + + } +} From 201fd4c24aba5378447dc6b521e86d3b85ee485f Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Thu, 30 Nov 2023 14:18:34 +0100 Subject: [PATCH 0824/1284] Fix query plan collection for TiDB (#900) For `EXPLAIN`, the output includes the random id for each operator, such as `IndexLookUp_10`. While, according to the technical paper of QPG, the random id should be removed. Therefore, we fix this issue by using `EXPLAIN FORMAT=simplified`. --- src/sqlancer/tidb/TiDBProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 3e4133761..c5c334f11 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -208,7 +208,7 @@ public String getQueryPlan(String selectStr, TiDBGlobalState globalState) throws } } - SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN " + selectStr); + SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN FORMAT=brief " + selectStr); try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs != null) { while (rs.next()) { From 24a42644b0fd03fa8e4b5476b6a20c4afcd1a445 Mon Sep 17 00:00:00 2001 From: Marcus Gartner <1128750+mgartner@users.noreply.github.com> Date: Wed, 13 Dec 2023 00:05:54 -0800 Subject: [PATCH 0825/1284] Upgrade CockroachDB to v23.2.0-beta.1 (#902) --- .github/workflows/main.yml | 14 +++++++------- .../gen/CockroachDBSetClusterSettingGenerator.java | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 612c6f571..af92103d7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -134,11 +134,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-beta.2.linux-amd64.tgz | tar xvz - cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.2.0-beta.1.linux-amd64.tgz | tar xvz + cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -159,11 +159,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-beta.2.linux-amd64.tgz | tar xvz - cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.2.0-beta.1.linux-amd64.tgz | tar xvz + cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test @@ -567,4 +567,4 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - mvn -Dtest=TestStatementReducer test \ No newline at end of file + mvn -Dtest=TestStatementReducer test diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index cfc89562a..a948a1237 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -17,7 +17,8 @@ private CockroachDBSetClusterSettingGenerator() { private enum CockroachDBClusterSetting { BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), - RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), + RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", + (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), SQL_STATS_HISTOGRAM_COLLECTION_ENABLED("sql.stats.histogram_collection.enabled", CockroachDBSetSessionGenerator::onOff), From 228dd53a75abe7f436fbe6090d53b3765a1459e0 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 30 Dec 2023 07:29:19 +0100 Subject: [PATCH 0826/1284] Update QPG paper link (#903) --- docs/PAPERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/PAPERS.md b/docs/PAPERS.md index 6510930e1..c0e13dd3e 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -53,7 +53,7 @@ This paper describes TLP, a metamorphic testing approach that can detect various ## Testing Database Engines via Query Plan Guidance -This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. A preprint is available [here](http://jinshengba.me/assets/pdf/qpg_icse23.pdf). +This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. A preprint is available [here](http://bajinsheng.github.io/assets/pdf/qpg_icse23.pdf). ``` @inproceedings{Ba2023QPG, From 0d6ba67c92d2a844fb9c4adb713655f9e5934ae1 Mon Sep 17 00:00:00 2001 From: Aryan Patil <95161886+AryanP45@users.noreply.github.com> Date: Mon, 8 Jan 2024 22:00:48 +0530 Subject: [PATCH 0827/1284] Typo in testH2 (#905) --- test/sqlancer/dbms/TestH2.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/sqlancer/dbms/TestH2.java b/test/sqlancer/dbms/TestH2.java index 8f13170ff..701f6d50b 100644 --- a/test/sqlancer/dbms/TestH2.java +++ b/test/sqlancer/dbms/TestH2.java @@ -10,10 +10,10 @@ public class TestH2 { @Test - public void testDuckDB() { + public void testH2DB() { String h2Available = System.getenv("H2_AVAILABLE"); - boolean mariaDBIsAvailable = h2Available != null && h2Available.equalsIgnoreCase("true"); - assumeTrue(mariaDBIsAvailable); + boolean h2DBIsAvailable = h2Available != null && h2Available.equalsIgnoreCase("true"); + assumeTrue(h2DBIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "h2" })); From 3e960fb16fac42ed8f43eecf25eb4d09a5be9d85 Mon Sep 17 00:00:00 2001 From: Robins Date: Wed, 17 Jan 2024 03:18:01 +1030 Subject: [PATCH 0828/1284] Make --test-collations=false more robust. (#908) * Make --test-collations=false more robust. With --test-collations=false, database creation would now enforce using UTF8 and template0 database. This ensures that postgres clusters created in other locales still try to create UTF8 databases and test other aspects of the database engine. No change made to scenarios where --test-collations=true. * Conflate default CREATE DATABASE options into single string if --test-collations=false. --- src/sqlancer/postgres/PostgresProvider.java | 22 ++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index e9396a628..62c43cd5a 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -325,18 +325,22 @@ protected void prepareTables(PostgresGlobalState globalState) throws Exception { private String getCreateDatabaseCommand(PostgresGlobalState state) { StringBuilder sb = new StringBuilder(); sb.append("CREATE DATABASE " + databaseName + " "); - if (Randomly.getBoolean() && ((PostgresOptions) state.getDbmsSpecificOptions()).testCollations) { + if (((PostgresOptions) state.getDbmsSpecificOptions()).testCollations) { if (Randomly.getBoolean()) { - sb.append("WITH ENCODING '"); - sb.append(Randomly.fromOptions("utf8")); - sb.append("' "); - } - for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { - if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { - sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + if (Randomly.getBoolean()) { + sb.append("WITH ENCODING '"); + sb.append(Randomly.fromOptions("utf8")); + sb.append("' "); + } + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } } + sb.append(" TEMPLATE template0"); } - sb.append(" TEMPLATE template0"); + } else { + sb.append("WITH ENCODING 'UTF8' TEMPLATE template0"); } return sb.toString(); } From 5560b72e17dbb0f7817d0afd1f200884daaaebed Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Thu, 22 Feb 2024 20:26:25 +0800 Subject: [PATCH 0829/1284] Update options to be of the same type (#911) Fixes the build error: - java: The method fromOptions(T...) of type sqlancer.Randomly is not applicable as the formal varargs element type T is not accessible here --- src/sqlancer/mariadb/gen/MariaDBSetGenerator.java | 6 +++--- src/sqlancer/mysql/gen/MySQLAlterTable.java | 6 +++--- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index d437d4537..f4caf0ec6 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -35,11 +35,11 @@ private enum Action { AUTOCOMMIT("autocommit", (r) -> 1, Scope.GLOBAL, Scope.SESSION), // BIG_TABLES("big_tables", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - COMPLETION_TYPE("completion_type", (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", 0, 1, 2), - Scope.GLOBAL), // + COMPLETION_TYPE("completion_type", + (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", "0", "1", "2"), Scope.GLOBAL), // // BULK_INSERT_CACHE_SIZE("bulk_insert_buffer_size", (r) -> r.getLong(0, Long.MAX_VALUE), Scope.GLOBAL, // Scope.SESSION), - CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", 0, 1, 2), + CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", "0", "1", "2"), Scope.GLOBAL), CTE_MAX_RECURSION_DEPTH("cte_max_recursion_depth", (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), DELAY_KEY_WRITE("delay_key_write", (r) -> Randomly.fromOptions("ON", "OFF", "ALL"), Scope.GLOBAL), diff --git a/src/sqlancer/mysql/gen/MySQLAlterTable.java b/src/sqlancer/mysql/gen/MySQLAlterTable.java index cb5db6bde..f2c952016 100644 --- a/src/sqlancer/mysql/gen/MySQLAlterTable.java +++ b/src/sqlancer/mysql/gen/MySQLAlterTable.java @@ -130,15 +130,15 @@ private SQLQueryAdapter create() { break; case STATS_AUTO_RECALC: sb.append("STATS_AUTO_RECALC "); - sb.append(Randomly.fromOptions(0, 1, "DEFAULT")); + sb.append(Randomly.fromOptions("0", "1", "DEFAULT")); break; case STATS_PERSISTENT: sb.append("STATS_PERSISTENT "); - sb.append(Randomly.fromOptions(0, 1, "DEFAULT")); + sb.append(Randomly.fromOptions("0", "1", "DEFAULT")); break; case PACK_KEYS: sb.append("PACK_KEYS "); - sb.append(Randomly.fromOptions(0, 1, "DEFAULT")); + sb.append(Randomly.fromOptions("0", "1", "DEFAULT")); break; // not relevant: // case WITH_WITHOUT_VALIDATION: diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 546962fe6..821184617 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -34,11 +34,11 @@ private enum Action { AUTOCOMMIT("autocommit", (r) -> 1, Scope.GLOBAL, Scope.SESSION), // BIG_TABLES("big_tables", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - COMPLETION_TYPE("completion_type", (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", 0, 1, 2), - Scope.GLOBAL), // + COMPLETION_TYPE("completion_type", + (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", "0", "1", "2"), Scope.GLOBAL), // BULK_INSERT_CACHE_SIZE("bulk_insert_buffer_size", (r) -> r.getLong(0, Long.MAX_VALUE), Scope.GLOBAL, // Scope.SESSION), // - CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", 0, 1, 2), // + CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", "0", "1", "2"), // Scope.GLOBAL), // CTE_MAX_RECURSION_DEPTH("cte_max_recursion_depth", // (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), // From ece708d2c80150faa320f81cf203da7f93b89150 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Sat, 9 Mar 2024 09:27:45 +0800 Subject: [PATCH 0830/1284] Remove NoSQL database (#915) --- src/check_names.py | 2 - src/sqlancer/Main.java | 6 - .../arangodb/ArangoDBComparatorHelper.java | 73 ----- src/sqlancer/arangodb/ArangoDBConnection.java | 31 -- .../arangodb/ArangoDBLoggableFactory.java | 40 --- src/sqlancer/arangodb/ArangoDBOptions.java | 50 --- src/sqlancer/arangodb/ArangoDBProvider.java | 137 --------- .../arangodb/ArangoDBQueryAdapter.java | 16 - .../arangodb/ArangoDBQueryProvider.java | 6 - src/sqlancer/arangodb/ArangoDBSchema.java | 70 ----- .../arangodb/ast/ArangoDBConstant.java | 108 ------- .../arangodb/ast/ArangoDBExpression.java | 4 - src/sqlancer/arangodb/ast/ArangoDBSelect.java | 79 ----- .../ast/ArangoDBUnsupportedPredicate.java | 6 - .../ArangoDBComputedExpressionGenerator.java | 85 ----- .../gen/ArangoDBCreateIndexGenerator.java | 18 -- .../ArangoDBFilterExpressionGenerator.java | 153 --------- .../arangodb/gen/ArangoDBInsertGenerator.java | 39 --- .../arangodb/gen/ArangoDBTableGenerator.java | 44 --- .../query/ArangoDBConstantGenerator.java | 46 --- .../query/ArangoDBCreateIndexQuery.java | 54 ---- .../query/ArangoDBCreateTableQuery.java | 44 --- .../arangodb/query/ArangoDBInsertQuery.java | 66 ---- .../query/ArangoDBOptimizerRules.java | 57 ---- .../arangodb/query/ArangoDBSelectQuery.java | 88 ------ .../test/ArangoDBQueryPartitioningBase.java | 67 ---- .../ArangoDBQueryPartitioningWhereTester.java | 46 --- .../visitor/ArangoDBToQueryVisitor.java | 134 -------- .../arangodb/visitor/ArangoDBVisitor.java | 51 --- src/sqlancer/cosmos/CosmosProvider.java | 77 ----- .../mongodb/MongoDBComparatorHelper.java | 97 ------ src/sqlancer/mongodb/MongoDBConnection.java | 35 --- .../mongodb/MongoDBLoggableFactory.java | 40 --- src/sqlancer/mongodb/MongoDBOptions.java | 74 ----- src/sqlancer/mongodb/MongoDBProvider.java | 128 -------- src/sqlancer/mongodb/MongoDBQueryAdapter.java | 15 - .../mongodb/MongoDBQueryProvider.java | 6 - src/sqlancer/mongodb/MongoDBSchema.java | 97 ------ .../ast/MongoDBBinaryComparisonNode.java | 16 - .../mongodb/ast/MongoDBBinaryLogicalNode.java | 16 - src/sqlancer/mongodb/ast/MongoDBConstant.java | 252 --------------- .../mongodb/ast/MongoDBExpression.java | 4 - .../mongodb/ast/MongoDBRegexNode.java | 24 -- src/sqlancer/mongodb/ast/MongoDBSelect.java | 104 ------- .../ast/MongoDBUnaryLogicalOperatorNode.java | 16 - .../ast/MongoDBUnsupportedPredicate.java | 7 - .../MongoDBComputedExpressionGenerator.java | 89 ------ .../mongodb/gen/MongoDBConstantGenerator.java | 86 ------ .../mongodb/gen/MongoDBIndexGenerator.java | 25 -- .../mongodb/gen/MongoDBInsertGenerator.java | 38 --- .../gen/MongoDBMatchExpressionGenerator.java | 291 ------------------ .../mongodb/gen/MongoDBTableGenerator.java | 54 ---- .../query/MongoDBCreateIndexQuery.java | 77 ----- .../query/MongoDBCreateTableQuery.java | 115 ------- .../mongodb/query/MongoDBInsertQuery.java | 87 ------ .../mongodb/query/MongoDBRemoveQuery.java | 59 ---- .../mongodb/query/MongoDBSelectQuery.java | 147 --------- .../test/MongoDBColumnTestReference.java | 40 --- .../test/MongoDBDocumentRemovalBase.java | 90 ------ .../test/MongoDBDocumentRemovalTester.java | 49 --- .../test/MongoDBQueryPartitioningBase.java | 93 ------ .../MongoDBQueryPartitioningWhereTester.java | 48 --- .../mongodb/visitor/MongoDBNegateVisitor.java | 161 ---------- .../mongodb/visitor/MongoDBToLogVisitor.java | 195 ------------ .../visitor/MongoDBToQueryVisitor.java | 184 ----------- .../mongodb/visitor/MongoDBVisitor.java | 45 --- 66 files changed, 4601 deletions(-) delete mode 100644 src/sqlancer/arangodb/ArangoDBComparatorHelper.java delete mode 100644 src/sqlancer/arangodb/ArangoDBConnection.java delete mode 100644 src/sqlancer/arangodb/ArangoDBLoggableFactory.java delete mode 100644 src/sqlancer/arangodb/ArangoDBOptions.java delete mode 100644 src/sqlancer/arangodb/ArangoDBProvider.java delete mode 100644 src/sqlancer/arangodb/ArangoDBQueryAdapter.java delete mode 100644 src/sqlancer/arangodb/ArangoDBQueryProvider.java delete mode 100644 src/sqlancer/arangodb/ArangoDBSchema.java delete mode 100644 src/sqlancer/arangodb/ast/ArangoDBConstant.java delete mode 100644 src/sqlancer/arangodb/ast/ArangoDBExpression.java delete mode 100644 src/sqlancer/arangodb/ast/ArangoDBSelect.java delete mode 100644 src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBInsertQuery.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBSelectQuery.java delete mode 100644 src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java delete mode 100644 src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java delete mode 100644 src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java delete mode 100644 src/sqlancer/arangodb/visitor/ArangoDBVisitor.java delete mode 100644 src/sqlancer/cosmos/CosmosProvider.java delete mode 100644 src/sqlancer/mongodb/MongoDBComparatorHelper.java delete mode 100644 src/sqlancer/mongodb/MongoDBConnection.java delete mode 100644 src/sqlancer/mongodb/MongoDBLoggableFactory.java delete mode 100644 src/sqlancer/mongodb/MongoDBOptions.java delete mode 100644 src/sqlancer/mongodb/MongoDBProvider.java delete mode 100644 src/sqlancer/mongodb/MongoDBQueryAdapter.java delete mode 100644 src/sqlancer/mongodb/MongoDBQueryProvider.java delete mode 100644 src/sqlancer/mongodb/MongoDBSchema.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBConstant.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBExpression.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBRegexNode.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBSelect.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBTableGenerator.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBInsertQuery.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBRemoveQuery.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBSelectQuery.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBColumnTestReference.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java delete mode 100644 src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java delete mode 100644 src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java delete mode 100644 src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java delete mode 100644 src/sqlancer/mongodb/visitor/MongoDBVisitor.java diff --git a/src/check_names.py b/src/check_names.py index 048da7669..63c1f093f 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -33,12 +33,10 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): cwd = os.getcwd() print("Current working directory: {0}".format(cwd)) name_to_files: dict[str:List[str]] = dict() - name_to_files["ArangoDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "arangodb")) name_to_files["Citus"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "citus")) name_to_files["ClickHouse"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "clickhouse")) name_to_files["CnosDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cnosdb")) name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) - name_to_files["Cosmos"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cosmos")) name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) name_to_files["DuckDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "duckdb")) name_to_files["H2"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "h2")) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index ebb3f1213..1cbc02645 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -23,7 +23,6 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.JCommander.Builder; -import sqlancer.arangodb.ArangoDBProvider; import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.cnosdb.CnosDBProvider; @@ -31,7 +30,6 @@ import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.cosmos.CosmosProvider; import sqlancer.databend.DatabendProvider; import sqlancer.doris.DorisProvider; import sqlancer.duckdb.DuckDBProvider; @@ -39,7 +37,6 @@ import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.mariadb.MariaDBProvider; import sqlancer.materialize.MaterializeProvider; -import sqlancer.mongodb.MongoDBProvider; import sqlancer.mysql.MySQLProvider; import sqlancer.oceanbase.OceanBaseProvider; import sqlancer.postgres.PostgresProvider; @@ -732,12 +729,10 @@ private static void checkForIssue799(List> providers) if (providers.isEmpty()) { System.err.println( "No DBMS implementations (i.e., instantiations of the DatabaseProvider class) were found. You likely ran into an issue described in https://github.com/sqlancer/sqlancer/issues/799. As a workaround, I now statically load all supported providers as of June 7, 2023."); - providers.add(new ArangoDBProvider()); providers.add(new CitusProvider()); providers.add(new ClickHouseProvider()); providers.add(new CnosDBProvider()); providers.add(new CockroachDBProvider()); - providers.add(new CosmosProvider()); providers.add(new DatabendProvider()); providers.add(new DorisProvider()); providers.add(new DuckDBProvider()); @@ -745,7 +740,6 @@ private static void checkForIssue799(List> providers) providers.add(new HSQLDBProvider()); providers.add(new MariaDBProvider()); providers.add(new MaterializeProvider()); - providers.add(new MongoDBProvider()); providers.add(new MySQLProvider()); providers.add(new OceanBaseProvider()); providers.add(new PrestoProvider()); diff --git a/src/sqlancer/arangodb/ArangoDBComparatorHelper.java b/src/sqlancer/arangodb/ArangoDBComparatorHelper.java deleted file mode 100644 index 2a00a312d..000000000 --- a/src/sqlancer/arangodb/ArangoDBComparatorHelper.java +++ /dev/null @@ -1,73 +0,0 @@ -package sqlancer.arangodb; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.IgnoreMeException; -import sqlancer.Main; -import sqlancer.arangodb.query.ArangoDBSelectQuery; -import sqlancer.common.query.ExpectedErrors; - -public final class ArangoDBComparatorHelper { - - private ArangoDBComparatorHelper() { - - } - - public static List getResultSetAsDocumentList(ArangoDBSelectQuery query, - ArangoDBProvider.ArangoDBGlobalState state) throws Exception { - ExpectedErrors errors = query.getExpectedErrors(); - List result; - try { - query.executeAndGet(state); - Main.nrSuccessfulActions.addAndGet(1); - result = query.getResultSet(); - return result; - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw e; - } - Main.nrUnsuccessfulActions.addAndGet(1); - if (e.getMessage() == null) { - throw new AssertionError(query.getLogString(), e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(query.getLogString(), e); - } - - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - ArangoDBSelectQuery originalQuery) { - if (resultSet.size() != secondResultSet.size()) { - String assertionMessage = String.format("The Size of the result sets mismatch (%d and %d)!\n%s", - resultSet.size(), secondResultSet.size(), originalQuery.getLogString()); - throw new AssertionError(assertionMessage); - } - Set firstHashSet = new HashSet<>(resultSet); - Set secondHashSet = new HashSet<>(secondResultSet); - - if (!firstHashSet.equals(secondHashSet)) { - Set firstResultSetMisses = new HashSet<>(firstHashSet); - firstResultSetMisses.removeAll(secondHashSet); - Set secondResultSetMisses = new HashSet<>(secondHashSet); - secondResultSetMisses.removeAll(firstHashSet); - StringBuilder firstMisses = new StringBuilder(); - for (BaseDocument document : firstResultSetMisses) { - firstMisses.append(document).append(" "); - } - StringBuilder secondMisses = new StringBuilder(); - for (BaseDocument document : secondResultSetMisses) { - secondMisses.append(document).append(" "); - } - String assertMessage = String.format("The Content of the result sets mismatch!\n %s \n %s\n %s", - firstMisses.toString(), secondMisses.toString(), originalQuery.getLogString()); - throw new AssertionError(assertMessage); - } - } -} diff --git a/src/sqlancer/arangodb/ArangoDBConnection.java b/src/sqlancer/arangodb/ArangoDBConnection.java deleted file mode 100644 index b3e5b85d3..000000000 --- a/src/sqlancer/arangodb/ArangoDBConnection.java +++ /dev/null @@ -1,31 +0,0 @@ -package sqlancer.arangodb; - -import com.arangodb.ArangoDB; -import com.arangodb.ArangoDatabase; - -import sqlancer.SQLancerDBConnection; - -public class ArangoDBConnection implements SQLancerDBConnection { - - private final ArangoDB client; - private final ArangoDatabase database; - - public ArangoDBConnection(ArangoDB client, ArangoDatabase database) { - this.client = client; - this.database = database; - } - - @Override - public String getDatabaseVersion() throws Exception { - return client.getVersion().getVersion(); - } - - @Override - public void close() throws Exception { - client.shutdown(); - } - - public ArangoDatabase getDatabase() { - return database; - } -} diff --git a/src/sqlancer/arangodb/ArangoDBLoggableFactory.java b/src/sqlancer/arangodb/ArangoDBLoggableFactory.java deleted file mode 100644 index 927d9f320..000000000 --- a/src/sqlancer/arangodb/ArangoDBLoggableFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -package sqlancer.arangodb; - -import java.util.Arrays; - -import sqlancer.common.log.Loggable; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.log.LoggedString; -import sqlancer.common.query.Query; - -public class ArangoDBLoggableFactory extends LoggableFactory { - @Override - protected Loggable createLoggable(String input, String suffix) { - return new LoggedString(input + suffix); - } - - @Override - public Query getQueryForStateToReproduce(String queryString) { - throw new UnsupportedOperationException(); - } - - @Override - public Query commentOutQuery(Query query) { - throw new UnsupportedOperationException(); - } - - @Override - protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { - StringBuilder sb = new StringBuilder(); - sb.append("// Time: ").append(time).append("\n"); - sb.append("// Database: ").append(databaseName).append("\n"); - sb.append("// Database version: ").append(databaseVersion).append("\n"); - sb.append("// seed value: ").append(seedValue).append("\n"); - return new LoggedString(sb.toString()); - } - - @Override - public Loggable convertStacktraceToLoggable(Throwable throwable) { - return new LoggedString(Arrays.toString(throwable.getStackTrace()) + "\n" + throwable.getMessage()); - } -} diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java deleted file mode 100644 index 7e20883d5..000000000 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ /dev/null @@ -1,50 +0,0 @@ -package sqlancer.arangodb; - -import static sqlancer.arangodb.ArangoDBOptions.ArangoDBOracleFactory.QUERY_PARTITIONING; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.arangodb.ArangoDBProvider.ArangoDBGlobalState; -import sqlancer.arangodb.test.ArangoDBQueryPartitioningWhereTester; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; - -@Parameters(commandDescription = "ArangoDB (experimental)") -public class ArangoDBOptions implements DBMSSpecificOptions { - - @Parameter(names = "--oracle") - public List oracles = Arrays.asList(QUERY_PARTITIONING); - - @Parameter(names = "--test-random-type-inserts", description = "Insert random types instead of schema types.") - public boolean testRandomTypeInserts; - - @Parameter(names = "--max-number-indexes", description = "The maximum number of indexes used.", arity = 1) - public int maxNumberIndexes = 15; - - @Parameter(names = "--with-optimizer-rule-tests", description = "Adds an additional query, where a random set" - + "of optimizer rules are disabled.", arity = 1) - public boolean withOptimizerRuleTests; - - @Override - public List getTestOracleFactory() { - return oracles; - } - - public enum ArangoDBOracleFactory implements OracleFactory { - QUERY_PARTITIONING { - @Override - public TestOracle create(ArangoDBGlobalState globalState) throws Exception { - List> oracles = new ArrayList<>(); - oracles.add(new ArangoDBQueryPartitioningWhereTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - } - } -} diff --git a/src/sqlancer/arangodb/ArangoDBProvider.java b/src/sqlancer/arangodb/ArangoDBProvider.java deleted file mode 100644 index 7bdc2fa01..000000000 --- a/src/sqlancer/arangodb/ArangoDBProvider.java +++ /dev/null @@ -1,137 +0,0 @@ -package sqlancer.arangodb; - -import java.util.ArrayList; -import java.util.List; - -import com.arangodb.ArangoDB; -import com.arangodb.ArangoDatabase; -import com.google.auto.service.AutoService; - -import sqlancer.AbstractAction; -import sqlancer.DatabaseProvider; -import sqlancer.ExecutionTimer; -import sqlancer.GlobalState; -import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; -import sqlancer.Randomly; -import sqlancer.StatementExecutor; -import sqlancer.arangodb.gen.ArangoDBCreateIndexGenerator; -import sqlancer.arangodb.gen.ArangoDBInsertGenerator; -import sqlancer.arangodb.gen.ArangoDBTableGenerator; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.query.Query; - -@AutoService(DatabaseProvider.class) -public class ArangoDBProvider - extends ProviderAdapter { - - public ArangoDBProvider() { - super(ArangoDBGlobalState.class, ArangoDBOptions.class); - } - - enum Action implements AbstractAction { - INSERT(ArangoDBInsertGenerator::getQuery), CREATE_INDEX(ArangoDBCreateIndexGenerator::getQuery); - - private final ArangoDBQueryProvider queryProvider; - - Action(ArangoDBQueryProvider queryProvider) { - this.queryProvider = queryProvider; - } - - @Override - public Query getQuery(ArangoDBGlobalState globalState) throws Exception { - return queryProvider.getQuery(globalState); - } - } - - private static int mapActions(ArangoDBGlobalState globalState, Action a) { - Randomly r = globalState.getRandomly(); - switch (a) { - case INSERT: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case CREATE_INDEX: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumberIndexes); - default: - throw new AssertionError(a); - } - } - - public static class ArangoDBGlobalState extends GlobalState { - - private final List schemaTables = new ArrayList<>(); - - public void addTable(ArangoDBSchema.ArangoDBTable table) { - schemaTables.add(table); - } - - @Override - protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { - boolean logExecutionTime = getOptions().logExecutionTime(); - if (success && getOptions().printSucceedingStatements()) { - System.out.println(q.getLogString()); - } - if (logExecutionTime) { - getLogger().writeCurrent("//" + timer.end().asString()); - } - if (q.couldAffectSchema()) { - updateSchema(); - } - } - - @Override - protected ArangoDBSchema readSchema() throws Exception { - return new ArangoDBSchema(schemaTables); - } - } - - @Override - protected void checkViewsAreValid(ArangoDBGlobalState globalState) { - - } - - @Override - public void generateDatabase(ArangoDBGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { - boolean success; - do { - ArangoDBQueryAdapter queryAdapter = new ArangoDBTableGenerator().getQuery(globalState); - success = globalState.executeStatement(queryAdapter); - } while (!success); - } - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - ArangoDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public ArangoDBConnection createDatabase(ArangoDBGlobalState globalState) throws Exception { - ArangoDB arangoDB = new ArangoDB.Builder().user(globalState.getOptions().getUserName()) - .password(globalState.getOptions().getPassword()).build(); - ArangoDatabase database = arangoDB.db(globalState.getDatabaseName()); - try { - database.drop(); - // When the database does not exist, an ArangoDB exception is thrown. Since we are not sure - // if this is the first time the database is used, the simplest is dropping it and ignoring - // the exception. - } catch (Exception ignored) { - - } - arangoDB.createDatabase(globalState.getDatabaseName()); - database = arangoDB.db(globalState.getDatabaseName()); - return new ArangoDBConnection(arangoDB, database); - } - - @Override - public String getDBMSName() { - return "arangodb"; - } - - @Override - public LoggableFactory getLoggableFactory() { - return new ArangoDBLoggableFactory(); - } -} diff --git a/src/sqlancer/arangodb/ArangoDBQueryAdapter.java b/src/sqlancer/arangodb/ArangoDBQueryAdapter.java deleted file mode 100644 index 34cdb3709..000000000 --- a/src/sqlancer/arangodb/ArangoDBQueryAdapter.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.arangodb; - -import sqlancer.common.query.Query; - -public abstract class ArangoDBQueryAdapter extends Query { - @Override - public String getQueryString() { - // Should not be called as it is used only in SQL dependent classes - throw new UnsupportedOperationException(); - } - - @Override - public String getUnterminatedQueryString() { - throw new UnsupportedOperationException(); - } -} diff --git a/src/sqlancer/arangodb/ArangoDBQueryProvider.java b/src/sqlancer/arangodb/ArangoDBQueryProvider.java deleted file mode 100644 index 94a4ffda3..000000000 --- a/src/sqlancer/arangodb/ArangoDBQueryProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.arangodb; - -@FunctionalInterface -public interface ArangoDBQueryProvider { - ArangoDBQueryAdapter getQuery(S globalState) throws Exception; -} diff --git a/src/sqlancer/arangodb/ArangoDBSchema.java b/src/sqlancer/arangodb/ArangoDBSchema.java deleted file mode 100644 index 35e251b8b..000000000 --- a/src/sqlancer/arangodb/ArangoDBSchema.java +++ /dev/null @@ -1,70 +0,0 @@ -package sqlancer.arangodb; - -import java.util.Collections; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.common.schema.AbstractTables; -import sqlancer.common.schema.TableIndex; - -public class ArangoDBSchema extends AbstractSchema { - - public enum ArangoDBDataType { - INTEGER, DOUBLE, STRING, BOOLEAN; - - public static ArangoDBDataType getRandom() { - return Randomly.fromOptions(values()); - } - } - - public static class ArangoDBColumn extends AbstractTableColumn { - - private final boolean isId; - private final boolean isNullable; - - public ArangoDBColumn(String name, ArangoDBDataType type, boolean isId, boolean isNullable) { - super(name, null, type); - this.isId = isId; - this.isNullable = isNullable; - } - - public boolean isId() { - return isId; - } - - public boolean isNullable() { - return isNullable; - } - } - - public ArangoDBSchema(List databaseTables) { - super(databaseTables); - } - - public static class ArangoDBTables extends AbstractTables { - - public ArangoDBTables(List tables) { - super(tables); - } - } - - public static class ArangoDBTable - extends AbstractTable { - - public ArangoDBTable(String name, List columns, boolean isView) { - super(name, columns, Collections.emptyList(), isView); - } - - @Override - public long getNrRows(ArangoDBProvider.ArangoDBGlobalState globalState) { - throw new UnsupportedOperationException(); - } - } - - public ArangoDBTables getRandomTableNonEmptyTables() { - return new ArangoDBTables(Randomly.nonEmptySubset(getDatabaseTables())); - } -} diff --git a/src/sqlancer/arangodb/ast/ArangoDBConstant.java b/src/sqlancer/arangodb/ast/ArangoDBConstant.java deleted file mode 100644 index 351dbd822..000000000 --- a/src/sqlancer/arangodb/ast/ArangoDBConstant.java +++ /dev/null @@ -1,108 +0,0 @@ -package sqlancer.arangodb.ast; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.common.ast.newast.Node; - -public abstract class ArangoDBConstant implements Node { - private ArangoDBConstant() { - - } - - public abstract void setValueInDocument(BaseDocument document, String key); - - public abstract Object getValue(); - - public static class ArangoDBIntegerConstant extends ArangoDBConstant { - - private final int value; - - public ArangoDBIntegerConstant(int value) { - this.value = value; - } - - @Override - public void setValueInDocument(BaseDocument document, String key) { - document.addAttribute(key, value); - } - - @Override - public Object getValue() { - return value; - } - } - - public static Node createIntegerConstant(int value) { - return new ArangoDBIntegerConstant(value); - } - - public static class ArangoDBStringConstant extends ArangoDBConstant { - private final String value; - - public ArangoDBStringConstant(String value) { - this.value = value; - } - - @Override - public void setValueInDocument(BaseDocument document, String key) { - document.addAttribute(key, value); - } - - @Override - public Object getValue() { - return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; - } - } - - public static Node createStringConstant(String value) { - return new ArangoDBStringConstant(value); - } - - public static class ArangoDBBooleanConstant extends ArangoDBConstant { - private final boolean value; - - public ArangoDBBooleanConstant(boolean value) { - this.value = value; - } - - @Override - public void setValueInDocument(BaseDocument document, String key) { - document.addAttribute(key, value); - } - - @Override - public Object getValue() { - return value; - } - } - - public static Node createBooleanConstant(boolean value) { - return new ArangoDBBooleanConstant(value); - } - - public static class ArangoDBDoubleConstant extends ArangoDBConstant { - private final double value; - - public ArangoDBDoubleConstant(double value) { - if (Double.isInfinite(value) || Double.isNaN(value)) { - this.value = 0.0; - } else { - this.value = value; - } - } - - @Override - public void setValueInDocument(BaseDocument document, String key) { - document.addAttribute(key, value); - } - - @Override - public Object getValue() { - return value; - } - } - - public static Node createDoubleConstant(double value) { - return new ArangoDBDoubleConstant(value); - } -} diff --git a/src/sqlancer/arangodb/ast/ArangoDBExpression.java b/src/sqlancer/arangodb/ast/ArangoDBExpression.java deleted file mode 100644 index facbbfe9e..000000000 --- a/src/sqlancer/arangodb/ast/ArangoDBExpression.java +++ /dev/null @@ -1,4 +0,0 @@ -package sqlancer.arangodb.ast; - -public interface ArangoDBExpression { -} diff --git a/src/sqlancer/arangodb/ast/ArangoDBSelect.java b/src/sqlancer/arangodb/ast/ArangoDBSelect.java deleted file mode 100644 index 9fb91d553..000000000 --- a/src/sqlancer/arangodb/ast/ArangoDBSelect.java +++ /dev/null @@ -1,79 +0,0 @@ -package sqlancer.arangodb.ast; - -import java.util.List; - -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.common.ast.newast.Node; - -public class ArangoDBSelect implements Node { - private List fromColumns; - private List projectionColumns; - private boolean hasFilter; - private Node filterClause; - private boolean hasComputed; - private List> computedClause; - - public List getFromColumns() { - if (fromColumns == null || fromColumns.isEmpty()) { - throw new IllegalStateException(); - } - return fromColumns; - } - - public void setFromColumns(List fromColumns) { - if (fromColumns == null || fromColumns.isEmpty()) { - throw new IllegalStateException(); - } - this.fromColumns = fromColumns; - } - - public List getProjectionColumns() { - if (projectionColumns == null) { - throw new IllegalStateException(); - } - return projectionColumns; - } - - public void setProjectionColumns(List projectionColumns) { - if (projectionColumns == null) { - throw new IllegalStateException(); - } - this.projectionColumns = projectionColumns; - } - - public void setFilterClause(Node filterClause) { - if (filterClause == null) { - hasFilter = false; - this.filterClause = null; - return; - } - hasFilter = true; - this.filterClause = filterClause; - } - - public Node getFilterClause() { - return filterClause; - } - - public boolean hasFilter() { - return hasFilter; - } - - public void setComputedClause(List> computedColumns) { - if (computedColumns == null || computedColumns.isEmpty()) { - hasComputed = false; - this.computedClause = null; - return; - } - hasComputed = true; - this.computedClause = computedColumns; - } - - public List> getComputedClause() { - return computedClause; - } - - public boolean hasComputed() { - return hasComputed; - } -} diff --git a/src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java b/src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java deleted file mode 100644 index eabd25578..000000000 --- a/src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.arangodb.ast; - -import sqlancer.common.ast.newast.Node; - -public class ArangoDBUnsupportedPredicate implements Node { -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java deleted file mode 100644 index 01e2e557a..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java +++ /dev/null @@ -1,85 +0,0 @@ -package sqlancer.arangodb.gen; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBConstant; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; - -public class ArangoDBComputedExpressionGenerator - extends UntypedExpressionGenerator, ArangoDBSchema.ArangoDBColumn> { - private final ArangoDBProvider.ArangoDBGlobalState globalState; - - public ArangoDBComputedExpressionGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { - this.globalState = globalState; - } - - @Override - public Node generateConstant() { - ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); - switch (dataType) { - case INTEGER: - return ArangoDBConstant.createIntegerConstant((int) globalState.getRandomly().getInteger()); - case BOOLEAN: - return ArangoDBConstant.createBooleanConstant(Randomly.getBoolean()); - case DOUBLE: - return ArangoDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - case STRING: - return ArangoDBConstant.createStringConstant(globalState.getRandomly().getString()); - default: - throw new AssertionError(dataType); - } - } - - public enum ComputedFunction { - ADD(2, "+"), MINUS(2, "-"), MULTIPLY(2, "*"), DIVISION(2, "/"), MODULUS(2, "%"); - - private final int nrArgs; - private final String operatorName; - - ComputedFunction(int nrArgs, String operatorName) { - this.nrArgs = nrArgs; - this.operatorName = operatorName; - } - - public static ComputedFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - - public String getOperatorName() { - return operatorName; - } - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - ComputedFunction function = ComputedFunction.getRandom(); - return new NewFunctionNode<>(generateExpressions(function.getNrArgs(), depth + 1), function); - } - - @Override - protected Node generateColumn() { - return new ColumnReferenceNode<>(Randomly.fromList(columns)); - } - - @Override - public Node negatePredicate(Node predicate) { - throw new UnsupportedOperationException(); - } - - @Override - public Node isNull(Node expr) { - throw new UnsupportedOperationException(); - } -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java deleted file mode 100644 index 6a1b872da..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java +++ /dev/null @@ -1,18 +0,0 @@ -package sqlancer.arangodb.gen; - -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.query.ArangoDBCreateIndexQuery; - -public final class ArangoDBCreateIndexGenerator { - private ArangoDBCreateIndexGenerator() { - - } - - public static ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { - ArangoDBSchema.ArangoDBTable randomTable = globalState.getSchema().getRandomTable(); - ArangoDBSchema.ArangoDBColumn column = randomTable.getRandomColumn(); - return new ArangoDBCreateIndexQuery(column); - } -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java deleted file mode 100644 index 1a2fc4b5e..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java +++ /dev/null @@ -1,153 +0,0 @@ -package sqlancer.arangodb.gen; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBConstant; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.arangodb.ast.ArangoDBUnsupportedPredicate; -import sqlancer.common.ast.BinaryOperatorNode; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; - -public class ArangoDBFilterExpressionGenerator - extends UntypedExpressionGenerator, ArangoDBSchema.ArangoDBColumn> { - - private final ArangoDBProvider.ArangoDBGlobalState globalState; - private int numberOfComputedVariables; - - private enum Expression { - BINARY_LOGICAL, UNARY_PREFIX, BINARY_COMPARISON - } - - public ArangoDBFilterExpressionGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { - this.globalState = globalState; - } - - public void setNumberOfComputedVariables(int numberOfComputedVariables) { - this.numberOfComputedVariables = numberOfComputedVariables; - } - - @Override - public Node generateConstant() { - ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); - switch (dataType) { - case INTEGER: - return ArangoDBConstant.createIntegerConstant((int) globalState.getRandomly().getInteger()); - case BOOLEAN: - return ArangoDBConstant.createBooleanConstant(Randomly.getBoolean()); - case DOUBLE: - return ArangoDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - case STRING: - return ArangoDBConstant.createStringConstant(globalState.getRandomly().getString()); - default: - throw new AssertionError(dataType); - } - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - Expression expression = Randomly.fromList(possibleOptions); - switch (expression) { - case BINARY_COMPARISON: - BinaryOperatorNode.Operator op = ArangoDBBinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), - ArangoDBUnaryPrefixOperator.getRandom()); - case BINARY_LOGICAL: - op = ArangoDBBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - default: - throw new AssertionError(expression); - } - } - - @Override - protected Node generateColumn() { - ArangoDBSchema.ArangoDBTable dummy = new ArangoDBSchema.ArangoDBTable("", new ArrayList<>(), false); - if (Randomly.getBoolean() || numberOfComputedVariables == 0) { - ArangoDBSchema.ArangoDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode<>(column); - } else { - int maxNumber = globalState.getRandomly().getInteger(0, numberOfComputedVariables); - ArangoDBSchema.ArangoDBColumn column = new ArangoDBSchema.ArangoDBColumn("c" + maxNumber, - ArangoDBSchema.ArangoDBDataType.INTEGER, false, false); - column.setTable(dummy); - return new ColumnReferenceNode<>(column); - } - } - - @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, ArangoDBUnaryPrefixOperator.NOT); - } - - @Override - public Node isNull(Node expr) { - return new ArangoDBUnsupportedPredicate<>(); - } - - public enum ArangoDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { - EQUALS("=="), NOT_EQUALS("!="), LESS_THAN("<"), LESS_OR_EQUAL("<="), GREATER_THAN(">"), GREATER_OR_EQUAL(">="); - - private final String representation; - - ArangoDBBinaryComparisonOperator(String representation) { - this.representation = representation; - } - - @Override - public String getTextRepresentation() { - return representation; - } - - public static ArangoDBBinaryComparisonOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum ArangoDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { - NOT("!"); - - private final String representation; - - ArangoDBUnaryPrefixOperator(String representation) { - this.representation = representation; - } - - @Override - public String getTextRepresentation() { - return representation; - } - - public static ArangoDBUnaryPrefixOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum ArangoDBBinaryLogicalOperator implements BinaryOperatorNode.Operator { - AND, OR; - - @Override - public String getTextRepresentation() { - return toString(); - } - - public static BinaryOperatorNode.Operator getRandom() { - return Randomly.fromOptions(values()); - } - } - -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java deleted file mode 100644 index 3cfceeed4..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.arangodb.gen; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.query.ArangoDBConstantGenerator; -import sqlancer.arangodb.query.ArangoDBInsertQuery; - -public final class ArangoDBInsertGenerator { - - private final ArangoDBProvider.ArangoDBGlobalState globalState; - - private ArangoDBInsertGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { - this.globalState = globalState; - } - - public static ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { - return new ArangoDBInsertGenerator(globalState).generate(); - } - - private ArangoDBQueryAdapter generate() { - BaseDocument result = new BaseDocument(); - ArangoDBSchema.ArangoDBTable table = globalState.getSchema().getRandomTable(); - ArangoDBConstantGenerator constantGenerator = new ArangoDBConstantGenerator(globalState); - - for (int i = 0; i < table.getColumns().size(); i++) { - if (!globalState.getDbmsSpecificOptions().testRandomTypeInserts) { - constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), - table.getColumns().get(i).getType()); - } else { - constantGenerator.addRandomConstant(result, table.getColumns().get(i).getName()); - } - } - - return new ArangoDBInsertQuery(table, result); - } -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java deleted file mode 100644 index 1236c3ce4..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.arangodb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.query.ArangoDBCreateTableQuery; - -public class ArangoDBTableGenerator { - - private ArangoDBSchema.ArangoDBTable table; - private final List columnsToBeAdded = new ArrayList<>(); - - public ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { - String tableName = globalState.getSchema().getFreeTableName(); - ArangoDBCreateTableQuery createTableQuery = new ArangoDBCreateTableQuery(tableName); - table = new ArangoDBSchema.ArangoDBTable(tableName, columnsToBeAdded, false); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - String columnName = String.format("c%d", i); - createColumn(columnName); - } - globalState.addTable(table); - return createTableQuery; - } - - private ArangoDBSchema.ArangoDBDataType createColumn(String columnName) { - ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); - ArangoDBSchema.ArangoDBColumn newColumn = new ArangoDBSchema.ArangoDBColumn(columnName, dataType, false, false); - newColumn.setTable(table); - columnsToBeAdded.add(newColumn); - return dataType; - } - - public String getTableName() { - return table.getName(); - } - - public ArangoDBSchema.ArangoDBTable getGeneratedTable() { - return table; - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java b/src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java deleted file mode 100644 index 406e8adca..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.arangodb.query; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBConstant; - -public class ArangoDBConstantGenerator { - private final ArangoDBProvider.ArangoDBGlobalState globalState; - - public ArangoDBConstantGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { - this.globalState = globalState; - } - - public void addRandomConstant(BaseDocument document, String key) { - ArangoDBSchema.ArangoDBDataType type = ArangoDBSchema.ArangoDBDataType.getRandom(); - addRandomConstantWithType(document, key, type); - } - - public void addRandomConstantWithType(BaseDocument document, String key, ArangoDBSchema.ArangoDBDataType dataType) { - ArangoDBConstant constant; - switch (dataType) { - case STRING: - constant = new ArangoDBConstant.ArangoDBStringConstant(globalState.getRandomly().getString()); - constant.setValueInDocument(document, key); - return; - case DOUBLE: - constant = new ArangoDBConstant.ArangoDBDoubleConstant(globalState.getRandomly().getDouble()); - constant.setValueInDocument(document, key); - return; - case BOOLEAN: - constant = new ArangoDBConstant.ArangoDBBooleanConstant(Randomly.getBoolean()); - constant.setValueInDocument(document, key); - return; - case INTEGER: - constant = new ArangoDBConstant.ArangoDBIntegerConstant((int) globalState.getRandomly().getInteger()); - constant.setValueInDocument(document, key); - return; - default: - throw new AssertionError(dataType); - } - - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java b/src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java deleted file mode 100644 index 6c2cc1b75..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java +++ /dev/null @@ -1,54 +0,0 @@ -package sqlancer.arangodb.query; - -import java.util.Collections; - -import com.arangodb.ArangoCollection; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.arangodb.ArangoDBConnection; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.common.query.ExpectedErrors; - -public class ArangoDBCreateIndexQuery extends ArangoDBQueryAdapter { - - private final ArangoDBSchema.ArangoDBColumn column; - - public ArangoDBCreateIndexQuery(ArangoDBSchema.ArangoDBColumn column) { - this.column = column; - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - ArangoCollection collection = globalState.getConnection().getDatabase() - .collection(column.getTable().getName()); - collection.ensureHashIndex(Collections.singletonList(column.getName()), null); - Main.nrSuccessfulActions.addAndGet(1); - return true; - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - throw e; - } - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("db.").append(column.getTable().getName()) - .append(".ensureIndex({type: \"hash\", fields: [ \"").append(column.getName()).append("\" ]});"); - return stringBuilder.toString(); - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java b/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java deleted file mode 100644 index 00b3276d0..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.arangodb.query; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.arangodb.ArangoDBConnection; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.common.query.ExpectedErrors; - -public class ArangoDBCreateTableQuery extends ArangoDBQueryAdapter { - - private final String tableName; - - public ArangoDBCreateTableQuery(String tableName) { - this.tableName = tableName; - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - globalState.getConnection().getDatabase().createCollection(tableName); - Main.nrSuccessfulActions.addAndGet(1); - return true; - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - throw e; - } - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - return "db._create(\"" + tableName + "\")"; - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java b/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java deleted file mode 100644 index 9a3612062..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java +++ /dev/null @@ -1,66 +0,0 @@ -package sqlancer.arangodb.query; - -import java.util.Map; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.arangodb.ArangoDBConnection; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.common.query.ExpectedErrors; - -public class ArangoDBInsertQuery extends ArangoDBQueryAdapter { - - private final ArangoDBSchema.ArangoDBTable table; - private final BaseDocument documentToBeInserted; - - public ArangoDBInsertQuery(ArangoDBSchema.ArangoDBTable table, BaseDocument documentToBeInserted) { - this.table = table; - this.documentToBeInserted = documentToBeInserted; - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - globalState.getConnection().getDatabase().collection(table.getName()).insertDocument(documentToBeInserted); - Main.nrSuccessfulActions.addAndGet(1); - return true; - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - throw e; - } - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("db._query(\"INSERT { "); - String filler = ""; - for (Map.Entry stringObjectEntry : documentToBeInserted.getProperties().entrySet()) { - stringBuilder.append(filler); - filler = ", "; - stringBuilder.append(stringObjectEntry.getKey()).append(": "); - Object value = stringObjectEntry.getValue(); - if (value instanceof String) { - stringBuilder.append("'").append(value).append("'"); - } else { - stringBuilder.append(value); - } - } - stringBuilder.append("} IN ").append(table.getName()).append("\")"); - return stringBuilder.toString(); - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java b/src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java deleted file mode 100644 index 835849b92..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java +++ /dev/null @@ -1,57 +0,0 @@ -package sqlancer.arangodb.query; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; - -public class ArangoDBOptimizerRules { - - private final List allRules = new ArrayList<>(); - - public ArangoDBOptimizerRules() { - // SRC: - // https://www.arangodb.com/docs/stable/aql/execution-and-performance-optimizer.html#list-of-optimizer-rules - // Filtered out irrelevant ones - allRules.add("-fuse-filters"); - // allRules.add("-geo-index-optimizer"); - // allRules.add("-handle-arangosearch-views"); - // allRules.add("-inline-subqueries"); - allRules.add("-interchange-adjacent-enumerations"); - allRules.add("-late-document-materialization"); - // allRules.add("-late-document-materialization-arangosearch"); - allRules.add("-move-calculations-down"); - allRules.add("-move-calculations-up"); - allRules.add("-move-filters-into-enumerate"); - allRules.add("-move-filters-up"); - // allRules.add("-optimize-count"); - // allRules.add("-optimize-subqueries"); - // allRules.add("-optimize-traversals"); - // allRules.add("-patch-update-statements"); - allRules.add("-propagate-constant-attributes"); - allRules.add("-reduce-extraction-to-projection"); - // allRules.add("-remove-collect-variables"); - // allRules.add("-remove-data-modification-out-variables"); - allRules.add("-remove-filter-covered-by-index"); - // allRules.add("-remove-filter-covered-by-traversal"); - allRules.add("-remove-redundant-calculations"); - allRules.add("-remove-redundant-or"); - // allRules.add("-remove-redundant-path-var"); - // allRules.add("-remove-redundant-sorts"); - // allRules.add("-remove-sort-rand"); - allRules.add("-remove-unnecessary-calculations"); - allRules.add("-remove-unnecessary-filters"); - // allRules.add("-replace-function-with-index"); - allRules.add("-replace-or-with-in"); - allRules.add("-simplify-conditions"); - // allRules.add("-sort-in-values"); - // allRules.add("-sort-limit"); - // allRules.add("-splice-subqueries"); - // allRules.add("-use-index-for-sort"); - allRules.add("-use-indexes"); - } - - public List getRandomRules() { - return Randomly.subset(allRules); - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java b/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java deleted file mode 100644 index 4725e4178..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java +++ /dev/null @@ -1,88 +0,0 @@ -package sqlancer.arangodb.query; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import com.arangodb.ArangoCursor; -import com.arangodb.entity.BaseDocument; -import com.arangodb.model.AqlQueryOptions; - -import sqlancer.GlobalState; -import sqlancer.arangodb.ArangoDBConnection; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLancerResultSet; - -public class ArangoDBSelectQuery extends ArangoDBQueryAdapter { - - private final String query; - - private List optimizerRules; - - private List resultSet; - - public ArangoDBSelectQuery(String query) { - this.query = query; - optimizerRules = new ArrayList<>(); - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - throw new UnsupportedOperationException(); - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - if (optimizerRules.isEmpty()) { - return "db._query(\"" + query + "\")"; - } else { - String rules = optimizerRules.stream().map(Object::toString).collect(Collectors.joining("\",\"")); - return "db._query(\"" + query + "\", null, { optimizer: { rules: [\"" + rules + "\"] } } )"; - } - } - - @Override - public > SQLancerResultSet executeAndGet(G globalState, - String... fills) throws Exception { - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(this.getLogString()); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - ArangoCursor cursor; - if (optimizerRules.isEmpty()) { - cursor = globalState.getConnection().getDatabase().query(query, BaseDocument.class); - } else { - AqlQueryOptions options = new AqlQueryOptions(); - cursor = globalState.getConnection().getDatabase().query(query, options.rules(optimizerRules), - BaseDocument.class); - } - resultSet = cursor.asListRemaining(); - return null; - } - - public List getResultSet() { - return resultSet; - } - - public void excludeRandomOptRules() { - optimizerRules = new ArangoDBOptimizerRules().getRandomRules(); - } -} diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java deleted file mode 100644 index 859ffc74e..000000000 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java +++ /dev/null @@ -1,67 +0,0 @@ -package sqlancer.arangodb.test; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.arangodb.ast.ArangoDBSelect; -import sqlancer.arangodb.gen.ArangoDBComputedExpressionGenerator; -import sqlancer.arangodb.gen.ArangoDBFilterExpressionGenerator; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; - -public class ArangoDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, ArangoDBProvider.ArangoDBGlobalState> - implements TestOracle { - - protected ArangoDBSchema schema; - protected List targetColumns; - protected ArangoDBFilterExpressionGenerator expressionGenerator; - protected ArangoDBSelect select; - protected int numberComputedColumns; - - protected ArangoDBQueryPartitioningBase(ArangoDBProvider.ArangoDBGlobalState state) { - super(state); - } - - @Override - protected ExpressionGenerator> getGen() { - return expressionGenerator; - } - - @Override - public void check() throws Exception { - numberComputedColumns = state.getRandomly().getInteger(0, 4); - schema = state.getSchema(); - generateTargetColumns(); - expressionGenerator = new ArangoDBFilterExpressionGenerator(state).setColumns(targetColumns); - expressionGenerator.setNumberOfComputedVariables(numberComputedColumns); - initializeTernaryPredicateVariants(); - select = new ArangoDBSelect<>(); - select.setFromColumns(targetColumns); - select.setProjectionColumns(Randomly.nonEmptySubset(targetColumns)); - generateComputedClause(); - } - - private void generateComputedClause() { - List> computedColumns = new ArrayList<>(); - ArangoDBComputedExpressionGenerator generator = new ArangoDBComputedExpressionGenerator(state); - generator.setColumns(targetColumns); - for (int i = 0; i < numberComputedColumns; i++) { - computedColumns.add(generator.generateExpression()); - } - select.setComputedClause(computedColumns); - } - - private void generateTargetColumns() { - ArangoDBSchema.ArangoDBTables targetTables; - targetTables = schema.getRandomTableNonEmptyTables(); - List allColumns = targetTables.getColumns(); - targetColumns = Randomly.nonEmptySubset(allColumns); - } -} diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java deleted file mode 100644 index 6ad19fabf..000000000 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.arangodb.test; - -import static sqlancer.arangodb.ArangoDBComparatorHelper.assumeResultSetsAreEqual; -import static sqlancer.arangodb.ArangoDBComparatorHelper.getResultSetAsDocumentList; - -import java.util.List; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.query.ArangoDBSelectQuery; -import sqlancer.arangodb.visitor.ArangoDBVisitor; - -public class ArangoDBQueryPartitioningWhereTester extends ArangoDBQueryPartitioningBase { - public ArangoDBQueryPartitioningWhereTester(ArangoDBProvider.ArangoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - select.setFilterClause(null); - - ArangoDBSelectQuery query = ArangoDBVisitor.asSelectQuery(select); - List firstResultSet = getResultSetAsDocumentList(query, state); - - select.setFilterClause(predicate); - query = ArangoDBVisitor.asSelectQuery(select); - List secondResultSet = getResultSetAsDocumentList(query, state); - - select.setFilterClause(negatedPredicate); - query = ArangoDBVisitor.asSelectQuery(select); - List thirdResultSet = getResultSetAsDocumentList(query, state); - - thirdResultSet.addAll(secondResultSet); - assumeResultSetsAreEqual(firstResultSet, thirdResultSet, query); - - if (state.getDbmsSpecificOptions().withOptimizerRuleTests) { - select.setFilterClause(predicate); - query = ArangoDBVisitor.asSelectQuery(select); - query.excludeRandomOptRules(); - List forthResultSet = getResultSetAsDocumentList(query, state); - assumeResultSetsAreEqual(secondResultSet, forthResultSet, query); - } - } -} diff --git a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java deleted file mode 100644 index f82995d5e..000000000 --- a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java +++ /dev/null @@ -1,134 +0,0 @@ -package sqlancer.arangodb.visitor; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBConstant; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.arangodb.ast.ArangoDBSelect; -import sqlancer.arangodb.gen.ArangoDBComputedExpressionGenerator; -import sqlancer.arangodb.query.ArangoDBSelectQuery; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; - -public class ArangoDBToQueryVisitor extends ArangoDBVisitor { - - private final StringBuilder stringBuilder; - - public ArangoDBToQueryVisitor() { - stringBuilder = new StringBuilder(); - } - - @Override - protected void visit(ArangoDBSelect expression) { - generateFrom(expression); - generateComputed(expression); - generateFilter(expression); - generateProject(expression); - } - - private void generateFilter(ArangoDBSelect expression) { - if (expression.hasFilter()) { - stringBuilder.append("FILTER "); - visit(expression.getFilterClause()); - stringBuilder.append(" "); - } - } - - private void generateComputed(ArangoDBSelect expression) { - if (expression.hasComputed()) { - List> computedClause = expression.getComputedClause(); - int computedNumber = 0; - for (Node computedExpression : computedClause) { - stringBuilder.append("LET c").append(computedNumber).append(" = "); - visit(computedExpression); - stringBuilder.append(" "); - computedNumber++; - } - } - } - - @Override - protected void visit(ColumnReferenceNode expression) { - if (expression.getColumn().getTable().getName().equals("")) { - stringBuilder.append(expression.getColumn().getName()); - } else { - stringBuilder.append("r").append(expression.getColumn().getTable().getName()).append(".") - .append(expression.getColumn().getName()); - } - } - - @Override - protected void visit(ArangoDBConstant expression) { - stringBuilder.append(expression.getValue()); - } - - @Override - protected void visit(NewBinaryOperatorNode expression) { - stringBuilder.append("("); - visit(expression.getLeft()); - stringBuilder.append(" ").append(expression.getOperatorRepresentation()).append(" "); - visit(expression.getRight()); - stringBuilder.append(")"); - } - - @Override - protected void visit(NewUnaryPrefixOperatorNode expression) { - stringBuilder.append(expression.getOperatorRepresentation()).append("("); - visit(expression.getExpr()); - stringBuilder.append(")"); - } - - @Override - protected void visit(NewFunctionNode expression) { - if (!(expression.getFunc() instanceof ArangoDBComputedExpressionGenerator.ComputedFunction)) { - throw new UnsupportedOperationException(); - } - ArangoDBComputedExpressionGenerator.ComputedFunction function = (ArangoDBComputedExpressionGenerator.ComputedFunction) expression - .getFunc(); - // TODO: Support functions with a different number of arguments. - if (function.getNrArgs() != 2) { - throw new UnsupportedOperationException(); - } - stringBuilder.append("("); - visit(expression.getArgs().get(0)); - stringBuilder.append(" ").append(function.getOperatorName()).append(" "); - visit(expression.getArgs().get(1)); - stringBuilder.append(")"); - } - - private void generateFrom(ArangoDBSelect expression) { - List forColumns = expression.getFromColumns(); - Set tables = new HashSet<>(); - for (ArangoDBSchema.ArangoDBColumn column : forColumns) { - tables.add(column.getTable()); - } - - for (ArangoDBSchema.ArangoDBTable table : tables) { - stringBuilder.append("FOR r").append(table.getName()).append(" IN ").append(table.getName()).append(" "); - } - } - - private void generateProject(ArangoDBSelect expression) { - List projectColumns = expression.getProjectionColumns(); - stringBuilder.append("RETURN {"); - String filler = ""; - for (ArangoDBSchema.ArangoDBColumn column : projectColumns) { - stringBuilder.append(filler); - filler = ", "; - stringBuilder.append(column.getTable().getName()).append("_").append(column.getName()).append(": r") - .append(column.getTable().getName()).append(".").append(column.getName()); - } - stringBuilder.append("}"); - } - - public ArangoDBSelectQuery getQuery() { - return new ArangoDBSelectQuery(stringBuilder.toString()); - } - -} diff --git a/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java deleted file mode 100644 index f1db84cf5..000000000 --- a/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java +++ /dev/null @@ -1,51 +0,0 @@ -package sqlancer.arangodb.visitor; - -import sqlancer.arangodb.ast.ArangoDBConstant; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.arangodb.ast.ArangoDBSelect; -import sqlancer.arangodb.query.ArangoDBSelectQuery; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; - -public abstract class ArangoDBVisitor { - - protected abstract void visit(ArangoDBSelect expression); - - protected abstract void visit(ColumnReferenceNode expression); - - protected abstract void visit(ArangoDBConstant expression); - - protected abstract void visit(NewBinaryOperatorNode expression); - - protected abstract void visit(NewUnaryPrefixOperatorNode expression); - - protected abstract void visit(NewFunctionNode expression); - - @SuppressWarnings("unchecked") - public void visit(Node expressionNode) { - if (expressionNode instanceof ArangoDBSelect) { - visit((ArangoDBSelect) expressionNode); - } else if (expressionNode instanceof ColumnReferenceNode) { - visit((ColumnReferenceNode) expressionNode); - } else if (expressionNode instanceof ArangoDBConstant) { - visit((ArangoDBConstant) expressionNode); - } else if (expressionNode instanceof NewBinaryOperatorNode) { - visit((NewBinaryOperatorNode) expressionNode); - } else if (expressionNode instanceof NewUnaryPrefixOperatorNode) { - visit((NewUnaryPrefixOperatorNode) expressionNode); - } else if (expressionNode instanceof NewFunctionNode) { - visit((NewFunctionNode) expressionNode); - } else { - throw new AssertionError(expressionNode); - } - } - - public static ArangoDBSelectQuery asSelectQuery(Node expressionNode) { - ArangoDBToQueryVisitor visitor = new ArangoDBToQueryVisitor(); - visitor.visit(expressionNode); - return visitor.getQuery(); - } -} diff --git a/src/sqlancer/cosmos/CosmosProvider.java b/src/sqlancer/cosmos/CosmosProvider.java deleted file mode 100644 index 424625b4f..000000000 --- a/src/sqlancer/cosmos/CosmosProvider.java +++ /dev/null @@ -1,77 +0,0 @@ -package sqlancer.cosmos; - -import com.google.auto.service.AutoService; -import com.mongodb.ConnectionString; -import com.mongodb.MongoClientSettings; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoDatabase; - -import sqlancer.DatabaseProvider; -import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; -import sqlancer.Randomly; -import sqlancer.StatementExecutor; -import sqlancer.common.log.LoggableFactory; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBLoggableFactory; -import sqlancer.mongodb.MongoDBOptions; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.gen.MongoDBTableGenerator; - -@AutoService(DatabaseProvider.class) -public class CosmosProvider extends - ProviderAdapter { - - public CosmosProvider() { - super(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState.class, MongoDBOptions.class); - } - - @Override - public void generateDatabase(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { - boolean success; - do { - MongoDBQueryAdapter query = new MongoDBTableGenerator(globalState).getQuery(globalState); - success = globalState.executeStatement(query); - } while (!success); - } - StatementExecutor se = new StatementExecutor<>( - globalState, sqlancer.mongodb.MongoDBProvider.Action.values(), - sqlancer.mongodb.MongoDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public MongoDBConnection createDatabase(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState globalState) - throws Exception { - String connectionString = ""; - if (connectionString.equals("")) { - throw new AssertionError("Please set connection string for cosmos database, located in CosmosProvider"); - } - MongoClientSettings settings = MongoClientSettings.builder() - .applyConnectionString(new ConnectionString(connectionString)).build(); - MongoClient mongoClient = MongoClients.create(settings); - MongoDatabase database = mongoClient.getDatabase(globalState.getDatabaseName()); - database.drop(); - return new MongoDBConnection(mongoClient, database); - } - - @Override - public String getDBMSName() { - return "cosmos"; - } - - @Override - public LoggableFactory getLoggableFactory() { - return new MongoDBLoggableFactory(); - } - - @Override - protected void checkViewsAreValid(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState globalState) { - } -} diff --git a/src/sqlancer/mongodb/MongoDBComparatorHelper.java b/src/sqlancer/mongodb/MongoDBComparatorHelper.java deleted file mode 100644 index 49b692645..000000000 --- a/src/sqlancer/mongodb/MongoDBComparatorHelper.java +++ /dev/null @@ -1,97 +0,0 @@ -package sqlancer.mongodb; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.bson.Document; - -import sqlancer.IgnoreMeException; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.query.MongoDBSelectQuery; - -public final class MongoDBComparatorHelper { - - private MongoDBComparatorHelper() { - } - - public static List getResultSetAsDocumentList(MongoDBSelectQuery adapter, MongoDBGlobalState state) - throws Exception { - ExpectedErrors errors = adapter.getExpectedErrors(); - List result; - try { - adapter.executeAndGet(state); - Main.nrSuccessfulActions.addAndGet(1); - result = adapter.getResultSet(); - return result; - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw e; - } - Main.nrUnsuccessfulActions.addAndGet(1); - if (e.getMessage() == null) { - throw new AssertionError(adapter.getLogString(), e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(adapter.getLogString(), e); - } - } - - public static void assumeCountIsEqual(List resultSet, List secondResultSet, - MongoDBSelectQuery originalQuery) { - int originalSize = resultSet.size(); - if (secondResultSet.isEmpty()) { - if (originalSize == 0) { - return; - } else { - String assertMessage = String.format("The Count of the result set mismatches!\n %s", - originalQuery.getLogString()); - throw new AssertionError(assertMessage); - } - } - if (secondResultSet.size() != 1) { - throw new AssertionError( - String.format("Count query result bigger than one \n %s", originalQuery.getLogString())); - } - int withCount = (int) secondResultSet.get(0).get("count"); - if (originalSize != withCount) { - String assertMessage = String.format("The Count of the result set mismatches!\n %s", - originalQuery.getLogString()); - throw new AssertionError(assertMessage); - } - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - MongoDBSelectQuery originalQuery) { - if (resultSet.size() != secondResultSet.size()) { - String assertionMessage = String.format("The Size of the result sets mismatch (%d and %d)!\n%s", - resultSet.size(), secondResultSet.size(), originalQuery.getLogString()); - throw new AssertionError(assertionMessage); - } - - Set firstHashSet = new HashSet<>(resultSet); - Set secondHashSet = new HashSet<>(secondResultSet); - - if (!firstHashSet.equals(secondHashSet)) { - Set firstResultSetMisses = new HashSet<>(firstHashSet); - firstResultSetMisses.removeAll(secondHashSet); - Set secondResultSetMisses = new HashSet<>(secondHashSet); - secondResultSetMisses.removeAll(firstHashSet); - StringBuilder firstMisses = new StringBuilder(); - for (Document document : firstResultSetMisses) { - firstMisses.append(document.toJson()).append(" "); - } - StringBuilder secondMisses = new StringBuilder(); - for (Document document : secondResultSetMisses) { - secondMisses.append(document.toJson()).append(" "); - } - String assertMessage = String.format("The Content of the result sets mismatch!\n %s \n %s\n %s", - firstMisses.toString(), secondMisses.toString(), originalQuery.getLogString()); - throw new AssertionError(assertMessage); - } - } -} diff --git a/src/sqlancer/mongodb/MongoDBConnection.java b/src/sqlancer/mongodb/MongoDBConnection.java deleted file mode 100644 index 6971bd79c..000000000 --- a/src/sqlancer/mongodb/MongoDBConnection.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.mongodb; - -import org.bson.BsonDocument; -import org.bson.BsonString; - -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoDatabase; - -import sqlancer.SQLancerDBConnection; - -public class MongoDBConnection implements SQLancerDBConnection { - - private final MongoClient client; - private final MongoDatabase database; - - public MongoDBConnection(MongoClient client, MongoDatabase database) { - this.client = client; - this.database = database; - } - - @Override - public String getDatabaseVersion() throws Exception { - return client.getDatabase("dbname").runCommand(new BsonDocument("buildinfo", new BsonString(""))).get("version") - .toString(); - } - - @Override - public void close() throws Exception { - client.close(); - } - - public MongoDatabase getDatabase() { - return database; - } -} diff --git a/src/sqlancer/mongodb/MongoDBLoggableFactory.java b/src/sqlancer/mongodb/MongoDBLoggableFactory.java deleted file mode 100644 index b668301b3..000000000 --- a/src/sqlancer/mongodb/MongoDBLoggableFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -package sqlancer.mongodb; - -import java.util.Arrays; - -import sqlancer.common.log.Loggable; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.log.LoggedString; -import sqlancer.common.query.Query; - -public class MongoDBLoggableFactory extends LoggableFactory { - @Override - protected Loggable createLoggable(String input, String suffix) { - return new LoggedString(input + suffix); - } - - @Override - public Query getQueryForStateToReproduce(String queryString) { - throw new UnsupportedOperationException(); - } - - @Override - public Query commentOutQuery(Query query) { - throw new UnsupportedOperationException(); - } - - @Override - protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { - StringBuilder sb = new StringBuilder(); - sb.append("// Time: ").append(time).append("\n"); - sb.append("// Database: ").append(databaseName).append("\n"); - sb.append("// Database version: ").append(databaseVersion).append("\n"); - sb.append("// seed value: ").append(seedValue).append("\n"); - return new LoggedString(sb.toString()); - } - - @Override - public Loggable convertStacktraceToLoggable(Throwable throwable) { - return new LoggedString(Arrays.toString(throwable.getStackTrace()) + "\n" + throwable.getMessage()); - } -} diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java deleted file mode 100644 index 78bd73ffc..000000000 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ /dev/null @@ -1,74 +0,0 @@ -package sqlancer.mongodb; - -import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.DOCUMENT_REMOVAL; -import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.QUERY_PARTITIONING; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.test.MongoDBDocumentRemovalTester; -import sqlancer.mongodb.test.MongoDBQueryPartitioningWhereTester; - -@Parameters(commandDescription = "MongoDB (experimental)") -public class MongoDBOptions implements DBMSSpecificOptions { - - @Parameter(names = "--test-validation", description = "Enable/Disable validation of schema with Schema Validation", arity = 1) - public boolean testValidation = true; - - @Parameter(names = "--test-null-inserts", description = "Enables to test inserting with null values, validation has to be off", arity = 1) - public boolean testNullInserts; - - @Parameter(names = "--test-random-types", description = "Insert random types instead of schema types, validation has to be off", arity = 1) - public boolean testRandomTypes; - - @Parameter(names = "--max-number-indexes", description = "The maximum number of indexes used.", arity = 1) - public int maxNumberIndexes = 15; - - @Parameter(names = "--test-computed-values", description = "Enable adding computed values to query", arity = 1) - public boolean testComputedValues; - - @Parameter(names = "--test-with-regex", description = "Enable Regex Leaf Nodes", arity = 1) - public boolean testWithRegex; - - @Parameter(names = "--test-with-count", description = "Count the number of documents and check with count command", arity = 1) - public boolean testWithCount; - - @Parameter(names = "--null-safety", description = "", arity = 1) - public boolean nullSafety; - - @Parameter(names = "--oracle") - public List oracles = Arrays.asList(QUERY_PARTITIONING, DOCUMENT_REMOVAL); - - @Override - public List getTestOracleFactory() { - return oracles; - } - - public enum MongoDBOracleFactory implements OracleFactory { - QUERY_PARTITIONING { - @Override - public TestOracle create(MongoDBGlobalState globalState) throws Exception { - List> oracles = new ArrayList<>(); - oracles.add(new MongoDBQueryPartitioningWhereTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - DOCUMENT_REMOVAL { - @Override - public TestOracle create(MongoDBGlobalState globalState) throws Exception { - List> oracles = new ArrayList<>(); - oracles.add(new MongoDBDocumentRemovalTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - } - } -} diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java deleted file mode 100644 index 09662a863..000000000 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ /dev/null @@ -1,128 +0,0 @@ -package sqlancer.mongodb; - -import java.util.ArrayList; -import java.util.List; - -import com.google.auto.service.AutoService; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoDatabase; - -import sqlancer.AbstractAction; -import sqlancer.DatabaseProvider; -import sqlancer.ExecutionTimer; -import sqlancer.GlobalState; -import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; -import sqlancer.Randomly; -import sqlancer.StatementExecutor; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.query.Query; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.gen.MongoDBIndexGenerator; -import sqlancer.mongodb.gen.MongoDBInsertGenerator; -import sqlancer.mongodb.gen.MongoDBTableGenerator; - -@AutoService(DatabaseProvider.class) -public class MongoDBProvider - extends ProviderAdapter { - - public MongoDBProvider() { - super(MongoDBGlobalState.class, MongoDBOptions.class); - } - - public enum Action implements AbstractAction { - INSERT(MongoDBInsertGenerator::getQuery), CREATE_INDEX(MongoDBIndexGenerator::getQuery); - - private final MongoDBQueryProvider queryProvider; - - Action(MongoDBQueryProvider queryProvider) { - this.queryProvider = queryProvider; - } - - @Override - public Query getQuery(MongoDBGlobalState globalState) throws Exception { - return queryProvider.getQuery(globalState); - } - } - - public static int mapActions(MongoDBGlobalState globalState, Action a) { - Randomly r = globalState.getRandomly(); - switch (a) { - case INSERT: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case CREATE_INDEX: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumberIndexes); - default: - throw new AssertionError(a); - } - } - - public static class MongoDBGlobalState extends GlobalState { - - private final List schemaTables = new ArrayList<>(); - - public void addTable(MongoDBTable table) { - schemaTables.add(table); - } - - @Override - protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { - boolean logExecutionTime = getOptions().logExecutionTime(); - if (success && getOptions().printSucceedingStatements()) { - System.out.println(q.getLogString()); - } - if (logExecutionTime) { - getLogger().writeCurrent("// " + timer.end().asString()); - } - if (q.couldAffectSchema()) { - updateSchema(); - } - } - - @Override - protected MongoDBSchema readSchema() throws Exception { - return new MongoDBSchema(schemaTables); - } - } - - @Override - public void generateDatabase(MongoDBGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { - boolean success; - do { - MongoDBQueryAdapter query = new MongoDBTableGenerator(globalState).getQuery(globalState); - success = globalState.executeStatement(query); - } while (!success); - } - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - MongoDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public MongoDBConnection createDatabase(MongoDBGlobalState globalState) throws Exception { - MongoClient mongoClient = MongoClients.create(); - MongoDatabase database = mongoClient.getDatabase(globalState.getDatabaseName()); - database.drop(); - return new MongoDBConnection(mongoClient, database); - } - - @Override - public String getDBMSName() { - return "mongodb"; - } - - @Override - public LoggableFactory getLoggableFactory() { - return new MongoDBLoggableFactory(); - } - - @Override - protected void checkViewsAreValid(MongoDBGlobalState globalState) { - } -} diff --git a/src/sqlancer/mongodb/MongoDBQueryAdapter.java b/src/sqlancer/mongodb/MongoDBQueryAdapter.java deleted file mode 100644 index e2add3242..000000000 --- a/src/sqlancer/mongodb/MongoDBQueryAdapter.java +++ /dev/null @@ -1,15 +0,0 @@ -package sqlancer.mongodb; - -import sqlancer.common.query.Query; - -public abstract class MongoDBQueryAdapter extends Query { - @Override - public String getQueryString() { - throw new UnsupportedOperationException(); - } - - @Override - public String getUnterminatedQueryString() { - throw new UnsupportedOperationException(); - } -} diff --git a/src/sqlancer/mongodb/MongoDBQueryProvider.java b/src/sqlancer/mongodb/MongoDBQueryProvider.java deleted file mode 100644 index 970c90cea..000000000 --- a/src/sqlancer/mongodb/MongoDBQueryProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.mongodb; - -@FunctionalInterface -public interface MongoDBQueryProvider { - MongoDBQueryAdapter getQuery(S globalState) throws Exception; -} diff --git a/src/sqlancer/mongodb/MongoDBSchema.java b/src/sqlancer/mongodb/MongoDBSchema.java deleted file mode 100644 index 5ae3cdd24..000000000 --- a/src/sqlancer/mongodb/MongoDBSchema.java +++ /dev/null @@ -1,97 +0,0 @@ -package sqlancer.mongodb; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.bson.BsonType; - -import com.mongodb.client.MongoDatabase; - -import sqlancer.Randomly; -import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.common.schema.AbstractTables; -import sqlancer.common.schema.TableIndex; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; - -public class MongoDBSchema extends AbstractSchema { - - public enum MongoDBDataType { - INTEGER(BsonType.INT32), STRING(BsonType.STRING), BOOLEAN(BsonType.BOOLEAN), DOUBLE(BsonType.DOUBLE), - DATE_TIME(BsonType.DATE_TIME), TIMESTAMP(BsonType.TIMESTAMP); - - private final BsonType bsonType; - - MongoDBDataType(BsonType type) { - this.bsonType = type; - } - - public BsonType getBsonType() { - return bsonType; - } - - public static MongoDBDataType getRandom(MongoDBGlobalState state) { - Set valueSet = new HashSet<>(Arrays.asList(values())); - if (state.getDbmsSpecificOptions().nullSafety) { - valueSet.remove(STRING); - } - MongoDBDataType[] configuredValues = new MongoDBDataType[valueSet.size()]; - return Randomly.fromOptions(valueSet.toArray(configuredValues)); - } - } - - public static class MongoDBColumn extends AbstractTableColumn { - - private final boolean isId; - private final boolean isNullable; - - public MongoDBColumn(String name, MongoDBDataType type, boolean isId, boolean isNullable) { - super(name, null, type); - this.isId = isId; - this.isNullable = isNullable; - } - - public boolean isId() { - return isId; - } - - public boolean isNullable() { - return isNullable; - } - - } - - public static class MongoDBTables extends AbstractTables { - - public MongoDBTables(List tables) { - super(tables); - } - } - - public MongoDBSchema(List databaseTables) { - super(databaseTables); - } - - public static class MongoDBTable extends AbstractTable { - public MongoDBTable(String name, List columns, boolean isView) { - super(name, columns, Collections.emptyList(), isView); - } - - @Override - public long getNrRows(MongoDBGlobalState globalState) { - throw new UnsupportedOperationException(); - } - } - - public static MongoDBSchema fromConnection(MongoDatabase connection, String databaseName) { - throw new UnsupportedOperationException(); - } - - public MongoDBTables getRandomTableNonEmptyTables() { - return new MongoDBTables(Randomly.nonEmptySubset(getDatabaseTables())); - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java b/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java deleted file mode 100644 index 21675250a..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.mongodb.ast; - -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator; - -public class MongoDBBinaryComparisonNode extends NewBinaryOperatorNode { - public MongoDBBinaryComparisonNode(Node left, Node right, - MongoDBBinaryComparisonOperator op) { - super(left, right, op); - } - - public MongoDBBinaryComparisonOperator operator() { - return (MongoDBBinaryComparisonOperator) op; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java b/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java deleted file mode 100644 index efb8d8294..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.mongodb.ast; - -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator; - -public class MongoDBBinaryLogicalNode extends NewBinaryOperatorNode { - public MongoDBBinaryLogicalNode(Node left, Node right, - MongoDBBinaryLogicalOperator op) { - super(left, right, op); - } - - public MongoDBBinaryLogicalOperator operator() { - return (MongoDBBinaryLogicalOperator) op; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBConstant.java b/src/sqlancer/mongodb/ast/MongoDBConstant.java deleted file mode 100644 index 86f783b48..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBConstant.java +++ /dev/null @@ -1,252 +0,0 @@ -package sqlancer.mongodb.ast; - -import java.io.Serializable; - -import org.bson.BsonDateTime; -import org.bson.BsonTimestamp; -import org.bson.Document; - -import sqlancer.common.ast.newast.Node; - -public abstract class MongoDBConstant implements Node { - private MongoDBConstant() { - } - - public abstract void setValueInDocument(Document document, String key); - - public abstract String getLogValue(); - - public abstract Object getValue(); - - public abstract Serializable getSerializedValue(); - - public static class MongoDBNullConstant extends MongoDBConstant { - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, null); - } - - @Override - public String getLogValue() { - return "null"; - } - - @Override - public Object getValue() { - return null; - } - - @Override - public Serializable getSerializedValue() { - return null; - } - } - - public static Node createNullConstant() { - return new MongoDBNullConstant(); - } - - public static class MongoDBIntegerConstant extends MongoDBConstant { - - private final int value; - - public MongoDBIntegerConstant(int value) { - this.value = value; - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return "NumberInt(" + value + ")"; - } - - @Override - public Integer getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value; - } - } - - public static Node createIntegerConstant(int value) { - return new MongoDBIntegerConstant(value); - } - - public static class MongoDBStringConstant extends MongoDBConstant { - - private final String value; - - public MongoDBStringConstant(String value) { - this.value = value; - } - - public String getStringValue() { - return value; - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n") + "\""; - } - - @Override - public String getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value; - } - } - - public static Node createStringConstant(String value) { - return new MongoDBStringConstant(value); - } - - public static class MongoDBBooleanConstant extends MongoDBConstant { - - private final boolean value; - - public MongoDBBooleanConstant(boolean value) { - this.value = value; - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return String.valueOf(value); - } - - @Override - public Boolean getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value; - } - } - - public static Node createBooleanConstant(boolean value) { - return new MongoDBBooleanConstant(value); - } - - public static class MongoDBDoubleConstant extends MongoDBConstant { - - private final double value; - - public MongoDBDoubleConstant(double value) { - this.value = value; - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return String.valueOf(value); - } - - @Override - public Double getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value; - } - } - - public static Node createDoubleConstant(double value) { - return new MongoDBDoubleConstant(value); - } - - public static class MongoDBDateTimeConstant extends MongoDBConstant { - - private final BsonDateTime value; - - public MongoDBDateTimeConstant(long val) { - this.value = new BsonDateTime(val); - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return "new Date(" + value.getValue() + ")"; - } - - @Override - public BsonDateTime getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value.getValue(); - } - } - - public static Node createDateTimeConstant(long value) { - return new MongoDBDateTimeConstant(value); - } - - public static class MongoDBTimestampConstant extends MongoDBConstant { - - private final BsonTimestamp value; - - public MongoDBTimestampConstant(long value) { - this.value = new BsonTimestamp(value); - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return "Timestamp(" + value.getValue() + ",1)"; - } - - @Override - public BsonTimestamp getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value.getValue(); - } - } - - public static Node createTimestampConstant(long value) { - return new MongoDBTimestampConstant(value); - } - -} diff --git a/src/sqlancer/mongodb/ast/MongoDBExpression.java b/src/sqlancer/mongodb/ast/MongoDBExpression.java deleted file mode 100644 index 1235a1fbc..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBExpression.java +++ /dev/null @@ -1,4 +0,0 @@ -package sqlancer.mongodb.ast; - -public interface MongoDBExpression { -} diff --git a/src/sqlancer/mongodb/ast/MongoDBRegexNode.java b/src/sqlancer/mongodb/ast/MongoDBRegexNode.java deleted file mode 100644 index 76c608586..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBRegexNode.java +++ /dev/null @@ -1,24 +0,0 @@ -package sqlancer.mongodb.ast; - -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBRegexOperator.REGEX; - -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBRegexOperator; - -public class MongoDBRegexNode extends NewBinaryOperatorNode { - private final String options; - - public MongoDBRegexNode(Node left, Node right, String options) { - super(left, right, REGEX); - this.options = options; - } - - public String getOptions() { - return options; - } - - public MongoDBRegexOperator operator() { - return (MongoDBRegexOperator) op; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBSelect.java b/src/sqlancer/mongodb/ast/MongoDBSelect.java deleted file mode 100644 index 0fe91ba4a..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBSelect.java +++ /dev/null @@ -1,104 +0,0 @@ -package sqlancer.mongodb.ast; - -import java.util.List; - -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.test.MongoDBColumnTestReference; - -public class MongoDBSelect implements Node { - - private final String mainTableName; - private final MongoDBColumnTestReference joinColumn; - List projectionColumns; - List lookupList; - boolean hasFilter; - Node filterClause; - boolean hasComputed; - List> computedClauses; - private boolean withCountClause; - - public MongoDBSelect(String mainTableName, MongoDBColumnTestReference joinColumn) { - this.mainTableName = mainTableName; - this.joinColumn = joinColumn; - } - - public String getMainTableName() { - return mainTableName; - } - - public MongoDBColumnTestReference getJoinColumn() { - return joinColumn; - } - - public void setProjectionList(List fetchColumns) { - if (fetchColumns == null || fetchColumns.isEmpty()) { - throw new IllegalArgumentException(); - } - this.projectionColumns = fetchColumns; - } - - public List getProjectionList() { - if (projectionColumns == null) { - throw new IllegalStateException(); - } - return projectionColumns; - } - - public void setLookupList(List lookupList) { - if (lookupList == null || lookupList.isEmpty()) { - throw new IllegalArgumentException(); - } - this.lookupList = lookupList; - } - - public List getLookupList() { - if (lookupList == null) { - throw new IllegalStateException(); - } - return lookupList; - } - - public void setFilterClause(Node filterClause) { - if (filterClause == null) { - hasFilter = false; - this.filterClause = null; - return; - } - hasFilter = true; - this.filterClause = filterClause; - } - - public Node getFilterClause() { - return filterClause; - } - - public boolean hasFilter() { - return hasFilter; - } - - public void setComputedClause(List> computedClause) { - if (computedClause == null) { - hasComputed = false; - this.computedClauses = null; - return; - } - hasComputed = true; - this.computedClauses = computedClause; - } - - public List> getComputedClause() { - return computedClauses; - } - - public boolean hasComputed() { - return hasComputed; - } - - public boolean getWithCountClause() { - return withCountClause; - } - - public void setWithCountClause(boolean withCountClause) { - this.withCountClause = withCountClause; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java b/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java deleted file mode 100644 index a34fe27e5..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.mongodb.ast; - -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBUnaryLogicalOperator; - -public class MongoDBUnaryLogicalOperatorNode extends NewUnaryPrefixOperatorNode { - - public MongoDBUnaryLogicalOperatorNode(Node expr, MongoDBUnaryLogicalOperator op) { - super(expr, op); - } - - public MongoDBUnaryLogicalOperator operator() { - return (MongoDBUnaryLogicalOperator) op; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java b/src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java deleted file mode 100644 index eae143e7d..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java +++ /dev/null @@ -1,7 +0,0 @@ -package sqlancer.mongodb.ast; - -import sqlancer.common.ast.newast.Node; - -public class MongoDBUnsupportedPredicate implements Node { - -} diff --git a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java deleted file mode 100644 index 347d5bc97..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java +++ /dev/null @@ -1,89 +0,0 @@ -package sqlancer.mongodb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBSchema; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.test.MongoDBColumnTestReference; - -public class MongoDBComputedExpressionGenerator - extends UntypedExpressionGenerator, MongoDBColumnTestReference> { - - private final MongoDBGlobalState globalState; - - @Override - public Node generateLeafNode() { - ComputedFunction function = ComputedFunction.getRandom(); - List> expressions = new ArrayList<>(); - for (int i = 0; i < function.getNrArgs(); i++) { - expressions.add(super.generateLeafNode()); - } - return new NewFunctionNode<>(expressions, function); - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - ComputedFunction func = ComputedFunction.getRandom(); - return new NewFunctionNode<>(generateExpressions(func.getNrArgs(), depth + 1), func); - } - - public MongoDBComputedExpressionGenerator(MongoDBGlobalState globalState) { - this.globalState = globalState; - } - - public enum ComputedFunction { - ADD(2, "$add"), MULTIPLY(2, "$multiply"), DIVIDE(2, "$divide"), POW(2, "$pow"), SQRT(1, "$sqrt"), - LOG(2, "$log"), AVG(2, "$avg"), EXP(1, "$exp"); - - private final int nrArgs; - private final String operatorName; - - ComputedFunction(int nrArgs, String operatorName) { - this.nrArgs = nrArgs; - this.operatorName = operatorName; - } - - public static ComputedFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - - public String getOperator() { - return operatorName; - } - } - - @Override - public Node generateConstant() { - MongoDBSchema.MongoDBDataType type = MongoDBSchema.MongoDBDataType.getRandom(globalState); - MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); - return generator.generateConstantWithType(type); - } - - @Override - protected Node generateColumn() { - return Randomly.fromList(columns); - } - - @Override - public Node negatePredicate(Node predicate) { - throw new UnsupportedOperationException(); - } - - @Override - public Node isNull(Node expr) { - throw new UnsupportedOperationException(); - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java deleted file mode 100644 index 2e6b15048..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java +++ /dev/null @@ -1,86 +0,0 @@ -package sqlancer.mongodb.gen; - -import org.bson.Document; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBBooleanConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBDateTimeConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBDoubleConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBIntegerConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBNullConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBTimestampConstant; -import sqlancer.mongodb.ast.MongoDBExpression; - -public class MongoDBConstantGenerator { - private final MongoDBGlobalState globalState; - - public MongoDBConstantGenerator(MongoDBGlobalState globalState) { - this.globalState = globalState; - } - - public Node generateConstantWithType(MongoDBDataType option) { - switch (option) { - case DATE_TIME: - return MongoDBConstant.createDateTimeConstant(globalState.getRandomly().getInteger()); - case BOOLEAN: - return MongoDBConstant.createBooleanConstant(Randomly.getBoolean()); - case DOUBLE: - return MongoDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - case STRING: - return MongoDBConstant.createStringConstant(globalState.getRandomly().getString()); - case INTEGER: - return MongoDBConstant.createIntegerConstant((int) globalState.getRandomly().getInteger()); - case TIMESTAMP: - return MongoDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); - default: - throw new AssertionError(option); - } - } - - public void addRandomConstant(Document document, String key) { - MongoDBDataType type = MongoDBDataType.getRandom(globalState); - addRandomConstantWithType(document, key, type); - } - - public void addRandomConstantWithType(Document document, String key, MongoDBDataType option) { - MongoDBConstant constant; - if (globalState.getDbmsSpecificOptions().testNullInserts && Randomly.getBooleanWithSmallProbability()) { - constant = new MongoDBNullConstant(); - constant.setValueInDocument(document, key); - return; - } - switch (option) { - case DATE_TIME: - constant = new MongoDBDateTimeConstant(globalState.getRandomly().getInteger()); - constant.setValueInDocument(document, key); - return; - - case BOOLEAN: - constant = new MongoDBBooleanConstant(Randomly.getBoolean()); - constant.setValueInDocument(document, key); - return; - case DOUBLE: - constant = new MongoDBDoubleConstant(globalState.getRandomly().getDouble()); - constant.setValueInDocument(document, key); - return; - case STRING: - constant = new MongoDBConstant.MongoDBStringConstant(globalState.getRandomly().getString()); - constant.setValueInDocument(document, key); - return; - case INTEGER: - constant = new MongoDBIntegerConstant((int) globalState.getRandomly().getInteger()); - constant.setValueInDocument(document, key); - return; - case TIMESTAMP: - constant = new MongoDBTimestampConstant(globalState.getRandomly().getInteger()); - constant.setValueInDocument(document, key); - return; - default: - throw new AssertionError(option); - } - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java b/src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java deleted file mode 100644 index 8687fd45c..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java +++ /dev/null @@ -1,25 +0,0 @@ -package sqlancer.mongodb.gen; - -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.query.MongoDBCreateIndexQuery; - -public final class MongoDBIndexGenerator { - private MongoDBIndexGenerator() { - } - - public static MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { - MongoDBTable randomTable = globalState.getSchema().getRandomTable(); - List columns = Randomly.nonEmptySubset(randomTable.getColumns()); - MongoDBCreateIndexQuery createIndexQuery = new MongoDBCreateIndexQuery(randomTable); - for (MongoDBColumn column : columns) { - createIndexQuery.addIndex(column.getName(), Randomly.getBoolean()); - } - return createIndexQuery; - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java b/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java deleted file mode 100644 index f8b8b3ffd..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java +++ /dev/null @@ -1,38 +0,0 @@ -package sqlancer.mongodb.gen; - -import org.bson.Document; - -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.query.MongoDBInsertQuery; - -public final class MongoDBInsertGenerator { - - private final MongoDBGlobalState globalState; - - private MongoDBInsertGenerator(MongoDBGlobalState globalState) { - this.globalState = globalState; - } - - public static MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { - return new MongoDBInsertGenerator(globalState).generate(); - } - - public MongoDBQueryAdapter generate() { - Document result = new Document(); - MongoDBTable table = globalState.getSchema().getRandomTable(); - MongoDBConstantGenerator constantGenerator = new MongoDBConstantGenerator(globalState); - - for (int i = 0; i < table.getColumns().size(); i++) { - if (!globalState.getDbmsSpecificOptions().testRandomTypes) { - constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), - table.getColumns().get(i).getType()); - } else { - constantGenerator.addRandomConstant(result, table.getColumns().get(i).getName()); - } - } - - return new MongoDBInsertQuery(table, result); - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java deleted file mode 100644 index 3e6c833d3..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ /dev/null @@ -1,291 +0,0 @@ -package sqlancer.mongodb.gen; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.bson.conversions.Bson; - -import com.mongodb.client.model.Filters; - -import sqlancer.Randomly; -import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; -import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; -import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBRegexNode; -import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; -import sqlancer.mongodb.ast.MongoDBUnsupportedPredicate; -import sqlancer.mongodb.test.MongoDBColumnTestReference; -import sqlancer.mongodb.visitor.MongoDBNegateVisitor; - -public class MongoDBMatchExpressionGenerator - extends UntypedExpressionGenerator, MongoDBColumnTestReference> { - - private final MongoDBGlobalState globalState; - - private enum LeafExpression { - BINARY_COMPARISON, REGEX - } - - private enum NonLeafExpression { - BINARY_LOGICAL, UNARY_LOGICAL - } - - public MongoDBMatchExpressionGenerator(MongoDBGlobalState globalState) { - this.globalState = globalState; - } - - @Override - public Node generateLeafNode() { - List possibleOptions = new ArrayList<>(Arrays.asList(LeafExpression.values())); - if (!globalState.getDbmsSpecificOptions().testWithRegex) { - possibleOptions.remove(LeafExpression.REGEX); - } - LeafExpression expr = Randomly.fromList(possibleOptions); - switch (expr) { - case BINARY_COMPARISON: - MongoDBBinaryComparisonOperator operator = MongoDBBinaryComparisonOperator.getRandom(); - MongoDBColumnTestReference reference = (MongoDBColumnTestReference) generateColumn(); - - return new MongoDBBinaryComparisonNode(reference, - generateConstant(reference.getColumnReference().getType()), operator); - case REGEX: - return new MongoDBRegexNode(generateColumn(), - new MongoDBConstantGenerator(globalState).generateConstantWithType(MongoDBDataType.STRING), - getRandomizedRegexOptions()); - default: - throw new AssertionError(); - } - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - - List possibleOptions = new ArrayList<>(Arrays.asList(NonLeafExpression.values())); - NonLeafExpression expr = Randomly.fromList(possibleOptions); - switch (expr) { - case BINARY_LOGICAL: - MongoDBBinaryLogicalOperator binaryOperator = MongoDBBinaryLogicalOperator.getRandom(); - return new MongoDBBinaryLogicalNode(generateExpression(depth + 1), generateExpression(depth + 1), - binaryOperator); - case UNARY_LOGICAL: - MongoDBUnaryLogicalOperator unaryOperator = MongoDBUnaryLogicalOperator.getRandom(); - return new MongoDBUnaryLogicalOperatorNode(generateExpression(depth + 1), unaryOperator); - default: - throw new AssertionError(); - } - } - - @Override - public Node generateConstant() { - MongoDBDataType type = MongoDBDataType.getRandom(globalState); - MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); - if (Randomly.getBooleanWithSmallProbability()) { - return MongoDBConstant.createNullConstant(); - } - return generator.generateConstantWithType(type); - } - - public Node generateConstant(MongoDBDataType type) { - MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); - if (Randomly.getBooleanWithSmallProbability() && !globalState.getDbmsSpecificOptions().nullSafety) { - return MongoDBConstant.createNullConstant(); - } - return generator.generateConstantWithType(type); - } - - private String getRandomizedRegexOptions() { - List s = Randomly.subset("i", "m", "x", "s"); - return String.join("", s); - } - - @Override - protected Node generateColumn() { - return Randomly.fromList(columns); - } - - @Override - public Node generatePredicate() { - Node result = super.generatePredicate(); - return MongoDBNegateVisitor.cleanNegations(result); - } - - @Override - public Node negatePredicate(Node predicate) { - Node result = new MongoDBUnaryLogicalOperatorNode(predicate, - MongoDBUnaryLogicalOperator.NOT); - return MongoDBNegateVisitor.cleanNegations(result); - } - - @Override - public Node isNull(Node expr) { - return new MongoDBUnsupportedPredicate<>(); - } - - public enum MongoDBUnaryLogicalOperator implements Operator { - NOT { - @Override - public Bson applyOperator(Bson inner) { - return Filters.not(inner); - } - - @Override - public String getTextRepresentation() { - return "$not"; - } - }; - - public abstract Bson applyOperator(Bson inner); - - public static MongoDBUnaryLogicalOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum MongoDBBinaryLogicalOperator implements Operator { - AND { - @Override - public Bson applyOperator(Bson left, Bson right) { - return Filters.and(left, right); - } - - @Override - public String getTextRepresentation() { - return "$and"; - } - }, - OR { - @Override - public Bson applyOperator(Bson left, Bson right) { - return Filters.or(left, right); - } - - @Override - public String getTextRepresentation() { - return "$or"; - } - }, - NOR { - @Override - public Bson applyOperator(Bson left, Bson right) { - return Filters.nor(left, right); - } - - @Override - public String getTextRepresentation() { - return "$nor"; - } - }; - - public abstract Bson applyOperator(Bson left, Bson right); - - public static MongoDBBinaryLogicalOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum MongoDBBinaryComparisonOperator implements Operator { - EQUALS { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.eq(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$eq"; - } - }, - NOT_EQUALS { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.ne(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$ne"; - } - }, - GREATER { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.gt(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$gt"; - } - - }, - LESS { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.lt(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$lt"; - } - - }, - GREATER_EQUAL { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.gte(columnName, constant.getValue()); - - } - - @Override - public String getTextRepresentation() { - return "$gte"; - } - - }, - LESS_EQUAL { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.lte(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$lte"; - } - }; - - public abstract Bson applyOperator(String columnName, MongoDBConstant constant); - - public static MongoDBBinaryComparisonOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum MongoDBRegexOperator implements Operator { - REGEX { - @Override - public Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex, String options) { - return Filters.regex(columnName, regex.getStringValue(), options); - } - - @Override - public String getTextRepresentation() { - return "$regex"; - } - }; - - public abstract Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex, - String options); - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java deleted file mode 100644 index 0153a6334..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java +++ /dev/null @@ -1,54 +0,0 @@ -package sqlancer.mongodb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; -import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.query.MongoDBCreateTableQuery; - -public class MongoDBTableGenerator { - - private MongoDBTable table; - private final List columnsToBeAdded = new ArrayList<>(); - private final MongoDBGlobalState state; - - public MongoDBTableGenerator(MongoDBGlobalState state) { - this.state = state; - } - - public MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { - String tableName = globalState.getSchema().getFreeTableName(); - MongoDBCreateTableQuery createTableQuery = new MongoDBCreateTableQuery(tableName); - table = new MongoDBTable(tableName, columnsToBeAdded, false); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - String columnName = String.format("c%d", i); - MongoDBDataType type = createColumn(columnName); - if (globalState.getDbmsSpecificOptions().testValidation) { - createTableQuery.addValidation(columnName, type.getBsonType()); - } - } - globalState.addTable(table); - return createTableQuery; - } - - private MongoDBDataType createColumn(String columnName) { - MongoDBDataType columnType = MongoDBDataType.getRandom(state); - MongoDBColumn newColumn = new MongoDBColumn(columnName, columnType, false, false); - newColumn.setTable(table); - columnsToBeAdded.add(newColumn); - return columnType; - } - - public String getTableName() { - return table.getName(); - } - - public MongoDBTable getGeneratedTable() { - return table; - } -} diff --git a/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java b/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java deleted file mode 100644 index c873b5924..000000000 --- a/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java +++ /dev/null @@ -1,77 +0,0 @@ -package sqlancer.mongodb.query; - -import java.util.ArrayList; -import java.util.List; - -import org.bson.conversions.Bson; - -import com.mongodb.client.model.Indexes; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; - -public class MongoDBCreateIndexQuery extends MongoDBQueryAdapter { - - private final MongoDBTable table; - private final List indeces; - private final List logIndeces; - - public MongoDBCreateIndexQuery(MongoDBTable table) { - this.table = table; - this.indeces = new ArrayList<>(); - this.logIndeces = new ArrayList<>(); - } - - public void addIndex(String column, boolean ascending) { - if (ascending) { - indeces.add(Indexes.ascending(column)); - logIndeces.add(column + ": 1"); - } else { - indeces.add(Indexes.descending(column)); - logIndeces.add(column + ": -1"); - } - } - - @Override - public String getLogString() { - StringBuilder sb = new StringBuilder(); - sb.append("db.").append(table.getName()).append(".createIndex({"); - String helper = ""; - for (String index : logIndeces) { - sb.append(helper); - helper = ","; - sb.append(index); - } - sb.append("})\n"); - return sb.toString(); - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - Main.nrSuccessfulActions.addAndGet(1); - Bson index; - if (indeces.size() > 1) { - index = Indexes.compoundIndex(indeces); - } else { - index = indeces.get(0); - } - globalState.getConnection().getDatabase().getCollection(table.getName()).createIndex(index); - return true; - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - -} diff --git a/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java b/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java deleted file mode 100644 index 7bc174c77..000000000 --- a/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java +++ /dev/null @@ -1,115 +0,0 @@ -package sqlancer.mongodb.query; - -import java.util.ArrayList; -import java.util.List; - -import org.bson.BsonType; -import org.bson.conversions.Bson; - -import com.mongodb.client.model.CreateCollectionOptions; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.ValidationOptions; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; - -public class MongoDBCreateTableQuery extends MongoDBQueryAdapter { - - private final String tableName; - private Bson validationFilter; - private final List logRequiredList; - private final List logPropertiesList; - - public MongoDBCreateTableQuery(String tableName) { - this.tableName = tableName; - this.validationFilter = null; - logRequiredList = new ArrayList<>(); - logPropertiesList = new ArrayList<>(); - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - ValidationOptions collOptions = new ValidationOptions().validator(this.validationFilter); - Main.nrSuccessfulActions.addAndGet(1); - globalState.getConnection().getDatabase().createCollection(tableName, - new CreateCollectionOptions().validationOptions(collOptions)); - return true; - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - String helper = ""; - StringBuilder sb = new StringBuilder(); - sb.append("db.createCollection(\"").append(tableName).append("\", {\n"); - - if (!logPropertiesList.isEmpty()) { - sb.append("validator: {"); - sb.append("$jsonSchema: {"); - sb.append("bsonType:\"object\","); - sb.append("required: [\n"); - for (String req : logRequiredList) { - sb.append(helper); - helper = ","; - sb.append(req); - } - sb.append("],"); - sb.append("properties: {\n"); - for (String prop : logPropertiesList) { - sb.append(prop); - } - sb.append("}}}})"); - } else { - sb.append("})"); - } - - return sb.toString(); - } - - public void addValidation(String columnName, BsonType type) { - Bson nameFilter = Filters.exists(columnName); - Bson typeFilter = Filters.type(columnName, type); - - if (validationFilter == null) { - validationFilter = Filters.and(nameFilter, typeFilter); - } else { - validationFilter = Filters.and(validationFilter, Filters.and(nameFilter, typeFilter)); - } - - logRequiredList.add("\"" + columnName + "\""); - logPropertiesList.add(columnName + ": { bsonType:\"" + bsonTypeToString(type) + "\"},\n"); - } - - public String bsonTypeToString(BsonType type) { - switch (type) { - case DOUBLE: - return "double"; - case STRING: - return "string"; - case BOOLEAN: - return "bool"; - case INT32: - case INT64: - return "int"; - case DATE_TIME: - return "date"; - case TIMESTAMP: - return "timestamp"; - default: - throw new IllegalStateException(); - } - } -} diff --git a/src/sqlancer/mongodb/query/MongoDBInsertQuery.java b/src/sqlancer/mongodb/query/MongoDBInsertQuery.java deleted file mode 100644 index 127dc82d0..000000000 --- a/src/sqlancer/mongodb/query/MongoDBInsertQuery.java +++ /dev/null @@ -1,87 +0,0 @@ -package sqlancer.mongodb.query; - -import org.bson.BsonDateTime; -import org.bson.BsonTimestamp; -import org.bson.Document; -import org.bson.types.ObjectId; - -import com.mongodb.client.result.InsertOneResult; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; - -public class MongoDBInsertQuery extends MongoDBQueryAdapter { - boolean excluded; - private final MongoDBTable table; - private final Document documentToBeInserted; - - public MongoDBInsertQuery(MongoDBTable table, Document documentToBeInserted) { - this.table = table; - this.documentToBeInserted = documentToBeInserted; - this.excluded = false; - } - - @Override - public String getLogString() { - StringBuilder sb = new StringBuilder(); - sb.append("db." + table.getName() + ".insert({"); - String helper = ""; - for (String key : documentToBeInserted.keySet()) { - sb.append(helper); - helper = ", "; - if (documentToBeInserted.get(key) instanceof ObjectId) { - continue; - } - Object value = documentToBeInserted.get(key); - sb.append(key); - sb.append(": "); - sb.append(getStringRepresentation(value)); - } - sb.append("})\n"); - - return sb.toString(); - } - - private String getStringRepresentation(Object value) { - if (value instanceof Double) { - return String.valueOf(value); - } else if (value instanceof Integer) { - return "NumberInt(" + value + ")"; - } else if (value instanceof String) { - return "\"" + value + "\""; - } else if (value instanceof BsonDateTime) { - return "new Date(" + ((BsonDateTime) value).getValue() + ")"; - } else if (value instanceof BsonTimestamp) { - return "Timestamp(" + ((BsonTimestamp) value).getValue() + ",1)"; - } else if (value instanceof Boolean) { - return String.valueOf(value); - } else if (value == null) { - return "null"; - } else { - throw new IllegalStateException(); - } - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - Main.nrSuccessfulActions.addAndGet(1); - InsertOneResult result = globalState.getConnection().getDatabase().getCollection(table.getName()) - .insertOne(documentToBeInserted); - return result.wasAcknowledged(); - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } -} diff --git a/src/sqlancer/mongodb/query/MongoDBRemoveQuery.java b/src/sqlancer/mongodb/query/MongoDBRemoveQuery.java deleted file mode 100644 index 6fe1c9e3f..000000000 --- a/src/sqlancer/mongodb/query/MongoDBRemoveQuery.java +++ /dev/null @@ -1,59 +0,0 @@ -package sqlancer.mongodb.query; - -import org.bson.Document; -import org.bson.types.ObjectId; - -import com.mongodb.client.result.DeleteResult; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema; - -public class MongoDBRemoveQuery extends MongoDBQueryAdapter { - - private final String objectId; - private final MongoDBSchema.MongoDBTable table; - - public MongoDBRemoveQuery(MongoDBSchema.MongoDBTable table, String objectId) { - this.objectId = objectId; - this.table = table; - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - DeleteResult result = globalState.getConnection().getDatabase().getCollection(table.getName()) - .deleteOne(new Document("_id", new ObjectId(objectId))); - if (result.wasAcknowledged()) { - Main.nrSuccessfulActions.addAndGet(1); - } else { - Main.nrUnsuccessfulActions.addAndGet(1); - } - return result.wasAcknowledged(); - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - return false; - } - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("db.").append(table.getName()).append(".remove({'_id': '").append(objectId).append("'})"); - return stringBuilder.toString(); - } -} diff --git a/src/sqlancer/mongodb/query/MongoDBSelectQuery.java b/src/sqlancer/mongodb/query/MongoDBSelectQuery.java deleted file mode 100644 index 1288f114c..000000000 --- a/src/sqlancer/mongodb/query/MongoDBSelectQuery.java +++ /dev/null @@ -1,147 +0,0 @@ -package sqlancer.mongodb.query; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.bson.Document; -import org.bson.conversions.Bson; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; - -import sqlancer.GlobalState; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.visitor.MongoDBVisitor; - -public class MongoDBSelectQuery extends MongoDBQueryAdapter { - - private final MongoDBSelect select; - - private List resultSet; - - public MongoDBSelectQuery(MongoDBSelect select) { - this.select = select; - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - throw new UnsupportedOperationException(); - } - - @Override - public ExpectedErrors getExpectedErrors() { - ExpectedErrors errors = new ExpectedErrors(); - // ARITHMETIC - errors.add("Failed to optimize pipeline :: caused by :: Can't coerce out of range value"); - errors.add("Can't coerce out of range value"); - errors.add("date overflow in $add"); - errors.add("Failed to optimize pipeline :: caused by :: $sqrt only supports numeric types, not"); - errors.add("Failed to optimize pipeline :: caused by :: $sqrt's argument must be greater than or equal to 0"); - errors.add("Failed to optimize pipeline :: caused by :: $pow's base must be numeric, not"); - errors.add("Failed to optimize pipeline :: caused by :: $pow cannot take a base of 0 and a negative exponent"); - errors.add("Failed to optimize pipeline :: caused by :: $add only supports numeric or date types, not"); - errors.add("Failed to optimize pipeline :: caused by :: $exp only supports numeric types, not"); - errors.add("Failed to optimize pipeline :: caused by :: $log's base must be numeric, not"); - errors.add("Failed to optimize pipeline :: caused by :: $log's base must be a positive number not equal to 1"); - errors.add("Failed to optimize pipeline :: caused by :: $multiply only supports numeric types, not"); - errors.add("$log's argument must be numeric, not"); - errors.add("$log's argument must be a positive number, but"); - errors.add("$log's base must be numeric, not"); - errors.add("$log's base must be a positive number not equal to 1"); - errors.add("$divide only supports numeric types, not"); - errors.add("can't $divide by zero"); - errors.add("$pow's exponent must be numeric, not"); - errors.add("$pow's base must be numeric, not"); - errors.add("$pow cannot take a base of 0 and a negative exponent"); - errors.add("$add only supports numeric or date types, not"); - errors.add("only one date allowed in an $add expression"); - errors.add("$multiply only supports numeric types, not"); - errors.add("$exp only supports numeric types, not"); - errors.add("$sqrt's argument must be greater than or equal to 0"); - errors.add("$sqrt only supports numeric types, not"); - - // REGEX - errors.add("Regular expression is invalid: nothing to repeat"); - errors.add("Regular expression is invalid: missing terminating ] for character class"); - errors.add("Regular expression is invalid: unmatched parentheses"); - errors.add("Regular expression is invalid: missing )"); - errors.add("Regular expression is invalid: invalid UTF-8 string"); - errors.add("Regular expression is invalid: \\k is not followed by a braced, angle-bracketed, or quoted name"); - errors.add("Regular expression is invalid: missing opening brace after \\\\o"); - errors.add("Regular expression is invalid: reference to non-existent subpattern"); - errors.add("Regular expression is invalid: \\ at end of pattern"); - errors.add("Regular expression is invalid: PCRE does not support \\L, \\l, \\N{name}, \\U, or \\u"); - errors.add("Regular expression is invalid: (?R or (?[+-]digits must be followed by )"); - errors.add("Regular expression is invalid: unknown property name after \\P or \\p"); - errors.add("Regular expression is invalid: (*VERB) not recognized or malformed"); - errors.add("Regular expression is invalid: a numbered reference must not be zero"); - errors.add("Regular expression is invalid: unrecognized character after (? or (?-"); - errors.add("Regular expression is invalid: \\c at end of pattern"); - errors.add("Regular expression is invalid: malformed \\P or \\p sequence"); - errors.add("Regular expression is invalid: range out of order in character class"); - errors.add("Regular expression is invalid: group name must start with a non-digit"); - errors.add("Regular expression is invalid: \\c must be followed by an ASCII character"); - errors.add("Regular expression is invalid: subpattern name expected"); - errors.add("Regular expression is invalid: POSIX collating elements are not supported"); - errors.add("Regular expression is invalid: closing ) for (?C expected"); - errors.add("Regular expression is invalid: syntax error in subpattern name (missing terminator)"); - errors.add("Regular expression is invalid: \\\\N is not supported in a class"); - errors.add("Regular expression is invalid: non-octal character in \\o{} (closing brace missing?)"); - errors.add("Regular expression is invalid: non-hex character in \\x{} (closing brace missing?)"); - errors.add( - "Regular expression is invalid: \\g is not followed by a braced, angle-bracketed, or quoted name/number or by a plain number"); - errors.add("Regular expression is invalid: digits missing in \\x{} or \\o{}"); - errors.add("Regular expression is invalid: malformed number or name after (?("); - errors.add("Regular expression is invalid: digit expected after (?+"); - errors.add("Regular expression is invalid: assertion expected after (?( or (?(?C)"); - errors.add("Regular expression is invalid: unrecognized character after (?P"); - - return errors; - } - - @Override - public > SQLancerResultSet executeAndGet(G globalState, - String... fills) throws Exception { - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(this.getLogString()); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - List pipeline = MongoDBVisitor.asQuery(select); - - MongoCollection collection = globalState.getConnection().getDatabase() - .getCollection(select.getMainTableName()); - MongoCursor cursor = collection.aggregate(pipeline).cursor(); - resultSet = new ArrayList<>(); - while (cursor.hasNext()) { - Document document = cursor.next(); - resultSet.add(document); - } - return null; - } - - @Override - public String getLogString() { - return MongoDBVisitor.asStringLog(select); - } - - public List getResultSet() { - return resultSet; - } - -} diff --git a/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java b/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java deleted file mode 100644 index 59a2a6724..000000000 --- a/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java +++ /dev/null @@ -1,40 +0,0 @@ -package sqlancer.mongodb.test; - -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; -import sqlancer.mongodb.ast.MongoDBExpression; - -public class MongoDBColumnTestReference implements Node { - - private final MongoDBColumn columnReference; - private final boolean inMainTable; - - public MongoDBColumnTestReference(MongoDBColumn columnReference, boolean inMainTable) { - this.columnReference = columnReference; - this.inMainTable = inMainTable; - } - - public String getQueryString() { - if (inMainTable) { - return this.columnReference.getName(); - } else { - return "join_" + this.columnReference.getTable().getName() + "." + this.columnReference.getName(); - } - } - - public boolean inMainTable() { - return inMainTable; - } - - public String getTableName() { - return this.columnReference.getTable().getName(); - } - - public String getPlainName() { - return this.columnReference.getName(); - } - - public MongoDBColumn getColumnReference() { - return columnReference; - } -} diff --git a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java deleted file mode 100644 index bf39643c1..000000000 --- a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java +++ /dev/null @@ -1,90 +0,0 @@ -package sqlancer.mongodb.test; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.DocumentRemovalOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mongodb.MongoDBProvider; -import sqlancer.mongodb.MongoDBSchema; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; - -public class MongoDBDocumentRemovalBase - extends DocumentRemovalOracleBase, MongoDBProvider.MongoDBGlobalState> - implements TestOracle { - - protected MongoDBSchema schema; - protected MongoDBSchema.MongoDBTables targetTables; - protected MongoDBSchema.MongoDBTable mainTable; - protected List targetColumns; - protected MongoDBMatchExpressionGenerator expressionGenerator; - protected MongoDBSelect select; - - protected MongoDBDocumentRemovalBase(MongoDBProvider.MongoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getRandomTableNonEmptyTables(); - mainTable = targetTables.getTables().get(0); - generateTargetColumns(); - expressionGenerator = new MongoDBMatchExpressionGenerator(state).setColumns(targetColumns); - initializeDocumentRemovalOracle(); - select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); - select.setProjectionList(targetColumns); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setLookupList(targetColumns); - } else { - select.setLookupList(Randomly.nonEmptySubset(targetColumns)); - } - if (state.getDbmsSpecificOptions().testComputedValues) { - generateComputedColumns(); - } - } - - private void generateTargetColumns() { - targetColumns = new ArrayList<>(); - for (MongoDBSchema.MongoDBColumn c : mainTable.getColumns()) { - targetColumns.add(new MongoDBColumnTestReference(c, true)); - } - List joinsOtherTables = new ArrayList<>(); - if (!state.getDbmsSpecificOptions().nullSafety) { - for (int i = 1; i < targetTables.getTables().size(); i++) { - MongoDBSchema.MongoDBTable procTable = targetTables.getTables().get(i); - for (MongoDBSchema.MongoDBColumn c : procTable.getColumns()) { - joinsOtherTables.add(new MongoDBColumnTestReference(c, false)); - } - } - } - if (!joinsOtherTables.isEmpty()) { - int randNumber = state.getRandomly().getInteger(1, Math.min(joinsOtherTables.size(), 4)); - List subsetJoinsOtherTables = Randomly.nonEmptySubset(joinsOtherTables, - randNumber); - targetColumns.addAll(subsetJoinsOtherTables); - } - } - - private void generateComputedColumns() { - List> computedColumns = new ArrayList<>(); - int numberComputedColumns = state.getRandomly().getInteger(1, 4); - MongoDBComputedExpressionGenerator generator = new MongoDBComputedExpressionGenerator(state) - .setColumns(targetColumns); - for (int i = 0; i < numberComputedColumns; i++) { - computedColumns.add(generator.generateExpression()); - } - select.setComputedClause(computedColumns); - } - - @Override - protected ExpressionGenerator> getGen() { - return expressionGenerator; - } -} diff --git a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java deleted file mode 100644 index ece3193d1..000000000 --- a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java +++ /dev/null @@ -1,49 +0,0 @@ -package sqlancer.mongodb.test; - -import static sqlancer.mongodb.MongoDBComparatorHelper.getResultSetAsDocumentList; - -import java.util.List; - -import org.bson.Document; - -import sqlancer.Randomly; -import sqlancer.mongodb.MongoDBProvider; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.gen.MongoDBInsertGenerator; -import sqlancer.mongodb.query.MongoDBRemoveQuery; -import sqlancer.mongodb.query.MongoDBSelectQuery; - -public class MongoDBDocumentRemovalTester extends MongoDBDocumentRemovalBase { - public MongoDBDocumentRemovalTester(MongoDBProvider.MongoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - - select.setWithCountClause(false); - - select.setFilterClause(predicate); - MongoDBSelectQuery selectQuery = new MongoDBSelectQuery(select); - List firstResultSet = getResultSetAsDocumentList(selectQuery, state); - if (firstResultSet == null || firstResultSet.isEmpty()) { - return; - } - - Document documentToRemove = Randomly.fromList(firstResultSet); - MongoDBRemoveQuery removeQuery = new MongoDBRemoveQuery(mainTable, documentToRemove.get("_id").toString()); - state.executeStatement(removeQuery); - - selectQuery = new MongoDBSelectQuery(select); - List secondResultSet = getResultSetAsDocumentList(selectQuery, state); - - MongoDBQueryAdapter insertQuery = MongoDBInsertGenerator.getQuery(state); - state.executeStatement(insertQuery); - - if (secondResultSet.size() + 1 != firstResultSet.size()) { - String assertMessage = "The Result Sizes mismatches!"; - throw new AssertionError(assertMessage); - } - } -} diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java deleted file mode 100644 index ea6d39bbf..000000000 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ /dev/null @@ -1,93 +0,0 @@ -package sqlancer.mongodb.test; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBSchema; -import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.MongoDBSchema.MongoDBTables; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; - -public class MongoDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, MongoDBGlobalState> - implements TestOracle { - - protected MongoDBSchema schema; - protected MongoDBTables targetTables; - protected MongoDBTable mainTable; - protected List targetColumns; - protected MongoDBMatchExpressionGenerator expressionGenerator; - protected MongoDBSelect select; - - public MongoDBQueryPartitioningBase(MongoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getRandomTableNonEmptyTables(); - mainTable = targetTables.getTables().get(0); - generateTargetColumns(); - expressionGenerator = new MongoDBMatchExpressionGenerator(state).setColumns(targetColumns); - initializeTernaryPredicateVariants(); - select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); - select.setProjectionList(targetColumns); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setLookupList(targetColumns); - } else { - select.setLookupList(Randomly.nonEmptySubset(targetColumns)); - } - if (state.getDbmsSpecificOptions().testComputedValues) { - generateComputedColumns(); - } - } - - private void generateComputedColumns() { - List> computedColumns = new ArrayList<>(); - int numberComputedColumns = state.getRandomly().getInteger(1, 4); - MongoDBComputedExpressionGenerator generator = new MongoDBComputedExpressionGenerator(state) - .setColumns(targetColumns); - for (int i = 0; i < numberComputedColumns; i++) { - computedColumns.add(generator.generateExpression()); - } - select.setComputedClause(computedColumns); - } - - private void generateTargetColumns() { - targetColumns = new ArrayList<>(); - for (MongoDBColumn c : mainTable.getColumns()) { - targetColumns.add(new MongoDBColumnTestReference(c, true)); - } - List joinsOtherTables = new ArrayList<>(); - if (!state.getDbmsSpecificOptions().nullSafety) { - for (int i = 1; i < targetTables.getTables().size(); i++) { - MongoDBTable procTable = targetTables.getTables().get(i); - for (MongoDBColumn c : procTable.getColumns()) { - joinsOtherTables.add(new MongoDBColumnTestReference(c, false)); - } - } - } - if (!joinsOtherTables.isEmpty()) { - int randNumber = state.getRandomly().getInteger(1, Math.min(joinsOtherTables.size(), 4)); - List subsetJoinsOtherTables = Randomly.nonEmptySubset(joinsOtherTables, - randNumber); - targetColumns.addAll(subsetJoinsOtherTables); - } - } - - @Override - protected ExpressionGenerator> getGen() { - return expressionGenerator; - } -} diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java deleted file mode 100644 index 53f625d9b..000000000 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,48 +0,0 @@ -package sqlancer.mongodb.test; - -import static sqlancer.mongodb.MongoDBComparatorHelper.getResultSetAsDocumentList; - -import java.util.List; - -import org.bson.Document; - -import sqlancer.mongodb.MongoDBComparatorHelper; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.query.MongoDBSelectQuery; - -public class MongoDBQueryPartitioningWhereTester extends MongoDBQueryPartitioningBase { - public MongoDBQueryPartitioningWhereTester(MongoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - - select.setWithCountClause(false); - - select.setFilterClause(null); - MongoDBSelectQuery q = new MongoDBSelectQuery(select); - List firstResultSet = getResultSetAsDocumentList(q, state); - - select.setFilterClause(predicate); - q = new MongoDBSelectQuery(select); - List secondResultSet = getResultSetAsDocumentList(q, state); - - select.setFilterClause(negatedPredicate); - q = new MongoDBSelectQuery(select); - List thirdResultSet = getResultSetAsDocumentList(q, state); - - if (state.getDbmsSpecificOptions().testWithCount) { - select.setWithCountClause(true); - select.setFilterClause(predicate); - q = new MongoDBSelectQuery(select); - List forthResultSet = getResultSetAsDocumentList(q, state); - MongoDBComparatorHelper.assumeCountIsEqual(secondResultSet, forthResultSet, q); - } - - secondResultSet.addAll(thirdResultSet); - MongoDBComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, q); - - } -} diff --git a/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java deleted file mode 100644 index 39b607f67..000000000 --- a/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java +++ /dev/null @@ -1,161 +0,0 @@ -package sqlancer.mongodb.visitor; - -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator.AND; -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator.NOR; -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator.OR; -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBUnaryLogicalOperator.NOT; - -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; -import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBRegexNode; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; - -public class MongoDBNegateVisitor extends MongoDBVisitor { - - private boolean negate; - Node negatedExpression; - - public MongoDBNegateVisitor(boolean negate) { - this.negate = negate; - } - - @Override - public void visit(Node expr) { - if (expr instanceof MongoDBConstant) { - visit((MongoDBConstant) expr); - } else if (expr instanceof MongoDBSelect) { - visit((MongoDBSelect) expr); - } else if (expr instanceof MongoDBBinaryComparisonNode) { - visit((MongoDBBinaryComparisonNode) expr); - } else if (expr instanceof MongoDBUnaryLogicalOperatorNode) { - visit((MongoDBUnaryLogicalOperatorNode) expr); - } else if (expr instanceof MongoDBRegexNode) { - visit((MongoDBRegexNode) expr); - } else if (expr instanceof MongoDBBinaryLogicalNode) { - visit((MongoDBBinaryLogicalNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public void visit(MongoDBBinaryComparisonNode expr) { - - if (negate) { - negatedExpression = new MongoDBUnaryLogicalOperatorNode(expr, NOT); - switch (expr.operator()) { - case EQUALS: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.NOT_EQUALS); - break; - case NOT_EQUALS: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.EQUALS); - break; - case LESS: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER_EQUAL); - break; - case LESS_EQUAL: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER); - break; - case GREATER: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS_EQUAL); - break; - case GREATER_EQUAL: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS); - break; - default: - throw new UnsupportedOperationException(); - } - } else { - negatedExpression = expr; - } - } - - public void visit(MongoDBRegexNode expr) { - if (negate) { - negatedExpression = new MongoDBUnaryLogicalOperatorNode(expr, NOT); - } else { - negatedExpression = expr; - } - } - - public void visit(MongoDBUnaryLogicalOperatorNode expr) { - if (!(expr.operator().equals(NOT))) { - throw new UnsupportedOperationException(); - } - negate = !negate; - visit(expr.getExpr()); - } - - public void visit(MongoDBBinaryLogicalNode expr) { - boolean saveNegate = negate; - Node left; - Node right; - switch (expr.operator()) { - case OR: - negate = false; - visit(expr.getLeft()); - left = negatedExpression; - negate = false; - visit(expr.getRight()); - right = negatedExpression; - if (saveNegate) { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, NOR); - } else { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, OR); - } - break; - case AND: - negate = saveNegate; - visit(expr.getLeft()); - left = negatedExpression; - negate = saveNegate; - visit(expr.getRight()); - right = negatedExpression; - if (saveNegate) { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, OR); - } else { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, AND); - } - break; - case NOR: - negate = false; - visit(expr.getLeft()); - left = negatedExpression; - negate = false; - visit(expr.getRight()); - right = negatedExpression; - if (saveNegate) { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, OR); - } else { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, NOR); - } - break; - default: - throw new UnsupportedOperationException(expr.getOperatorRepresentation()); - } - } - - @Override - public void visit(MongoDBConstant c) { - negatedExpression = c; - } - - @Override - public void visit(MongoDBSelect s) { - throw new UnsupportedOperationException(); - } - - public Node getNegatedExpression() { - return negatedExpression; - } -} diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java deleted file mode 100644 index 4c55e17a6..000000000 --- a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java +++ /dev/null @@ -1,195 +0,0 @@ -package sqlancer.mongodb.visitor; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; -import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBRegexNode; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; -import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator.ComputedFunction; -import sqlancer.mongodb.test.MongoDBColumnTestReference; - -public class MongoDBToLogVisitor extends MongoDBVisitor { - - private String mainTableName; - private List lookups; - private String filter; - private String projects; - private boolean hasFilter; - private boolean withCount; - - public String visitLog(Node expr) { - if (expr instanceof MongoDBUnaryLogicalOperatorNode) { - return visit((MongoDBUnaryLogicalOperatorNode) expr); - } else if (expr instanceof MongoDBBinaryLogicalNode) { - return visit((MongoDBBinaryLogicalNode) expr); - } else if (expr instanceof MongoDBBinaryComparisonNode) { - return visit((MongoDBBinaryComparisonNode) expr); - } else if (expr instanceof MongoDBRegexNode) { - return visit((MongoDBRegexNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public String visitComputed(Node expr) { - if (expr instanceof NewFunctionNode) { - return visitComputed((NewFunctionNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public String visitComputed(NewFunctionNode expr) { - List arguments = new ArrayList<>(); - for (int i = 0; i < expr.getArgs().size(); i++) { - if (expr.getArgs().get(i) instanceof MongoDBConstant) { - arguments.add(((MongoDBConstant) expr.getArgs().get(i)).getLogValue()); - continue; - } - if (expr.getArgs().get(i) instanceof MongoDBColumnTestReference) { - arguments.add("\"$" + ((MongoDBColumnTestReference) expr.getArgs().get(i)).getQueryString() + "\""); - continue; - } - if (expr.getArgs().get(i) instanceof NewFunctionNode) { - arguments.add(visitComputed((NewFunctionNode) expr.getArgs().get(i))); - } else { - throw new AssertionError(); - } - } - if (!(expr.getFunc() instanceof ComputedFunction)) { - throw new AssertionError(expr.getClass()); - } - - StringBuilder sb = new StringBuilder(); - sb.append("{"); - sb.append(((ComputedFunction) expr.getFunc()).getOperator()); - sb.append(": ["); - String helper = ""; - for (String arg : arguments) { - sb.append(helper); - helper = ", "; - sb.append(arg); - } - sb.append("]}"); - return sb.toString(); - } - - public String visit(MongoDBUnaryLogicalOperatorNode expr) { - String inner = visitLog(expr.getExpr()); - return "{ " + expr.operator().getTextRepresentation() + ": [" + inner + "]}"; - } - - public String visit(MongoDBBinaryLogicalNode expr) { - String left = visitLog(expr.getLeft()); - String right = visitLog(expr.getRight()); - - return "{" + expr.operator().getTextRepresentation() + ":[" + left + "," + right + "]}"; - } - - public String visit(MongoDBBinaryComparisonNode expr) { - Node left = expr.getLeft(); - Node right = expr.getRight(); - assert left instanceof MongoDBColumnTestReference; - assert right instanceof MongoDBConstant; - - return "{\"" + ((MongoDBColumnTestReference) left).getQueryString() + "\": {" - + expr.operator().getTextRepresentation() + ": " + ((MongoDBConstant) right).getLogValue() + "}}"; - } - - public String visit(MongoDBRegexNode expr) { - Node left = expr.getLeft(); - Node right = expr.getRight(); - - return "{\"" + ((MongoDBColumnTestReference) left).getQueryString() + "\": {" - + expr.operator().getTextRepresentation() + ": \'" - + ((MongoDBConstant.MongoDBStringConstant) right).getStringValue() + "\', $options: \'" - + expr.getOptions() + "\'}}"; - } - - @Override - public void visit(MongoDBConstant c) { - throw new UnsupportedOperationException(); - } - - @Override - public void visit(MongoDBSelect select) { - hasFilter = select.hasFilter(); - mainTableName = select.getMainTableName(); - setLookups(select); - if (hasFilter) { - setFilter(select); - } - setProjects(select); - withCount = select.getWithCountClause(); - } - - private void setFilter(MongoDBSelect select) { - filter = visitLog(select.getFilterClause()); - } - - private void setLookups(MongoDBSelect select) { - lookups = new ArrayList<>(); - for (MongoDBColumnTestReference testReference : select.getLookupList()) { - if (testReference.inMainTable()) { - continue; - } - String newLookup = "{ $lookup: { from: \"" + testReference.getTableName() + "\", localField: \"" - + select.getJoinColumn().getPlainName() + "\", foreignField: \"" + testReference.getPlainName() - + "\", as: \"" + testReference.getQueryString() + "\"}},\n"; - lookups.add(newLookup); - } - } - - private void setProjects(MongoDBSelect select) { - StringBuilder sb = new StringBuilder(); - sb.append("{"); - String helper = ""; - for (MongoDBColumnTestReference reference : select.getProjectionList()) { - sb.append(helper); - helper = ","; - sb.append("\"").append(reference.getQueryString()).append("\"").append(": 1"); - } - sb.append("\n"); - if (select.hasComputed()) { - String name = "computed"; - int number = 0; - for (Node expressionNode : select.getComputedClause()) { - sb.append(helper); - helper = ",\n"; - sb.append("\"" + name + number + "\": " + visitComputed(expressionNode)); - number++; - } - } - sb.append("}"); - projects = sb.toString(); - } - - public String getStringLog() { - StringBuilder sb = new StringBuilder(); - sb.append("db.").append(mainTableName).append(".aggregate([\n"); - for (String lookup : lookups) { - sb.append(lookup); - } - if (hasFilter) { - sb.append("{ $match: "); - sb.append(filter); - sb.append("},\n"); - } - sb.append("{ $project : "); - sb.append(projects); - sb.append("}"); - if (withCount) { - sb.append(",\n"); - sb.append(" {$count: \"count\"}\n"); - } - sb.append("])\n"); - return sb.toString(); - } -} diff --git a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java deleted file mode 100644 index 8efbf97e4..000000000 --- a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java +++ /dev/null @@ -1,184 +0,0 @@ -package sqlancer.mongodb.visitor; - -import static com.mongodb.client.model.Aggregates.match; -import static com.mongodb.client.model.Aggregates.project; -import static com.mongodb.client.model.Projections.fields; -import static com.mongodb.client.model.Projections.include; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import org.bson.Document; -import org.bson.conversions.Bson; - -import com.mongodb.client.model.Aggregates; -import com.mongodb.client.model.Projections; - -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; -import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBStringConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBRegexNode; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; -import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator.ComputedFunction; -import sqlancer.mongodb.test.MongoDBColumnTestReference; - -public class MongoDBToQueryVisitor extends MongoDBVisitor { - - private List lookup; - private Bson filter; - private Bson projection; - private Bson count; - private boolean hasFilter; - private boolean hasCountClause; - - public Bson visitBson(Node expr) { - if (expr instanceof MongoDBUnaryLogicalOperatorNode) { - return visit((MongoDBUnaryLogicalOperatorNode) expr); - } else if (expr instanceof MongoDBBinaryLogicalNode) { - return visit((MongoDBBinaryLogicalNode) expr); - } else if (expr instanceof MongoDBBinaryComparisonNode) { - return visit((MongoDBBinaryComparisonNode) expr); - } else if (expr instanceof MongoDBRegexNode) { - return visit((MongoDBRegexNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public Document visitComputed(Node expr) { - if (expr instanceof NewFunctionNode) { - return visitComputed((NewFunctionNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public Document visitComputed(NewFunctionNode expr) { - List visitedArgs = new ArrayList<>(); - for (int i = 0; i < expr.getArgs().size(); i++) { - if (expr.getArgs().get(i) instanceof MongoDBConstant) { - visitedArgs.add(((MongoDBConstant) expr.getArgs().get(i)).getSerializedValue()); - continue; - } - if (expr.getArgs().get(i) instanceof MongoDBColumnTestReference) { - visitedArgs.add("$" + ((MongoDBColumnTestReference) expr.getArgs().get(i)).getQueryString()); - continue; - } - if (expr.getArgs().get(i) instanceof NewFunctionNode) { - visitedArgs.add(visitComputed((NewFunctionNode) expr.getArgs().get(i))); - } else { - throw new AssertionError(); - } - } - if (expr.getFunc() instanceof ComputedFunction) { - return new Document(((ComputedFunction) expr.getFunc()).getOperator(), visitedArgs); - } else { - throw new AssertionError(expr.getClass()); - } - - } - - public Bson visit(MongoDBUnaryLogicalOperatorNode expr) { - Bson inner = visitBson(expr.getExpr()); - return expr.operator().applyOperator(inner); - } - - public Bson visit(MongoDBBinaryLogicalNode expr) { - Bson left = visitBson(expr.getLeft()); - Bson right = visitBson(expr.getRight()); - return expr.operator().applyOperator(left, right); - } - - public Bson visit(MongoDBRegexNode expr) { - Node left = expr.getLeft(); - Node right = expr.getRight(); - - String columnName = ((MongoDBColumnTestReference) left).getQueryString(); - - return expr.operator().applyOperator(columnName, (MongoDBStringConstant) right, expr.getOptions()); - } - - public Bson visit(MongoDBBinaryComparisonNode expr) { - Node left = expr.getLeft(); - Node right = expr.getRight(); - assert left instanceof MongoDBColumnTestReference; - assert right instanceof MongoDBConstant; - - String columnName = ((MongoDBColumnTestReference) left).getQueryString(); - return expr.operator().applyOperator(columnName, (MongoDBConstant) right); - } - - @Override - public void visit(MongoDBConstant c) { - throw new UnsupportedOperationException(); - } - - @Override - public void visit(MongoDBSelect select) { - hasFilter = select.hasFilter(); - setLookup(select); - if (hasFilter) { - setFilter(select); - } - setProjection(select); - hasCountClause = select.getWithCountClause(); - if (hasCountClause) { - setCount(); - } - } - - private void setCount() { - count = Aggregates.count("count"); - } - - private void setFilter(MongoDBSelect select) { - filter = match(this.visitBson(select.getFilterClause())); - } - - private void setLookup(MongoDBSelect select) { - lookup = new ArrayList<>(); - for (MongoDBColumnTestReference reference : select.getLookupList()) { - if (reference.inMainTable()) { - continue; - } - lookup.add(Aggregates.lookup(reference.getTableName(), select.getJoinColumn().getPlainName(), - reference.getPlainName(), reference.getQueryString())); - } - } - - private void setProjection(MongoDBSelect select) { - List stringProjects = new ArrayList<>(); - for (MongoDBColumnTestReference ref : select.getProjectionList()) { - stringProjects.add(ref.getQueryString()); - } - List projections = new ArrayList<>(); - projections.add(include(stringProjects)); - if (select.hasComputed()) { - String name = "computed"; - int number = 0; - for (Node expressionNode : select.getComputedClause()) { - projections.add(Projections.computed(name + number, visitComputed(expressionNode))); - number++; - } - } - projection = project(fields(projections)); - } - - public List getPipeline() { - List result = new ArrayList<>(lookup); - if (hasFilter) { - result.add(filter); - } - result.add(projection); - if (hasCountClause) { - result.add(count); - } - return result; - } -} diff --git a/src/sqlancer/mongodb/visitor/MongoDBVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBVisitor.java deleted file mode 100644 index e02a50f02..000000000 --- a/src/sqlancer/mongodb/visitor/MongoDBVisitor.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.mongodb.visitor; - -import java.util.List; - -import org.bson.conversions.Bson; - -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBSelect; - -public abstract class MongoDBVisitor { - - public abstract void visit(MongoDBConstant c); - - public abstract void visit(MongoDBSelect s); - - public void visit(Node expr) { - if (expr instanceof MongoDBConstant) { - visit((MongoDBConstant) expr); - } else if (expr instanceof MongoDBSelect) { - visit((MongoDBSelect) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public static List asQuery(Node expr) { - MongoDBToQueryVisitor visitor = new MongoDBToQueryVisitor(); - visitor.visit(expr); - return visitor.getPipeline(); - } - - public static String asStringLog(Node expr) { - MongoDBToLogVisitor visitor = new MongoDBToLogVisitor(); - visitor.visit(expr); - return visitor.getStringLog(); - } - - public static Node cleanNegations(Node expr) { - MongoDBNegateVisitor visitor = new MongoDBNegateVisitor(false); - visitor.visit(expr); - return visitor.getNegatedExpression(); - } -} From d01c4a5ddff7d8ae0250a02322ab539526ba7f41 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 17 Mar 2024 10:32:26 +0800 Subject: [PATCH 0831/1284] Support TiFlash for TiDB (#919) --- src/sqlancer/tidb/TiDBOptions.java | 3 +++ src/sqlancer/tidb/TiDBProvider.java | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 7ff20027d..d35ce9656 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -36,6 +36,9 @@ public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = "--enable-non-prepared-plan-cache") public boolean nonPreparePlanCache; + @Parameter(names = { "--tiflash" }, description = "Enable TiFlash") + public boolean tiflash; + public enum TiDBOracleFactory implements OracleFactory { HAVING { @Override diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index c5c334f11..05ef0d9c6 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -154,6 +154,21 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { } } } + + // TiFlash replication settings + if (globalState.getDbmsSpecificOptions().tiflash) { + ExpectedErrors errors = new ExpectedErrors(); + TiDBErrors.addExpressionErrors(errors); + for (TiDBTable table : globalState.getSchema().getDatabaseTables()) { + if (!table.isView()) { + globalState.executeStatement( + new SQLQueryAdapter("ALTER TABLE " + table.getName() + " SET TIFLASH REPLICA 1;", errors)); + } + } + if (Randomly.getBoolean()) { + globalState.executeStatement(new SQLQueryAdapter("set @@tidb_enforce_mpp=1;")); + } + } } @Override From 49afc88429744b54a7bc25d6006d63598f08da00 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 17 Mar 2024 11:26:50 +0800 Subject: [PATCH 0832/1284] TiDB DQP (#920) --- src/sqlancer/tidb/TiDBBugs.java | 15 ++++ src/sqlancer/tidb/TiDBErrors.java | 5 +- src/sqlancer/tidb/TiDBOptions.java | 7 ++ .../tidb/gen/TiDBAlterTableGenerator.java | 1 + src/sqlancer/tidb/gen/TiDBHintGenerator.java | 55 +++++++++++-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 4 +- src/sqlancer/tidb/oracle/TiDBDQPOracle.java | 80 +++++++++++++++++++ test/sqlancer/dqp/tidb/TestTiDBDQP.java | 22 +++++ 8 files changed, 182 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/tidb/oracle/TiDBDQPOracle.java create mode 100644 test/sqlancer/dqp/tidb/TestTiDBDQP.java diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index 4aef122e3..81f27b705 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -17,6 +17,21 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/44747 public static boolean bug44747 = true; + // https://github.com/pingcap/tidb/issues/46556 + public static boolean bug46556 = true; + + // https://github.com/pingcap/tidb/issues/46591 + public static boolean bug46591 = true; + + // https://github.com/pingcap/tidb/issues/46598 + public static boolean bug46598 = true; + + // https://github.com/pingcap/tidb/issues/47346 + public static boolean bug47346 = true; + + // https://github.com/pingcap/tidb/issues/47348 + public static boolean bug47348 = true; + private TiDBBugs() { } diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 2068f9f1e..fd4d23d3d 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -25,9 +25,12 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("is not valid for CHARACTER SET"); errors.add("DOUBLE value is out of range"); - errors.add("Data truncation: %s value is out of range in '%s'"); + errors.add("Data truncat"); errors.add("Truncated incorrect FLOAT value"); errors.add("Bad Number"); + errors.add("strconv.Atoi: parsing"); + errors.add("expected integer"); + errors.add("Duplicate entry"); // regex errors.add("error parsing regexp"); diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index d35ce9656..6c69e84a1 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -15,6 +15,7 @@ import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.oracle.TiDBCERTOracle; +import sqlancer.tidb.oracle.TiDBDQPOracle; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @@ -71,6 +72,12 @@ public TestOracle create(TiDBGlobalState globalState) throws SQ public boolean requiresAllTablesToContainRows() { return true; } + }, + DQP { + @Override + public TestOracle create(TiDBGlobalState globalState) throws SQLException { + return new TiDBDQPOracle(globalState); + } }; } diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 6a95fb9a9..535cf973f 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -32,6 +32,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("can't drop"); errors.add("A PRIMARY must include all columns in the table's partitioning function"); errors.add("key was too long"); + errors.add("Duplicate entry"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index b1a343c67..f58695bb8 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -1,5 +1,7 @@ package sqlancer.tidb.gen; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -22,12 +24,13 @@ enum IndexHint { INL_HASH_JOIN, // INL_MERGE_JOIN, // HASH_JOIN, // + READ_FROM_TIKV, // + READ_FROM_TIFLASH, // HASH_AGG, // STREAM_AGG, // USE_INDEX, // IGNORE_INDEX, // AGG_TO_COP, // - // READ_FROM_STORAGE USE_INDEX_MERGE, // NO_INDEX_MERGE, // USE_TOJA, // @@ -46,13 +49,40 @@ public TiDBHintGenerator(TiDBSelect select, List tables) { } public static void generateHints(TiDBSelect select, List tables) { - new TiDBHintGenerator(select, tables).generate(); + new TiDBHintGenerator(select, tables).randomHint(); + } + public static List generateAllHints(TiDBSelect select, List tables) { + TiDBHintGenerator generator = new TiDBHintGenerator(select, tables); + return generator.allHints(); } - private void generate() { + private void randomHint() { TiDBTable table = Randomly.fromList(tables); - switch (Randomly.fromOptions(IndexHint.values())) { + IndexHint chosenhint = Randomly.fromOptions(IndexHint.values()); + generate(table, chosenhint); + } + + private List allHints() { + List results = new ArrayList<>(); + IndexHint[] values = IndexHint.values(); + List availableHints = new ArrayList<>(Arrays.asList(values)); + + for (IndexHint hint : availableHints) { + try { + TiDBText generatedHint = generate(Randomly.fromList(tables), hint); + results.add(generatedHint); + } catch (IgnoreMeException e) { + continue; + } + } + return results; + } + + private TiDBText generate(TiDBTable table, IndexHint chosenhint) { + sb.setLength(0); + + switch (chosenhint) { case MERGE_JOIN: tablesHint("MERGE_JOIN"); break; @@ -68,6 +98,12 @@ private void generate() { case HASH_JOIN: tablesHint("HASH_JOIN"); break; + case READ_FROM_TIKV: + storageHint("READ_FROM_STORAGE(TIKV"); + break; + case READ_FROM_TIFLASH: + storageHint("READ_FROM_STORAGE(TIFLASH"); + break; case HASH_AGG: sb.append("HASH_AGG()"); break; @@ -126,7 +162,9 @@ private void generate() { default: throw new AssertionError(); } - select.setHint(new TiDBText(sb.toString())); + TiDBText hint = new TiDBText(sb.toString()); + select.setHint(hint); + return hint; } private void indexesHint(String string) { @@ -152,6 +190,13 @@ private void tablesHint(String string) { sb.append(")"); } + private void storageHint(String string) { + sb.append(string); + sb.append("["); + appendTables(); + sb.append("])"); + } + private void twoTablesHint(String string, TiDBTable table) { if (table.hasIndexes()) { sb.append(string); diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index b48ea31cc..11f599fc0 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -43,7 +43,9 @@ private enum Action { TIDB_ENABLE_WINDOW_FUNCTION("tidb_enable_window_function", (r) -> Randomly.fromOptions(0, 1)), - TIDB_ENABLE_FAST_ANALYZE("tidb_enable_fast_analyze", (r) -> Randomly.fromOptions(0, 1)), // + // TIDB_ENABLE_FAST_ANALYZE("tidb_enable_fast_analyze", (r) -> Randomly.fromOptions(0, 1)), // + // java.sql.SQLException: Fast analyze hasn't reached General Availability and only support analyze version 1 + // currently TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions(0, 1)), TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // diff --git a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java new file mode 100644 index 000000000..e35e4ee70 --- /dev/null +++ b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java @@ -0,0 +1,80 @@ +package sqlancer.tidb.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.tidb.TiDBErrors; +import sqlancer.tidb.TiDBExpressionGenerator; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.TiDBSchema.TiDBTables; +import sqlancer.tidb.ast.TiDBColumnReference; +import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableReference; +import sqlancer.tidb.ast.TiDBText; +import sqlancer.tidb.gen.TiDBHintGenerator; +import sqlancer.tidb.visitor.TiDBVisitor; + +public class TiDBDQPOracle implements TestOracle { + private TiDBExpressionGenerator gen; + private final TiDBGlobalState state; + private TiDBSelect select; + private final ExpectedErrors errors = new ExpectedErrors(); + + public TiDBDQPOracle(TiDBGlobalState globalState) { + state = globalState; + TiDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + // Randomly generate a query + TiDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new TiDBExpressionGenerator(state).setColumns(tables.getColumns()); + select = new TiDBSelect(); + + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new TiDBColumnReference(c)) + .collect(Collectors.toList())); + select.setFetchColumns(fetchColumns); + + List tableList = tables.getTables().stream().map(t -> new TiDBTableReference(t)) + .collect(Collectors.toList()); + List joins = TiDBJoin.getJoins(tableList, state); + select.setJoinList(joins); + select.setFromList(tableList); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(gen.generateExpression()); + } + + String originalQueryString = TiDBVisitor.asString(select); + List originalResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + List hintList = TiDBHintGenerator.generateAllHints(select, tables.getTables()); + for (TiDBText hint : hintList) { + select.setHint(hint); + String queryString = TiDBVisitor.asString(select); + List result = ComparatorHelper.getResultSetFirstColumnAsString(queryString, errors, state); + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, List.of(queryString), + state); + } + } + +} diff --git a/test/sqlancer/dqp/tidb/TestTiDBDQP.java b/test/sqlancer/dqp/tidb/TestTiDBDQP.java new file mode 100644 index 000000000..300dfde34 --- /dev/null +++ b/test/sqlancer/dqp/tidb/TestTiDBDQP.java @@ -0,0 +1,22 @@ +package sqlancer.dqp.tidb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestTiDBDQP { + + @Test + public void testTiDBQPG() { + String tiDB = System.getenv("TIDB_AVAILABLE"); + boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); + assumeTrue(tiDBIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "tidb", "--oracle", "DQP" })); + } + +} From 93626814cb51f77f6e86cfa95e5bced81dc0cb0f Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 18 Mar 2024 23:38:26 +0800 Subject: [PATCH 0833/1284] MySQL Join implementation (#921) --- .../mysql/MySQLExpectedValueVisitor.java | 13 +++ src/sqlancer/mysql/MySQLProvider.java | 10 +-- src/sqlancer/mysql/MySQLToStringVisitor.java | 39 +++++++++ src/sqlancer/mysql/MySQLVisitor.java | 10 +++ src/sqlancer/mysql/ast/MySQLJoin.java | 81 ++++++++++++++++++- src/sqlancer/mysql/ast/MySQLText.java | 14 ++++ 6 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 src/sqlancer/mysql/ast/MySQLText.java diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index fd77a9f85..0ff8e389d 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -13,10 +13,12 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; public class MySQLExpectedValueVisitor implements MySQLVisitor { @@ -153,4 +155,15 @@ public void visit(MySQLCollate collate) { visit(collate.getExpectedValue()); } + @Override + public void visit(MySQLJoin join) { + print(join); + visit(join.getOnClause()); + } + + @Override + public void visit(MySQLText text) { + print(text); + } + } diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 9392550cc..287f20a7e 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -63,11 +63,6 @@ enum Action implements AbstractAction { SELECT_INFO((g) -> new SQLQueryAdapter( "select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + g.getDatabaseName() + "'")), // - CREATE_TABLE((g) -> { - // TODO refactor - String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); - return MySQLTableGenerator.generate(g, tableName); - }), // UPDATE(MySQLUpdateGenerator::create), // DELETE(MySQLDeleteGenerator::delete), // DROP_INDEX(MySQLDropIndex::generate); @@ -94,9 +89,6 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { case SHOW_TABLES: nrPerformed = r.getInteger(0, 1); break; - case CREATE_TABLE: - nrPerformed = r.getInteger(0, 1); - break; case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; @@ -148,7 +140,7 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { @Override public void generateDatabase(MySQLGlobalState globalState) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.getNotCachedInteger(1, 2)) { String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); SQLQueryAdapter createTable = MySQLTableGenerator.generate(globalState, tableName); globalState.executeStatement(createTable); diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 06f13b008..b61e501e5 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -17,11 +17,13 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; public class MySQLToStringVisitor extends ToStringVisitor implements MySQLVisitor { @@ -278,4 +280,41 @@ public void visit(MySQLCollate collate) { sb.append(")"); } + @Override + public void visit(MySQLJoin join) { + sb.append(" "); + switch (join.getType()) { + case NATURAL: + sb.append("NATURAL "); + break; + case INNER: + sb.append("INNER "); + break; + case STRAIGHT: + sb.append("STRAIGHT_"); + break; + case LEFT: + sb.append("LEFT "); + break; + case RIGHT: + sb.append("RIGHT "); + break; + case CROSS: + sb.append("CROSS "); + break; + default: + throw new AssertionError(join.getType()); + } + sb.append("JOIN "); + sb.append(join.getTable().getName()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + @Override + public void visit(MySQLText text) { + sb.append(text.getText()); + } } diff --git a/src/sqlancer/mysql/MySQLVisitor.java b/src/sqlancer/mysql/MySQLVisitor.java index eeda4f681..7b8944f28 100644 --- a/src/sqlancer/mysql/MySQLVisitor.java +++ b/src/sqlancer/mysql/MySQLVisitor.java @@ -12,10 +12,12 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; public interface MySQLVisitor { @@ -52,6 +54,10 @@ public interface MySQLVisitor { void visit(MySQLCollate collate); + void visit(MySQLJoin join); + + void visit(MySQLText text); + default void visit(MySQLExpression expr) { if (expr instanceof MySQLConstant) { visit((MySQLConstant) expr); @@ -77,6 +83,8 @@ default void visit(MySQLExpression expr) { visit((MySQLOrderByTerm) expr); } else if (expr instanceof MySQLExists) { visit((MySQLExists) expr); + } else if (expr instanceof MySQLJoin) { + visit((MySQLJoin) expr); } else if (expr instanceof MySQLStringExpression) { visit((MySQLStringExpression) expr); } else if (expr instanceof MySQLBetweenOperation) { @@ -85,6 +93,8 @@ default void visit(MySQLExpression expr) { visit((MySQLTableReference) expr); } else if (expr instanceof MySQLCollate) { visit((MySQLCollate) expr); + } else if (expr instanceof MySQLText) { + visit((MySQLText) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/mysql/ast/MySQLJoin.java b/src/sqlancer/mysql/ast/MySQLJoin.java index 8558e43a9..13c21b7a0 100644 --- a/src/sqlancer/mysql/ast/MySQLJoin.java +++ b/src/sqlancer/mysql/ast/MySQLJoin.java @@ -1,10 +1,85 @@ package sqlancer.mysql.ast; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + public class MySQLJoin implements MySQLExpression { - @Override - public MySQLConstant getExpectedValue() { - throw new UnsupportedOperationException(); + public enum JoinType { + NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; + } + + private final MySQLTable table; + private MySQLExpression onClause; + private JoinType type; + + public MySQLJoin(MySQLJoin other) { + this.table = other.table; + this.onClause = other.onClause; + this.type = other.type; } + public MySQLJoin(MySQLTable table, MySQLExpression onClause, JoinType type) { + this.table = table; + this.onClause = onClause; + this.type = type; + } + + public MySQLTable getTable() { + return table; + } + + public MySQLExpression getOnClause() { + return onClause; + } + + public JoinType getType() { + return type; + } + + public void setOnClause(MySQLExpression onClause) { + this.onClause = onClause; + } + + public void setType(JoinType type) { + this.type = type; + } + + public static List getRandomJoinClauses(List tables, MySQLGlobalState globalState) { + List joinStatements = new ArrayList<>(); + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + List columns = new ArrayList<>(); + if (tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + MySQLTable table = Randomly.fromList(tables); + tables.remove(table); + columns.addAll(table.getColumns()); + MySQLExpressionGenerator joinGen = new MySQLExpressionGenerator(globalState).setColumns(columns); + MySQLExpression joinClause = joinGen.generateExpression(); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + joinClause = null; + } + MySQLJoin j = new MySQLJoin(table, joinClause, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } } diff --git a/src/sqlancer/mysql/ast/MySQLText.java b/src/sqlancer/mysql/ast/MySQLText.java new file mode 100644 index 000000000..36040a383 --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLText.java @@ -0,0 +1,14 @@ +package sqlancer.mysql.ast; + +public class MySQLText implements MySQLExpression { + + private final String text; + + public MySQLText(String text) { + this.text = text; + } + + public String getText() { + return text; + } +} From a546635d884ab88fba62703cfa9896f6e3db3b65 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 19 Mar 2024 19:31:01 +0800 Subject: [PATCH 0834/1284] MySQL DQP (#923) --- src/sqlancer/mysql/MySQLBugs.java | 9 + src/sqlancer/mysql/MySQLOptions.java | 7 + src/sqlancer/mysql/MySQLToStringVisitor.java | 5 + src/sqlancer/mysql/ast/MySQLSelect.java | 9 + .../mysql/gen/MySQLHintGenerator.java | 204 ++++++++++++++++++ src/sqlancer/mysql/gen/MySQLSetGenerator.java | 57 ++++- src/sqlancer/mysql/oracle/MySQLDQPOracle.java | 104 +++++++++ test/sqlancer/dqp/mysql/TestMySQLDQP.java | 22 ++ 8 files changed, 412 insertions(+), 5 deletions(-) create mode 100644 src/sqlancer/mysql/gen/MySQLHintGenerator.java create mode 100644 src/sqlancer/mysql/oracle/MySQLDQPOracle.java create mode 100644 test/sqlancer/dqp/mysql/TestMySQLDQP.java diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index ea44c83d4..00a7a0ddf 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -22,6 +22,15 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=111471 public static boolean bug111471 = true; + // https://bugs.mysql.com/bug.php?id=112242 + public static boolean bug112242 = true; + + // https://bugs.mysql.com/bug.php?id=112243 + public static boolean bug112243 = true; + + // https://bugs.mysql.com/bug.php?id=112264 + public static boolean bug112264 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 4e42e8b00..87408c39f 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -12,6 +12,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; import sqlancer.mysql.oracle.MySQLCERTOracle; +import sqlancer.mysql.oracle.MySQLDQPOracle; import sqlancer.mysql.oracle.MySQLFuzzer; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @@ -65,6 +66,12 @@ public TestOracle create(MySQLGlobalState globalState) throws return new MySQLFuzzer(globalState); } + }, + DQP { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLDQPOracle(globalState); + } }; } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index b61e501e5..757232406 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -38,6 +38,11 @@ public void visitSpecific(MySQLExpression expr) { @Override public void visit(MySQLSelect s) { sb.append("SELECT "); + if (s.getHint() != null) { + sb.append("/*+ "); + visit(s.getHint()); + sb.append("*/ "); + } switch (s.getFromOptions()) { case DISTINCT: sb.append("DISTINCT "); diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index 7abd1f639..a50007417 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -9,6 +9,7 @@ public class MySQLSelect extends SelectBase implements MySQLExp private SelectType fromOptions = SelectType.ALL; private List modifiers = Collections.emptyList(); + private MySQLText hint; public enum SelectType { DISTINCT, ALL, DISTINCTROW; @@ -39,4 +40,12 @@ public MySQLConstant getExpectedValue() { return null; } + public void setHint(MySQLText hint) { + this.hint = hint; + } + + public MySQLText getHint() { + return hint; + } + } diff --git a/src/sqlancer/mysql/gen/MySQLHintGenerator.java b/src/sqlancer/mysql/gen/MySQLHintGenerator.java new file mode 100644 index 000000000..141aea279 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLHintGenerator.java @@ -0,0 +1,204 @@ +package sqlancer.mysql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.mysql.MySQLSchema.MySQLIndex; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLText; + +public class MySQLHintGenerator { + + private final MySQLSelect select; + private final List tables; + private final StringBuilder sb = new StringBuilder(); + + enum OptimizeHint { + BKA, NO_BKA, BNL, NO_BNL, DERIVED_CONDITION_PUSHDOWN, NO_DERIVED_CONDITION_PUSHDOWN, GROUP_INDEX, + NO_GROUP_INDEX, HASH_JOIN, NO_HASH_JOIN, INDEX, NO_INDEX, INDEX_MERGE, NO_INDEX_MERGE, JOIN_FIXED_ORDER, + JOIN_INDEX, NO_JOIN_INDEX, JOIN_ORDER, JOIN_PREFIX, JOIN_SUFFIX, MERGE, NO_MERGE, MRR, NO_MRR, NO_ICP, + NO_RANGE_OPTIMIZATION, ORDER_INDEX, NO_ORDER_INDEX, SEMIJOIN, NO_SEMIJOIN, SKIP_SCAN, NO_SKIP_SCAN + } + + public MySQLHintGenerator(MySQLSelect select, List tables) { + this.select = select; + this.tables = tables; + } + + public static void generateHints(MySQLSelect select, List tables) { + new MySQLHintGenerator(select, tables).randomHint(); + } + + public static List generateAllHints(MySQLSelect select, List tables) { + MySQLHintGenerator generator = new MySQLHintGenerator(select, tables); + return generator.allHints(); + } + + private void randomHint() { + OptimizeHint chosenhint = Randomly.fromOptions(OptimizeHint.values()); + generate(chosenhint); + } + + private List allHints() { + List results = new ArrayList<>(); + for (OptimizeHint hint : OptimizeHint.values()) { + try { + MySQLText generatedHint = generate(hint); + results.add(generatedHint); + } catch (IgnoreMeException e) { + continue; + } + } + return results; + } + + private MySQLText generate(OptimizeHint chosenhint) { + sb.setLength(0); + + switch (chosenhint) { + case BKA: + tablesHint("BKA"); + break; + case NO_BKA: + tablesHint("NO_BKA"); + break; + case BNL: + tablesHint("BNL"); + break; + case NO_BNL: + tablesHint("NO_BNL"); + break; + case DERIVED_CONDITION_PUSHDOWN: + tablesHint("DERIVED_CONDITION_PUSHDOWN"); + break; + case NO_DERIVED_CONDITION_PUSHDOWN: + tablesHint("NO_DERIVED_CONDITION_PUSHDOWN"); + break; + case GROUP_INDEX: + indexesHint("GROUP_INDEX"); + break; + case NO_GROUP_INDEX: + indexesHint("NO_GROUP_INDEX"); + break; + case HASH_JOIN: + tablesHint("HASH_JOIN"); + break; + case NO_HASH_JOIN: + tablesHint("NO_HASH_JOIN"); + break; + case INDEX: + indexesHint("INDEX"); + break; + case NO_INDEX: + indexesHint("NO_INDEX"); + break; + case INDEX_MERGE: + indexesHint("INDEX_MERGE"); + break; + case NO_INDEX_MERGE: + indexesHint("NO_INDEX_MERGE"); + break; + case JOIN_FIXED_ORDER: + tablesHint("JOIN_FIXED_ORDER"); + break; + case JOIN_INDEX: + indexesHint("JOIN_INDEX"); + break; + case NO_JOIN_INDEX: + indexesHint("NO_JOIN_INDEX"); + break; + case JOIN_ORDER: + tablesHint("JOIN_ORDER"); + break; + case JOIN_PREFIX: + tablesHint("JOIN_PREFIX"); + break; + case JOIN_SUFFIX: + tablesHint("JOIN_SUFFIX"); + break; + case MERGE: + tablesHint("MERGE"); + break; + case NO_MERGE: + tablesHint("NO_MERGE"); + break; + case MRR: + indexesHint("MRR"); + break; + case NO_MRR: + indexesHint("NO_MRR"); + break; + case NO_ICP: + indexesHint("NO_ICP"); + break; + case NO_RANGE_OPTIMIZATION: + indexesHint("NO_RANGE_OPTIMIZATION"); + break; + case ORDER_INDEX: + indexesHint("ORDER_INDEX"); + break; + case NO_ORDER_INDEX: + indexesHint("NO_ORDER_INDEX"); + break; + case SEMIJOIN: + semiHint("SEMIJOIN"); + break; + case NO_SEMIJOIN: + semiHint("NO_SEMIJOIN"); + break; + case SKIP_SCAN: + indexesHint("SKIP_SCAN"); + break; + case NO_SKIP_SCAN: + indexesHint("NO_SKIP_SCAN"); + break; + default: + throw new AssertionError(); + } + MySQLText hint = new MySQLText(sb.toString()); + select.setHint(hint); + return hint; + } + + private void indexesHint(String string) { + sb.append(string); + sb.append("("); + MySQLTable table = Randomly.fromList(tables); + List allIndexes = table.getIndexes(); + sb.append(table.getName()); + sb.append(", "); + if (allIndexes.isEmpty()) { + sb.append("PRIMARY"); + } else { + List indexSubset = Randomly.nonEmptySubset(allIndexes); + sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); + } + sb.append(")"); + } + + private void tablesHint(String string) { + sb.append(string); + sb.append("("); + appendTables(); + sb.append(")"); + } + + private void semiHint(String string) { + sb.append(string); + sb.append("("); + String[] options = { "DUPSWEEDOUT", "FIRSTMATCH", "LOOSESCAN", "MATERIALIZATION" }; + List chosenOptions = Randomly.nonEmptySubset(options); + sb.append(chosenOptions.stream().collect(Collectors.joining(", "))); + sb.append(")"); + } + + private void appendTables() { + List tableSubset = Randomly.nonEmptySubset(tables); + sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + } + +} diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 821184617..94db1fab7 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -1,5 +1,7 @@ package sqlancer.mysql.gen; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; @@ -7,6 +9,7 @@ import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; public class MySQLSetGenerator { @@ -131,11 +134,13 @@ private enum Action { private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); - String[] options = { "batched_key_access", "block_nested_loop", "condition_fanout_filter", "derived_merge", - "engine_condition_pushdown", "index_condition_pushdown", "use_index_extensions", "index_merge", - "index_merge_intersection", "index_merge_sort_union", "index_merge_union", "use_invisible_indexes", - "mrr", "mrr_cost_based", "skip_scan", "semijoin", "duplicateweedout", "firstmatch", "loosescan", - "materialization", "subquery_materialization_cost_based" }; + String[] options = { "index_merge", "index_merge_union", "index_merge_sort_union", + "index_merge_intersection", "engine_condition_pushdown", "index_condition_pushdown", "mrr", + "mrr_cost_based", "block_nested_loop", "batched_key_access", "materialization", "semijoin", + "loosescan", "firstmatch", "duplicateweedout", "subquery_materialization_cost_based", + "use_index_extensions", "condition_fanout_filter", "derived_merge", "use_invisible_indexes", + "skip_scan", "hash_join", "subquery_to_derived", "prefer_ordering_index", + "derived_condition_pushdown" }; List optionSubset = Randomly.nonEmptySubset(options); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); @@ -188,4 +193,46 @@ private SQLQueryAdapter get() { return new SQLQueryAdapter(sb.toString()); } + public static SQLQueryAdapter resetOptimizer() { + return new SQLQueryAdapter("SET optimizer_switch='default'"); + } + + public static List getAllOptimizer(MySQLGlobalState globalState) { + List result = new ArrayList<>(); + String[] options = { "index_merge", "index_merge_union", "index_merge_sort_union", "index_merge_intersection", + "engine_condition_pushdown", "index_condition_pushdown", "mrr", "mrr_cost_based", "block_nested_loop", + "batched_key_access", "materialization", "semijoin", "loosescan", "firstmatch", "duplicateweedout", + "subquery_materialization_cost_based", "use_index_extensions", "condition_fanout_filter", + "derived_merge", "use_invisible_indexes", "skip_scan", "hash_join", "subquery_to_derived", + "prefer_ordering_index", "derived_condition_pushdown" }; + + List availableOptions = new ArrayList<>(Arrays.asList(options)); + if (MySQLBugs.bug112242) { + availableOptions.remove("use_invisible_indexes"); + } + if (MySQLBugs.bug112243) { + availableOptions.remove("subquery_to_derived"); + } + if (MySQLBugs.bug112264) { + availableOptions.remove("block_nested_loop"); + } + + StringBuilder sb = new StringBuilder(); + sb.append("SET "); + if (globalState.getOptions().getNumberConcurrentThreads() == 1 && Randomly.getBoolean()) { + sb.append("GLOBAL"); + } else { + sb.append("SESSION"); + } + sb.append(" optimizer_switch = '%s'"); + + for (String option : availableOptions) { + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=on"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=off"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=default"))); + } + + return result; + } + } diff --git a/src/sqlancer/mysql/oracle/MySQLDQPOracle.java b/src/sqlancer/mysql/oracle/MySQLDQPOracle.java new file mode 100644 index 000000000..414ffb156 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLDQPOracle.java @@ -0,0 +1,104 @@ +package sqlancer.mysql.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLJoin; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; +import sqlancer.mysql.gen.MySQLExpressionGenerator; +import sqlancer.mysql.gen.MySQLHintGenerator; +import sqlancer.mysql.gen.MySQLSetGenerator; + +public class MySQLDQPOracle implements TestOracle { + private final MySQLGlobalState state; + private MySQLExpressionGenerator gen; + private MySQLSelect select; + private final ExpectedErrors errors = new ExpectedErrors(); + + public MySQLDQPOracle(MySQLGlobalState globalState) { + state = globalState; + MySQLErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws Exception { + // Randomly generate a query + MySQLTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new MySQLExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList())); + + select = new MySQLSelect(); + select.setFetchColumns(fetchColumns); + + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateExpression()); + } + } + + // Set the join. + List joinExpressions = MySQLJoin.getRandomJoinClauses(tables.getTables(), state); + select.setJoinList(joinExpressions.stream().map(j -> (MySQLExpression) j).collect(Collectors.toList())); + + // Set the from clause from the tables that are not used in the join. + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) + .collect(Collectors.toList()); + select.setFromList(tableList); + + // Get the result of the first query + String originalQueryString = MySQLVisitor.asString(select); + List originalResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + // Check hints + List hintList = MySQLHintGenerator.generateAllHints(select, tables.getTables()); + for (MySQLText hint : hintList) { + select.setHint(hint); + String queryString = MySQLVisitor.asString(select); + List result = ComparatorHelper.getResultSetFirstColumnAsString(queryString, errors, state); + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, List.of(queryString), + state); + } + + // Check optimizer variables + List optimizationList = MySQLSetGenerator.getAllOptimizer(state); + for (SQLQueryAdapter optimization : optimizationList) { + optimization.execute(state); + List result = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, + List.of(originalQueryString), state); + } catch (AssertionError e) { + String assertionMessage = String.format( + "The size of the result sets mismatch (%d and %d)!" + System.lineSeparator() + + "First query: \"%s\", whose cardinality is: %d" + System.lineSeparator() + + "Second query:\"%s\", whose cardinality is: %d", + originalResult.size(), result.size(), originalQueryString, originalResult.size(), + String.join(";", originalQueryString), result.size()); + assertionMessage += System.lineSeparator() + "The setting: " + optimization.getQueryString(); + throw new AssertionError(assertionMessage); + } + } + } +} diff --git a/test/sqlancer/dqp/mysql/TestMySQLDQP.java b/test/sqlancer/dqp/mysql/TestMySQLDQP.java new file mode 100644 index 000000000..38e5eef61 --- /dev/null +++ b/test/sqlancer/dqp/mysql/TestMySQLDQP.java @@ -0,0 +1,22 @@ +package sqlancer.dqp.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestMySQLDQP { + + @Test + public void testmysqlQPG() { + String mysql = System.getenv("MYSQL_AVAILABLE"); + boolean mysqlIsAvailable = mysql != null && mysql.equalsIgnoreCase("true"); + assumeTrue(mysqlIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "mysql", "--oracle", "DQP" })); + } + +} From afc6792f1324b0675e3fe8bc879adc35e793fe26 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 19 Mar 2024 20:23:28 +0800 Subject: [PATCH 0835/1284] MariaDB Join implementation (#924) --- src/sqlancer/mariadb/MariaDBSchema.java | 4 + .../mariadb/ast/MariaDBAggregate.java | 2 +- .../mariadb/ast/MariaDBBinaryOperator.java | 2 +- .../mariadb/ast/MariaDBColumnName.java | 2 +- src/sqlancer/mariadb/ast/MariaDBConstant.java | 5 +- .../mariadb/ast/MariaDBExpression.java | 2 +- src/sqlancer/mariadb/ast/MariaDBFunction.java | 2 +- .../mariadb/ast/MariaDBInOperation.java | 2 +- src/sqlancer/mariadb/ast/MariaDBJoin.java | 86 +++++++++++++++++++ .../ast/MariaDBPostfixUnaryOperation.java | 2 +- .../mariadb/ast/MariaDBSelectStatement.java | 7 +- .../mariadb/ast/MariaDBStringVisitor.java | 40 ++++++++- src/sqlancer/mariadb/ast/MariaDBText.java | 2 +- .../ast/MariaDBUnaryPrefixOperation.java | 2 +- src/sqlancer/mariadb/ast/MariaDBVisitor.java | 4 + 15 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 src/sqlancer/mariadb/ast/MariaDBJoin.java diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index ed432e8da..d1bcfbd97 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -242,4 +242,8 @@ public MariaDBSchema(List databaseTables) { super(databaseTables); } + public MariaDBTables getRandomTableNonEmptyTables() { + return new MariaDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + } diff --git a/src/sqlancer/mariadb/ast/MariaDBAggregate.java b/src/sqlancer/mariadb/ast/MariaDBAggregate.java index 3977a6a95..ab5a47781 100644 --- a/src/sqlancer/mariadb/ast/MariaDBAggregate.java +++ b/src/sqlancer/mariadb/ast/MariaDBAggregate.java @@ -1,6 +1,6 @@ package sqlancer.mariadb.ast; -public class MariaDBAggregate extends MariaDBExpression { +public class MariaDBAggregate implements MariaDBExpression { private final MariaDBExpression expr; private final MariaDBAggregateFunction aggr; diff --git a/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java b/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java index 14f3c308a..5f56d168f 100644 --- a/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java +++ b/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class MariaDBBinaryOperator extends MariaDBExpression { +public class MariaDBBinaryOperator implements MariaDBExpression { private MariaDBExpression left; private MariaDBExpression right; diff --git a/src/sqlancer/mariadb/ast/MariaDBColumnName.java b/src/sqlancer/mariadb/ast/MariaDBColumnName.java index a4fd78c1b..65d4cea0c 100644 --- a/src/sqlancer/mariadb/ast/MariaDBColumnName.java +++ b/src/sqlancer/mariadb/ast/MariaDBColumnName.java @@ -2,7 +2,7 @@ import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; -public class MariaDBColumnName extends MariaDBExpression { +public class MariaDBColumnName implements MariaDBExpression { private final MariaDBColumn column; diff --git a/src/sqlancer/mariadb/ast/MariaDBConstant.java b/src/sqlancer/mariadb/ast/MariaDBConstant.java index 6e3d77691..8670c9ed2 100644 --- a/src/sqlancer/mariadb/ast/MariaDBConstant.java +++ b/src/sqlancer/mariadb/ast/MariaDBConstant.java @@ -1,6 +1,9 @@ package sqlancer.mariadb.ast; -public class MariaDBConstant extends MariaDBExpression { +public class MariaDBConstant implements MariaDBExpression { + + private MariaDBConstant() { + } public static class MariaDBNullConstant extends MariaDBConstant { diff --git a/src/sqlancer/mariadb/ast/MariaDBExpression.java b/src/sqlancer/mariadb/ast/MariaDBExpression.java index beff58866..5caec1b40 100644 --- a/src/sqlancer/mariadb/ast/MariaDBExpression.java +++ b/src/sqlancer/mariadb/ast/MariaDBExpression.java @@ -1,5 +1,5 @@ package sqlancer.mariadb.ast; -public class MariaDBExpression { +public interface MariaDBExpression { } diff --git a/src/sqlancer/mariadb/ast/MariaDBFunction.java b/src/sqlancer/mariadb/ast/MariaDBFunction.java index fd033d310..481a26aaf 100644 --- a/src/sqlancer/mariadb/ast/MariaDBFunction.java +++ b/src/sqlancer/mariadb/ast/MariaDBFunction.java @@ -2,7 +2,7 @@ import java.util.List; -public class MariaDBFunction extends MariaDBExpression { +public class MariaDBFunction implements MariaDBExpression { private final MariaDBFunctionName func; private final List args; diff --git a/src/sqlancer/mariadb/ast/MariaDBInOperation.java b/src/sqlancer/mariadb/ast/MariaDBInOperation.java index 85981b390..15aad4daa 100644 --- a/src/sqlancer/mariadb/ast/MariaDBInOperation.java +++ b/src/sqlancer/mariadb/ast/MariaDBInOperation.java @@ -2,7 +2,7 @@ import java.util.List; -public class MariaDBInOperation extends MariaDBExpression { +public class MariaDBInOperation implements MariaDBExpression { private final MariaDBExpression expr; private final List list; diff --git a/src/sqlancer/mariadb/ast/MariaDBJoin.java b/src/sqlancer/mariadb/ast/MariaDBJoin.java new file mode 100644 index 000000000..b8110b7f2 --- /dev/null +++ b/src/sqlancer/mariadb/ast/MariaDBJoin.java @@ -0,0 +1,86 @@ +package sqlancer.mariadb.ast; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; +import sqlancer.mariadb.gen.MariaDBExpressionGenerator; + +public class MariaDBJoin implements MariaDBExpression { + + public enum JoinType { + NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; + } + + private final MariaDBTable table; + private MariaDBExpression onClause; + private JoinType type; + + public MariaDBJoin(MariaDBJoin other) { + this.table = other.table; + this.onClause = other.onClause; + this.type = other.type; + } + + public MariaDBJoin(MariaDBTable table, MariaDBExpression onClause, JoinType type) { + this.table = table; + this.onClause = onClause; + this.type = type; + } + + public MariaDBTable getTable() { + return table; + } + + public MariaDBExpression getOnClause() { + return onClause; + } + + public JoinType getType() { + return type; + } + + public void setOnClause(MariaDBExpression onClause) { + this.onClause = onClause; + } + + public void setType(JoinType type) { + this.type = type; + } + + public static List getRandomJoinClauses(List tables, MariaDBGlobalState globalState) { + List joinStatements = new ArrayList<>(); + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + List columns = new ArrayList<>(); + if (tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + MariaDBTable table = Randomly.fromList(tables); + tables.remove(table); + columns.addAll(table.getColumns()); + MariaDBExpressionGenerator joinGen = new MariaDBExpressionGenerator(globalState.getRandomly()) + .setColumns(columns); + MariaDBExpression joinClause = joinGen.getRandomExpression(); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + joinClause = null; + } + MariaDBJoin j = new MariaDBJoin(table, joinClause, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } +} diff --git a/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java b/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java index cd655a5d1..9b56a44dd 100644 --- a/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java +++ b/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class MariaDBPostfixUnaryOperation extends MariaDBExpression { +public class MariaDBPostfixUnaryOperation implements MariaDBExpression { private MariaDBPostfixUnaryOperator operator; private MariaDBExpression randomWhereCondition; diff --git a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java index d1700f508..83b944a17 100644 --- a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java +++ b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java @@ -3,12 +3,13 @@ import java.util.ArrayList; import java.util.List; +import sqlancer.common.ast.SelectBase; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -public class MariaDBSelectStatement extends MariaDBExpression { +public class MariaDBSelectStatement extends SelectBase implements MariaDBExpression { public enum MariaDBSelectType { - ALL + ALL, DISTINCT, DISTINCTROW; } private List groupBys = new ArrayList<>(); @@ -21,6 +22,7 @@ public void setGroupByClause(List groupBys) { this.groupBys = groupBys; } + @Override public void setFetchColumns(List columns) { this.columns = columns; @@ -34,6 +36,7 @@ public void setSelectType(MariaDBSelectType selectType) { this.selectType = selectType; } + @Override public void setWhereClause(MariaDBExpression whereCondition) { this.whereCondition = whereCondition; } diff --git a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java index 53ed490d0..07951d9ac 100644 --- a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java @@ -27,7 +27,7 @@ public void visit(MariaDBPostfixUnaryOperation op) { @Override public void visit(MariaDBColumnName c) { - sb.append(c.getColumn().getName()); + sb.append(c.getColumn().getFullQualifiedName()); } @Override @@ -42,12 +42,16 @@ public void visit(MariaDBSelectStatement s) { } sb.append(" FROM "); sb.append(s.getTables().stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + + for (MariaDBExpression j : s.getJoinList()) { + visit(j); + } if (s.getWhereCondition() != null) { sb.append(" WHERE "); visit(s.getWhereCondition()); } if (s.getGroupBys().size() != 0) { - sb.append(" GROUP BY"); + sb.append(" GROUP BY "); for (i = 0; i < s.getGroupBys().size(); i++) { if (i != 0) { sb.append(", "); @@ -131,4 +135,36 @@ private void visitList(List list) { } } + @Override + public void visit(MariaDBJoin join) { + sb.append(" "); + switch (join.getType()) { + case NATURAL: + sb.append("NATURAL "); + break; + case INNER: + sb.append("INNER "); + break; + case STRAIGHT: + sb.append("STRAIGHT_"); + break; + case LEFT: + sb.append("LEFT "); + break; + case RIGHT: + sb.append("RIGHT "); + break; + case CROSS: + sb.append("CROSS "); + break; + default: + throw new AssertionError(join.getType()); + } + sb.append("JOIN "); + sb.append(join.getTable().getName()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } } diff --git a/src/sqlancer/mariadb/ast/MariaDBText.java b/src/sqlancer/mariadb/ast/MariaDBText.java index b96871063..c3d1c2d1a 100644 --- a/src/sqlancer/mariadb/ast/MariaDBText.java +++ b/src/sqlancer/mariadb/ast/MariaDBText.java @@ -1,6 +1,6 @@ package sqlancer.mariadb.ast; -public class MariaDBText extends MariaDBExpression { +public class MariaDBText implements MariaDBExpression { private final MariaDBExpression expr; private final String text; diff --git a/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java b/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java index af229ce6e..dda61dbee 100644 --- a/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java +++ b/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class MariaDBUnaryPrefixOperation extends MariaDBExpression { +public class MariaDBUnaryPrefixOperation implements MariaDBExpression { private MariaDBExpression expr; private MariaDBUnaryPrefixOperator op; diff --git a/src/sqlancer/mariadb/ast/MariaDBVisitor.java b/src/sqlancer/mariadb/ast/MariaDBVisitor.java index 8ea3f9d37..af94394b9 100644 --- a/src/sqlancer/mariadb/ast/MariaDBVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBVisitor.java @@ -22,6 +22,8 @@ public abstract class MariaDBVisitor { public abstract void visit(MariaDBInOperation op); + public abstract void visit(MariaDBJoin join); + public void visit(MariaDBExpression expr) { if (expr instanceof MariaDBConstant) { visit((MariaDBConstant) expr); @@ -43,6 +45,8 @@ public void visit(MariaDBExpression expr) { visit((MariaDBFunction) expr); } else if (expr instanceof MariaDBInOperation) { visit((MariaDBInOperation) expr); + } else if (expr instanceof MariaDBJoin) { + visit((MariaDBJoin) expr); } else { throw new AssertionError(expr.getClass()); } From 094756668c151084cf567e401aa8cca83370555d Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 19 Mar 2024 21:20:55 +0800 Subject: [PATCH 0836/1284] MariaDB DQP (#925) --- src/sqlancer/mariadb/MariaDBBugs.java | 21 +++++ src/sqlancer/mariadb/MariaDBErrors.java | 17 ++++ src/sqlancer/mariadb/MariaDBOptions.java | 7 ++ src/sqlancer/mariadb/MariaDBProvider.java | 2 +- .../mariadb/gen/MariaDBIndexGenerator.java | 1 + .../mariadb/gen/MariaDBSetGenerator.java | 92 ++++++++++++++----- .../MariaDBTableAdminCommandGenerator.java | 3 +- .../mariadb/gen/MariaDBTruncateGenerator.java | 6 +- .../mariadb/oracle/MariaDBDQPOracle.java | 91 ++++++++++++++++++ test/sqlancer/dqp/mariadb/TestMariaDBDQP.java | 22 +++++ 10 files changed, 236 insertions(+), 26 deletions(-) create mode 100644 src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java create mode 100644 test/sqlancer/dqp/mariadb/TestMariaDBDQP.java diff --git a/src/sqlancer/mariadb/MariaDBBugs.java b/src/sqlancer/mariadb/MariaDBBugs.java index 7a568b8fe..fb1770763 100644 --- a/src/sqlancer/mariadb/MariaDBBugs.java +++ b/src/sqlancer/mariadb/MariaDBBugs.java @@ -5,6 +5,27 @@ public final class MariaDBBugs { // https://jira.mariadb.org/browse/MDEV-21058 public static boolean bug21058 = true; + // https://jira.mariadb.org/browse/MDEV-32076 + public static boolean bug32076 = true; + + // https://jira.mariadb.org/browse/MDEV-32099 + public static boolean bug32099 = true; + + // https://jira.mariadb.org/browse/MDEV-32105 + public static boolean bug32105 = true; + + // https://jira.mariadb.org/browse/MDEV-32106 + public static boolean bug32106 = true; + + // https://jira.mariadb.org/browse/MDEV-32107 + public static boolean bug32107 = true; + + // https://jira.mariadb.org/browse/MDEV-32108 + public static boolean bug32108 = true; + + // https://jira.mariadb.org/browse/MDEV-32143 + public static boolean bug32143 = true; + private MariaDBBugs() { } diff --git a/src/sqlancer/mariadb/MariaDBErrors.java b/src/sqlancer/mariadb/MariaDBErrors.java index ab1bec4cd..4918accfb 100644 --- a/src/sqlancer/mariadb/MariaDBErrors.java +++ b/src/sqlancer/mariadb/MariaDBErrors.java @@ -7,6 +7,23 @@ public final class MariaDBErrors { private MariaDBErrors() { } + public static void addCommonErrors(ExpectedErrors errors) { + errors.add("is out of range"); + // regex + errors.add("unmatched parentheses"); + errors.add("nothing to repeat at offset"); + errors.add("missing )"); + errors.add("missing terminating ]"); + errors.add("range out of order in character class"); + errors.add("unrecognized character after "); + errors.add("Got error '(*VERB) not recognized or malformed"); + errors.add("must be followed by"); + errors.add("malformed number or name after"); + errors.add("digit expected after"); + errors.add("Regex error"); + errors.add("Lock wait timeout exceeded"); + } + public static void addInsertErrors(ExpectedErrors errors) { errors.add("Out of range"); errors.add("Duplicate entry"); // violates UNIQUE constraint diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 70e0aedbd..f601ab0a6 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -12,6 +12,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.mariadb.MariaDBOptions.MariaDBOracleFactory; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.oracle.MariaDBDQPOracle; import sqlancer.mariadb.oracle.MariaDBNoRECOracle; @Parameters(separators = "=", commandDescription = "MariaDB (default port: " + MariaDBOptions.DEFAULT_PORT @@ -32,6 +33,12 @@ public TestOracle create(MariaDBGlobalState globalState) thr return new MariaDBNoRECOracle(globalState); } + }, + DQP { + @Override + public TestOracle create(MariaDBGlobalState globalState) throws SQLException { + return new MariaDBDQPOracle(globalState); + } } } diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index c84691203..48254b8ec 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -53,7 +53,7 @@ enum Action { public void generateDatabase(MariaDBGlobalState globalState) throws Exception { MainOptions options = globalState.getOptions(); - while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.getNotCachedInteger(1, 3)) { String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); SQLQueryAdapter createTable = MariaDBTableGenerator.generate(tableName, globalState.getRandomly(), globalState.getSchema()); diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 19519ca2e..1ba3fbd4d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -19,6 +19,7 @@ public static SQLQueryAdapter generate(MariaDBSchema s) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE "); errors.add("Key/Index cannot be defined on a virtual generated column"); + errors.add("Specified key was too long"); if (Randomly.getBoolean()) { errors.add("Duplicate entry"); errors.add("Key/Index cannot be defined on a virtual generated column"); diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index f4caf0ec6..fc1f1d5a6 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -1,5 +1,6 @@ package sqlancer.mariadb.gen; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Function; @@ -9,6 +10,8 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mariadb.MariaDBBugs; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; public class MariaDBSetGenerator { @@ -41,7 +44,6 @@ private enum Action { // Scope.SESSION), CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", "0", "1", "2"), Scope.GLOBAL), - CTE_MAX_RECURSION_DEPTH("cte_max_recursion_depth", (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), DELAY_KEY_WRITE("delay_key_write", (r) -> Randomly.fromOptions("ON", "OFF", "ALL"), Scope.GLOBAL), EQ_RANGE_INDEX_DIVE_LIMIT("eq_range_index_dive_limit", (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), FLUSH("flush", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL), @@ -86,7 +88,6 @@ private enum Action { */ // READ_BUFFER_SIZE("read_buffer_size", (r) -> r.getLong(8200, 2147479552), Scope.GLOBAL, Scope.SESSION), // READ_RND_BUFFER_SIZE("read_rnd_buffer_size", (r) -> r.getLong(1, 2147483647), Scope.GLOBAL, Scope.SESSION), - SCHEMA_DEFINITION_CACHE("schema_definition_cache", (r) -> r.getLong(256, 524288), Scope.GLOBAL), /* * sort_buffer_size is commented out as a workaround for https://bugs.mysql.com/bug.php?id=95969 */ @@ -120,27 +121,16 @@ private enum Action { private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); - String[] options = { /* - * ("batched_key_access", /*"block_nested_loop", "condition_fanout_filter", - */ - "condition_pushdown_for_derived", // MariaDB - "derived_merge", // - "derived_with_keys", // MariaDB - "engine_condition_pushdown", // - "exists_to_in", // MariaDB - "extended_keys", // MariaDB - "firstmatch", // MariaDB - "index_condition_pushdown", // - /* "use_index_extensions", */ - "index_merge", // - "index_merge_intersection", // - "index_merge_sort_intersection", // - "index_merge_sort_union", // - "index_merge_union", "in_to_exists", // MariaDB - /* "use_invisible_indexes", */ "mrr", "mrr_cost_based", /* "skip_scan", */ "semijoin", /* - * "duplicateweedout", - */ - "firstmatch", "loosescan", "materialization", /* "subquery_materialization_cost_based" */ }; + String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "engine_condition_pushdown", + "exists_to_in", "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", + "index_merge", "index_merge_intersection", "index_merge_sort_intersection", + "index_merge_sort_union", "index_merge_union", "in_to_exists", "join_cache_bka", + "join_cache_hashed", "join_cache_incremental", "loosescan", "materialization", "mrr", + "mrr_cost_based", "mrr_sort_keys", "not_null_range_scan", "optimize_join_buffer_size", + "orderby_uses_equalities", "outer_join_with_cache", "partial_match_rowid_merge", + "partial_match_table_scan", "rowid_filter", "semijoin", "semijoin_with_cache", "split_materialized", + "subquery_cache", "table_elimination" }; List optionSubset = Arrays.asList(Randomly.fromOptions(options)); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); @@ -194,4 +184,60 @@ private SQLQueryAdapter get() { .from("At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'")); } + public static SQLQueryAdapter resetOptimizer() { + return new SQLQueryAdapter("SET optimizer_switch='default'"); + } + + public static List getAllOptimizer(MariaDBGlobalState globalState) { + List result = new ArrayList<>(); + String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "engine_condition_pushdown", + "exists_to_in", "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", + "index_merge", "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", + "index_merge_union", "in_to_exists", "join_cache_bka", "join_cache_hashed", "join_cache_incremental", + "loosescan", "materialization", "mrr", "mrr_cost_based", "mrr_sort_keys", "not_null_range_scan", + "optimize_join_buffer_size", "orderby_uses_equalities", "outer_join_with_cache", + "partial_match_rowid_merge", "partial_match_table_scan", "rowid_filter", "semijoin", + "semijoin_with_cache", "split_materialized", "subquery_cache", "table_elimination" }; + List availableOptions = new ArrayList<>(Arrays.asList(options)); + if (MariaDBBugs.bug21058) { + availableOptions.remove("in_to_exists"); // https://jira.mariadb.org/browse/MDEV-21058 + } + if (MariaDBBugs.bug32076) { + availableOptions.remove("not_null_range_scan"); // https://jira.mariadb.org/browse/MDEV-32076 + } + if (MariaDBBugs.bug32099) { + availableOptions.remove("optimize_join_buffer_size"); // https://jira.mariadb.org/browse/MDEV-32099 + } + if (MariaDBBugs.bug32105) { + availableOptions.remove("join_cache_hashed"); // https://jira.mariadb.org/browse/MDEV-32105 + } + if (MariaDBBugs.bug32106) { + availableOptions.remove("outer_join_with_cache"); // https://jira.mariadb.org/browse/MDEV-32106 + } + if (MariaDBBugs.bug32107) { + availableOptions.remove("table_elimination"); // https://jira.mariadb.org/browse/MDEV-32107 + } + if (MariaDBBugs.bug32108) { + availableOptions.remove("join_cache_incremental"); // https://jira.mariadb.org/browse/MDEV-32108 + } + if (MariaDBBugs.bug32143) { + availableOptions.remove("mrr"); // https://jira.mariadb.org/browse/MDEV-32143 + } + + StringBuilder sb = new StringBuilder(); + sb.append("SET SESSION optimizer_switch = '%s'"); + + for (String option : availableOptions) { + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=on"), ExpectedErrors.from( + "At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=off"), ExpectedErrors.from( + "At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=default"), ExpectedErrors.from( + "At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'"))); + } + + return result; + } + } diff --git a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java index bff83e1e0..a141ca58d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java @@ -57,7 +57,8 @@ public static SQLQueryAdapter optimizeTable(MariaDBSchema newSchema) { MariaDBCommon.addWaitClause(sb); return checkForMsgText(sb, s -> s.equals("OK") || s.equals("Table does not support optimize, doing recreate + analyze instead") - || s.contentEquals("Table is already up to date")); + || s.contentEquals("Table is already up to date") || s.contains("Lock wait timeout") + || s.contains("Operation failed")); } private static SQLQueryAdapter checkForMsgText(StringBuilder sb, Function checker) { diff --git a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java index 04e567444..ecc240bb3 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java @@ -1,6 +1,8 @@ package sqlancer.mariadb.gen; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; public final class MariaDBTruncateGenerator { @@ -13,7 +15,9 @@ public static SQLQueryAdapter truncate(MariaDBSchema s) { sb.append(s.getRandomTable().getName()); sb.append(" "); MariaDBCommon.addWaitClause(sb); - return new SQLQueryAdapter(sb.toString()); + ExpectedErrors errors = new ExpectedErrors(); + MariaDBErrors.addCommonErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java new file mode 100644 index 000000000..a92be5d1e --- /dev/null +++ b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java @@ -0,0 +1,91 @@ +package sqlancer.mariadb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mariadb.MariaDBErrors; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.MariaDBSchema; +import sqlancer.mariadb.MariaDBSchema.MariaDBTables; +import sqlancer.mariadb.ast.MariaDBColumnName; +import sqlancer.mariadb.ast.MariaDBExpression; +import sqlancer.mariadb.ast.MariaDBJoin; +import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBVisitor; +import sqlancer.mariadb.gen.MariaDBExpressionGenerator; +import sqlancer.mariadb.gen.MariaDBSetGenerator; + +public class MariaDBDQPOracle implements TestOracle { + private final MariaDBGlobalState state; + private final MariaDBSchema s; + private MariaDBExpressionGenerator gen; + private MariaDBSelectStatement select; + private final ExpectedErrors errors = new ExpectedErrors(); + + public MariaDBDQPOracle(MariaDBGlobalState globalState) { + state = globalState; + s = globalState.getSchema(); + MariaDBErrors.addCommonErrors(errors); + } + + @Override + public void check() throws Exception { + MariaDBTables tables = s.getRandomTableNonEmptyTables(); + gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(tables.getColumns()) + .setCon(state.getConnection()).setState(state.getState()); + + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new MariaDBColumnName(c)) + .collect(Collectors.toList())); + + select = new MariaDBSelectStatement(); + select.setFetchColumns(fetchColumns); + + select.setSelectType(Randomly.fromOptions(MariaDBSelectStatement.MariaDBSelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.getRandomExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByClause(fetchColumns); + } + + // Set the join. + List joinExpressions = MariaDBJoin.getRandomJoinClauses(tables.getTables(), state); + select.setJoinList(joinExpressions.stream().map(j -> (MariaDBExpression) j).collect(Collectors.toList())); + + // Set the from clause from the tables that are not used in the join. + select.setFromTables(tables.getTables()); + + // Get the result of the first query + String originalQueryString = MariaDBVisitor.asString(select); + List originalResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + List optimizationList = MariaDBSetGenerator.getAllOptimizer(state); + for (SQLQueryAdapter optimization : optimizationList) { + optimization.execute(state); + List result = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, + List.of(originalQueryString), state); + } catch (AssertionError e) { + String assertionMessage = String.format( + "The size of the result sets mismatch (%d and %d)!" + System.lineSeparator() + + "First query: \"%s\", whose cardinality is: %d" + System.lineSeparator() + + "Second query:\"%s\", whose cardinality is: %d", + originalResult.size(), result.size(), originalQueryString, originalResult.size(), + String.join(";", originalQueryString), result.size()); + assertionMessage += System.lineSeparator() + "The setting: " + optimization.getQueryString(); + throw new AssertionError(assertionMessage); + } + } + + } + +} diff --git a/test/sqlancer/dqp/mariadb/TestMariaDBDQP.java b/test/sqlancer/dqp/mariadb/TestMariaDBDQP.java new file mode 100644 index 000000000..47dabc225 --- /dev/null +++ b/test/sqlancer/dqp/mariadb/TestMariaDBDQP.java @@ -0,0 +1,22 @@ +package sqlancer.dqp.mariadb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestMariaDBDQP { + + @Test + public void testMariaDBDQPMethod() { + String mariadb = System.getenv("MARIADB_AVAILABLE"); + boolean mariadbIsAvailable = mariadb != null && mariadb.equalsIgnoreCase("true"); + assumeTrue(mariadbIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "mariadb", "--oracle", "DQP" })); + } + +} From 94b3782db7a434772bd0dbea606b12beeef1fca5 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Wed, 20 Mar 2024 10:56:38 +0800 Subject: [PATCH 0837/1284] add doc (#926) --- README.md | 5 +++-- docs/PAPERS.md | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d84a993b1..83e207485 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Man SQLancer operates in the following two phases: -1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. **News: we support Query Plan Guidance (QPG) now. See Generation Approaches below.** -2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. **News: we support Cardinality Estimation Restriction Testing (CERT) oracle now. See Testing Approaches below.** +1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. +2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. **News: we support Differential Query Plans (DQP) oracle now. See Testing Approaches below.** # Getting Started @@ -41,6 +41,7 @@ If you launch SQLancer without parameters, available options and commands are di | Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | | Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | | Cardinality Estimation Restriction Testing (CERT) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. It is described [here](https://arxiv.org/abs/2306.00355). It derives a query to a more restrict query, whose estimated cardinality should be no more than that for the original query. An violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. | +| Differential Query Plans (DQP) | DQP aims to find logic bugs in database systems by checking whether the query plans of the same query perform consistently. It is described [here](https://bajinsheng.github.io/assets/pdf/dqp_sigmod24.pdf). DQP supports MySQL, MariaDB, and TiDB.| # Generation Approaches | Approach | Description | diff --git a/docs/PAPERS.md b/docs/PAPERS.md index c0e13dd3e..a42b42c12 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -65,6 +65,37 @@ This paper describes Query Plan Guidance (QPG), a test case generation method gu } ``` +## CERT: Finding Performance Issues in Database Systems Through the Lens of Cardinality Estimation + +This paper describes CERT, a testing approach to find performance issues by inspecting inconsistent estimated cardinalities. A preprint is available [here](https://bajinsheng.github.io/assets/pdf/cert_icse24.pdf). + +``` +@inproceedings{cert, + author = {Ba, Jinsheng and Rigger, Manuel}, + title = {CERT: Finding Performance Issues in Database Systems Through the Lens of Cardinality Estimation}, + booktitle = {The 46th International Conference on Software Engineering (ICSE'24)}, + year = {2024}, + month = apr, +} +``` + +## Keep It Simple: Testing Databases via Differential Query Plans + +This paper describes DQP, a testing approach to find logic bugs in database systems by comparing the query plans of different database systems. A preprint is available [here](https://bajinsheng.github.io/assets/pdf/dqp_sigmod24.pdf). + +``` +@article{dqp, + author = {Ba, Jinsheng and Rigger, Manuel}, + title = {Keep It Simple: Testing Databases via Differential Query Plans}, + year = {2024}, + issue_date = {June 2024}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + journal = {Proceeding of ACM Management of Data (SIGMOD'24)}, + month = jun +} +``` + # Comparing SQLancer With Other Tools that Find Logic Bugs If you want to fairly compare other tools with SQLancer, we would be glad to provide feedback (e.g., feel free to send an email to manuel.rigger@inf.ethz.ch). We have the following general recommendations and comments: From 91a3a9b60ba81c57637b0f8f7bc673609736dfa3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 29 Feb 2024 17:59:39 +0800 Subject: [PATCH 0838/1284] Add interfaces for Expression --- src/sqlancer/common/ast/newast/Constant.java | 6 +++ .../common/ast/newast/Expression.java | 6 +++ src/sqlancer/common/ast/newast/Join.java | 14 ++++++ src/sqlancer/common/ast/newast/Select.java | 50 +++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 src/sqlancer/common/ast/newast/Constant.java create mode 100644 src/sqlancer/common/ast/newast/Expression.java create mode 100644 src/sqlancer/common/ast/newast/Join.java create mode 100644 src/sqlancer/common/ast/newast/Select.java diff --git a/src/sqlancer/common/ast/newast/Constant.java b/src/sqlancer/common/ast/newast/Constant.java new file mode 100644 index 000000000..a6a6dfd49 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Constant.java @@ -0,0 +1,6 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTableColumn; + +public interface Constant> extends Expression { +} diff --git a/src/sqlancer/common/ast/newast/Expression.java b/src/sqlancer/common/ast/newast/Expression.java new file mode 100644 index 000000000..925cb82c5 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Expression.java @@ -0,0 +1,6 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTableColumn; + +public interface Expression> { +} diff --git a/src/sqlancer/common/ast/newast/Join.java b/src/sqlancer/common/ast/newast/Join.java new file mode 100644 index 000000000..b1d1ed280 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Join.java @@ -0,0 +1,14 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; + +public interface Join, T extends AbstractTable, C extends AbstractTableColumn> + extends Expression { + + T getTable(); + + Expression getOnClause(); + + void setOnClause(E onClause); +} diff --git a/src/sqlancer/common/ast/newast/Select.java b/src/sqlancer/common/ast/newast/Select.java new file mode 100644 index 000000000..e96501b2b --- /dev/null +++ b/src/sqlancer/common/ast/newast/Select.java @@ -0,0 +1,50 @@ +package sqlancer.common.ast.newast; + +import java.util.List; + +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; + +public interface Select, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> + extends Expression { + + void setFromTables(List fromTables); + + List getFromList(); + + void setFromList(List fromList); + + Expression getWhereClause(); + + void setWhereClause(E whereClause); + + void setGroupByClause(List groupByClause); + + List getGroupByClause(); + + void setLimitClause(E limitClause); + + Expression getLimitClause(); + + List getOrderByClause(); + + void setOrderByExpressions(List orderBy); + + void setOffsetClause(E offsetClause); + + Expression getOffsetClause(); + + void setFetchColumns(List fetchColumns); + + List getFetchColumns(); + + void setJoinClauses(List joinStatements); + + List getJoinClauses(); + + void setHavingClause(E havingClause); + + Expression getHavingClause(); + + String asString(); +} From ab066addf7042cb4a6da0cc6439ba345809ecfb9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 25 Mar 2024 10:34:48 +0800 Subject: [PATCH 0839/1284] Change setFromTables to setFromList for SQLite and MariaDB --- src/sqlancer/common/ast/newast/Select.java | 2 -- .../mariadb/ast/MariaDBSelectStatement.java | 10 ------- .../mariadb/ast/MariaDBStringVisitor.java | 13 +++++++-- .../mariadb/ast/MariaDBTableReference.java | 16 +++++++++++ src/sqlancer/mariadb/ast/MariaDBVisitor.java | 4 +++ .../mariadb/oracle/MariaDBDQPOracle.java | 4 ++- .../mariadb/oracle/MariaDBNoRECOracle.java | 5 ++-- src/sqlancer/sqlite3/ast/SQLite3Select.java | 4 --- .../sqlite3/oracle/SQLite3NoRECOracle.java | 2 +- .../SQLite3PivotedQuerySynthesisOracle.java | 2 +- .../sqlite3/oracle/tlp/SQLite3TLPBase.java | 2 +- .../oracle/tlp/SQLite3TLPHavingOracle.java | 2 +- test/sqlancer/dbms/TestMariaDB.java | 28 +++++++++++++++++++ 13 files changed, 69 insertions(+), 25 deletions(-) create mode 100644 src/sqlancer/mariadb/ast/MariaDBTableReference.java diff --git a/src/sqlancer/common/ast/newast/Select.java b/src/sqlancer/common/ast/newast/Select.java index e96501b2b..0953f289a 100644 --- a/src/sqlancer/common/ast/newast/Select.java +++ b/src/sqlancer/common/ast/newast/Select.java @@ -8,8 +8,6 @@ public interface Select, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> extends Expression { - void setFromTables(List fromTables); - List getFromList(); void setFromList(List fromList); diff --git a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java index 83b944a17..a690276d1 100644 --- a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java +++ b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java @@ -4,7 +4,6 @@ import java.util.List; import sqlancer.common.ast.SelectBase; -import sqlancer.mariadb.MariaDBSchema.MariaDBTable; public class MariaDBSelectStatement extends SelectBase implements MariaDBExpression { @@ -14,7 +13,6 @@ public enum MariaDBSelectType { private List groupBys = new ArrayList<>(); private List columns = new ArrayList<>(); - private List tables = new ArrayList<>(); private MariaDBSelectType selectType = MariaDBSelectType.ALL; private MariaDBExpression whereCondition; @@ -28,10 +26,6 @@ public void setFetchColumns(List columns) { } - public void setFromTables(List tables) { - this.tables = tables; - } - public void setSelectType(MariaDBSelectType selectType) { this.selectType = selectType; } @@ -53,10 +47,6 @@ public MariaDBSelectType getSelectType() { return selectType; } - public List getTables() { - return tables; - } - public MariaDBExpression getWhereCondition() { return whereCondition; } diff --git a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java index 07951d9ac..17c84ef4a 100644 --- a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.mariadb.ast; import java.util.List; -import java.util.stream.Collectors; public class MariaDBStringVisitor extends MariaDBVisitor { @@ -41,8 +40,13 @@ public void visit(MariaDBSelectStatement s) { visit(column); } sb.append(" FROM "); - sb.append(s.getTables().stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + for (int j = 0; j < s.getFromList().size(); j++) { + if (j != 0) { + sb.append(", "); + } + visit(s.getFromList().get(j)); + } for (MariaDBExpression j : s.getJoinList()) { visit(j); } @@ -167,4 +171,9 @@ public void visit(MariaDBJoin join) { visit(join.getOnClause()); } } + + @Override + public void visit(MariaDBTableReference ref) { + sb.append(ref.getTable().getName()); + } } diff --git a/src/sqlancer/mariadb/ast/MariaDBTableReference.java b/src/sqlancer/mariadb/ast/MariaDBTableReference.java new file mode 100644 index 000000000..f045907ca --- /dev/null +++ b/src/sqlancer/mariadb/ast/MariaDBTableReference.java @@ -0,0 +1,16 @@ +package sqlancer.mariadb.ast; + +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; + +public class MariaDBTableReference implements MariaDBExpression { + + private final MariaDBTable table; + + public MariaDBTableReference(MariaDBTable table) { + this.table = table; + } + + public MariaDBTable getTable() { + return table; + } +} diff --git a/src/sqlancer/mariadb/ast/MariaDBVisitor.java b/src/sqlancer/mariadb/ast/MariaDBVisitor.java index af94394b9..8626dc967 100644 --- a/src/sqlancer/mariadb/ast/MariaDBVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBVisitor.java @@ -24,6 +24,8 @@ public abstract class MariaDBVisitor { public abstract void visit(MariaDBJoin join); + public abstract void visit(MariaDBTableReference join); + public void visit(MariaDBExpression expr) { if (expr instanceof MariaDBConstant) { visit((MariaDBConstant) expr); @@ -47,6 +49,8 @@ public void visit(MariaDBExpression expr) { visit((MariaDBInOperation) expr); } else if (expr instanceof MariaDBJoin) { visit((MariaDBJoin) expr); + } else if (expr instanceof MariaDBTableReference) { + visit((MariaDBTableReference) expr); } else { throw new AssertionError(expr.getClass()); } diff --git a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java index a92be5d1e..ee85f2df7 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java @@ -17,6 +17,7 @@ import sqlancer.mariadb.ast.MariaDBExpression; import sqlancer.mariadb.ast.MariaDBJoin; import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBTableReference; import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; import sqlancer.mariadb.gen.MariaDBSetGenerator; @@ -60,7 +61,8 @@ public void check() throws Exception { select.setJoinList(joinExpressions.stream().map(j -> (MariaDBExpression) j).collect(Collectors.toList())); // Set the from clause from the tables that are not used in the join. - select.setFromTables(tables.getTables()); + select.setFromList( + tables.getTables().stream().map(t -> new MariaDBTableReference(t)).collect(Collectors.toList())); // Get the result of the first query String originalQueryString = MariaDBVisitor.asString(select); diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index bba0627e4..ab6438275 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -23,6 +23,7 @@ import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation.MariaDBPostfixUnaryOperator; import sqlancer.mariadb.ast.MariaDBSelectStatement; import sqlancer.mariadb.ast.MariaDBSelectStatement.MariaDBSelectType; +import sqlancer.mariadb.ast.MariaDBTableReference; import sqlancer.mariadb.ast.MariaDBText; import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; @@ -76,7 +77,7 @@ private int getUnoptimizedQuery(MariaDBTable randomTable, MariaDBExpression rand randomWhereCondition); MariaDBText asText = new MariaDBText(isTrue, " as count", false); select.setFetchColumns(Arrays.asList(asText)); - select.setFromTables(Arrays.asList(randomTable)); + select.setFromList(Arrays.asList(new MariaDBTableReference(randomTable))); select.setSelectType(MariaDBSelectType.ALL); int secondCount = 0; @@ -103,7 +104,7 @@ private int getOptimizedQuery(MariaDBTable randomTable, MariaDBExpression random new MariaDBColumnName(new MariaDBColumn("*", MariaDBDataType.INT, false, 0)), MariaDBAggregateFunction.COUNT); select.setFetchColumns(Arrays.asList(aggr)); - select.setFromTables(Arrays.asList(randomTable)); + select.setFromList(Arrays.asList(new MariaDBTableReference(randomTable))); select.setWhereClause(randomWhereCondition); select.setSelectType(MariaDBSelectType.ALL); int firstCount; diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index bff7fb775..5ff008709 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -46,10 +46,6 @@ public void setSelectType(SelectType fromOptions) { this.setFromOptions(fromOptions); } - public void setFromTables(List fromTables) { - this.setFromList(fromTables); - } - public SelectType getFromOptions() { return fromOptions; } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 174efcbb6..45ec6631d 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -80,7 +80,7 @@ public void check() throws SQLException { List joinStatements = gen.getRandomJoinClauses(tables); List tableRefs = SQLite3Common.getTableRefs(tables, s); SQLite3Select select = new SQLite3Select(); - select.setFromTables(tableRefs); + select.setFromList(tableRefs); select.setJoinClauses(joinStatements); Function optimizedQuery = getOptimizedQuery(select, randomWhereCondition); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index c19a1059d..4021c88e9 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -75,7 +75,7 @@ public SQLite3Select getQuery() throws SQLException { .filter(c -> !SQLite3Schema.ROWID_STRINGS.contains(c.getName())).collect(Collectors.toList()); List joinStatements = getJoinStatements(globalState, tables, columnsWithoutRowid); selectStatement.setJoinClauses(joinStatements); - selectStatement.setFromTables(SQLite3Common.getTableRefs(tables, globalState.getSchema())); + selectStatement.setFromList(SQLite3Common.getTableRefs(tables, globalState.getSchema())); fetchColumns = Randomly.nonEmptySubset(columnsWithoutRowid); List allTables = new ArrayList<>(); diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index 650df8537..b946ebe38 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -48,7 +48,7 @@ public void check() throws SQLException { List joinStatements = gen.getRandomJoinClauses(tables); List tableRefs = SQLite3Common.getTableRefs(tables, s); select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); - select.setFromTables(tableRefs); + select.setFromList(tableRefs); select.setWhereClause(null); } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index 718c0d4ce..248d4db5f 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -57,7 +57,7 @@ public void check() throws SQLException { List from = SQLite3Common.getTableRefs(tables, state.getSchema()); select.setJoinClauses(joinStatements); select.setSelectType(SelectType.ALL); - select.setFromTables(from); + select.setFromList(from); // TODO order by? select.setGroupByClause(groupByColumns); select.setHavingClause(null); diff --git a/test/sqlancer/dbms/TestMariaDB.java b/test/sqlancer/dbms/TestMariaDB.java index e5c188d28..f1b86a339 100644 --- a/test/sqlancer/dbms/TestMariaDB.java +++ b/test/sqlancer/dbms/TestMariaDB.java @@ -3,12 +3,40 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import java.util.List; +import java.util.stream.Collectors; + import org.junit.jupiter.api.Test; import sqlancer.Main; +import sqlancer.mariadb.MariaDBSchema; +import sqlancer.mariadb.ast.MariaDBColumnName; +import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBTableReference; +import sqlancer.mariadb.ast.MariaDBVisitor; public class TestMariaDB { + @Test + public void testSelectAsString() { + MariaDBSchema.MariaDBColumn c0 = new MariaDBSchema.MariaDBColumn("c0", MariaDBSchema.MariaDBDataType.INT, true, + 0); + MariaDBSchema.MariaDBColumn c1 = new MariaDBSchema.MariaDBColumn("c1", MariaDBSchema.MariaDBDataType.INT, false, + 0); + List columns = List.of(c0, c1); + List indices = List.of(); + MariaDBSchema.MariaDBTable t1 = new MariaDBSchema.MariaDBTable("t1", columns, indices, + MariaDBSchema.MariaDBTable.MariaDBEngine.INNO_DB); + MariaDBSchema.MariaDBTables tables = new MariaDBSchema.MariaDBTables(List.of(t1)); + + MariaDBSelectStatement select = new MariaDBSelectStatement(); + select.setFetchColumns(tables.getColumns().stream().map(MariaDBColumnName::new).collect(Collectors.toList())); + select.setFromList(tables.getTables().stream().map(MariaDBTableReference::new).collect(Collectors.toList())); + + String selectString = MariaDBVisitor.asString(select); + assertEquals("SELECT c0, c1 FROM t1", selectString); + } + @Test public void testMariaDB() { String mariaDBAvailable = System.getenv("MARIADB_AVAILABLE"); From 3da3faf6238ca008ff3f5edd3548f2cabcf9cfce Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:33:24 +0800 Subject: [PATCH 0840/1284] Update expected errors (#922) * Refactor expected errors * Update creation of ExpectedErrors * Expand ExpectedErrors unit tests * Add missing errors to postgres * Update expected errors for sqlite3 --- src/sqlancer/citus/gen/CitusCommon.java | 11 +- src/sqlancer/clickhouse/ClickHouseErrors.java | 124 +++++++--------- src/sqlancer/cnosdb/CnosDBExpectedError.java | 116 +++++++-------- .../cockroachdb/CockroachDBErrors.java | 68 +++++++-- src/sqlancer/common/query/ExpectedErrors.java | 64 +++++++-- src/sqlancer/databend/DatabendErrors.java | 33 ++++- src/sqlancer/doris/DorisErrors.java | 23 ++- src/sqlancer/duckdb/DuckDBErrors.java | 53 +++++-- src/sqlancer/h2/H2Errors.java | 27 +++- src/sqlancer/hsqldb/HSQLDBErrors.java | 35 ++++- src/sqlancer/mariadb/MariaDBErrors.java | 32 ++++- .../materialize/gen/MaterializeCommon.java | 109 +++++++++++--- src/sqlancer/mysql/MySQLErrors.java | 23 ++- src/sqlancer/oceanbase/OceanBaseErrors.java | 22 ++- src/sqlancer/postgres/gen/PostgresCommon.java | 133 ++++++++++++++---- src/sqlancer/presto/PrestoErrors.java | 48 ++++++- src/sqlancer/questdb/QuestDBErrors.java | 31 +++- src/sqlancer/sqlite3/SQLite3Errors.java | 71 +++++++++- src/sqlancer/stonedb/StoneDBErrors.java | 43 ++++-- src/sqlancer/tidb/TiDBErrors.java | 33 ++++- .../timescaledb/gen/TimescaleDBCommon.java | 9 +- src/sqlancer/yugabyte/ycql/YCQLErrors.java | 13 +- src/sqlancer/yugabyte/ysql/YSQLErrors.java | 113 +++++++++++++-- test/sqlancer/TestExpectedErrors.java | 72 ++++++++++ 24 files changed, 1028 insertions(+), 278 deletions(-) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index f4a121efe..1ca2af138 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -1,5 +1,8 @@ package sqlancer.citus.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.citus.CitusBugs; import sqlancer.common.query.ExpectedErrors; @@ -8,8 +11,9 @@ public final class CitusCommon { private CitusCommon() { } - public static void addCitusErrors(ExpectedErrors errors) { + public static List getCitusErrors() { // not supported by Citus + ArrayList errors = new ArrayList<>(); errors.add("failed to evaluate partition key in insert"); errors.add("cannot perform an INSERT without a partition column value"); errors.add("cannot perform an INSERT with NULL in the partition column"); @@ -74,6 +78,11 @@ public static void addCitusErrors(ExpectedErrors errors) { if (CitusBugs.bug4079) { errors.add("aggregate function calls cannot be nested"); } + + return errors; } + public static void addCitusErrors(ExpectedErrors errors) { + errors.addAll(getCitusErrors()); + } } diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index e824faef9..23c27c8bf 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -1,5 +1,7 @@ package sqlancer.clickhouse; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class ClickHouseErrors { @@ -7,79 +9,57 @@ public final class ClickHouseErrors { private ClickHouseErrors() { } - public static void addExpectedExpressionErrors(ExpectedErrors errors) { - errors.add("Argument at index 1 for function like must be constant"); - errors.add("Argument at index 1 for function notLike must be constant"); - errors.add("Attempt to read after eof: while converting"); - errors.add("Bad get: has Int64, requested UInt64"); - errors.add("Cannot convert string"); - errors.add("Cannot insert NULL value into a column of type"); - errors.add("Cannot parse Int32 from String, because value is too short"); - errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 - errors.add("Cannot parse infinity."); - errors.add("Cannot parse number with a sign character but without any numeric character"); - errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); - errors.add("Cannot parse string"); - errors.add("Cannot read floating point value"); - errors.add("Cyclic aliases: default expression and column type are incompatible"); - errors.add("Directory for table data"); - errors.add("Directory not empty"); - errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () - errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); - errors.add("Illegal type"); - errors.add("Illegal value (aggregate function) for positional argument in GROUP BY"); - errors.add("Invalid escape sequence at the end of LIKE pattern"); - errors.add("Invalid type for filter in"); - errors.add("Memory limit"); - errors.add("OptimizedRegularExpression: cannot compile re2"); - errors.add("Partition key cannot contain constants"); - errors.add("Positional argument out of bounds"); - errors.add("Sampling expression must be present in the primary key"); - errors.add("Sorting key cannot contain constants"); - errors.add("There is no supertype for types"); - errors.add("argument of function"); - errors.add("but its arguments considered equal according to constraints"); - errors.add("does not return a value of type UInt8"); - errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database - errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 - errors.add("invalid character class range"); - errors.add("invalid escape sequence"); - errors.add("is not under aggregate function and not in GROUP BY"); - errors.add("is not under aggregate function"); - errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation - errors.add( - "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); - errors.add("there are only columns"); - errors.add("there are columns"); - errors.add("in block. (NOT_FOUND_COLUMN_IN_BLOCK)"); - errors.add("Missing columns"); - errors.add("Ambiguous column"); - errors.add("Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)"); - errors.add("Floating point partition key is not supported"); - errors.add("Cannot get JOIN keys from JOIN ON section"); - errors.add("ILLEGAL_DIVISION"); - errors.add("DECIMAL_OVERFLOW"); - errors.add("Cannot convert out of range floating point value to integer type"); - errors.add("Unexpected inf or nan to integer conversion"); - errors.add("No such name in Block::erase"); // https://github.com/ClickHouse/ClickHouse/issues/42769 - errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/43003 - errors.add("cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)"); - errors.add("AMBIGUOUS_IDENTIFIER"); - errors.add("CYCLIC_ALIASES"); - errors.add("Positional argument numeric constant expression is not representable as"); - errors.add("Positional argument must be constant with numeric type"); - errors.add(" is out of bounds. Expected in range"); - errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); - errors.add("Unexpected inf or nan to integer conversion"); - errors.add("Unsigned type must not contain"); - errors.add("Unexpected inf or nan to integer conversion"); + public static List getExpectedExpressionErrors() { + return List.of("Argument at index 1 for function like must be constant", + "Argument at index 1 for function notLike must be constant", + "Attempt to read after eof: while converting", "Bad get: has Int64, requested UInt64", + "Cannot convert string", "Cannot insert NULL value into a column of type", + "Cannot parse Int32 from String, because value is too short", "Cannot parse NaN.: while converting", // https://github.com/ClickHouse/ClickHouse/issues/22710 + "Cannot parse infinity.", "Cannot parse number with a sign character but without any numeric character", + "Cannot parse number with multiple sign (+/-) characters or intermediate sign character", + "Cannot parse string", "Cannot read floating point value", + "Cyclic aliases: default expression and column type are incompatible", "Directory for table data", + "Directory not empty", "Expected one of: compound identifier, identifier, list of elements (version", // VALUES + // () + "Function 'like' doesn't support search with non-constant needles in constant haystack", "Illegal type", + "Illegal value (aggregate function) for positional argument in GROUP BY", + "Invalid escape sequence at the end of LIKE pattern", "Invalid type for filter in", "Memory limit", + "OptimizedRegularExpression: cannot compile re2", "Partition key cannot contain constants", + "Positional argument out of bounds", "Sampling expression must be present in the primary key", + "Sorting key cannot contain constants", "There is no supertype for types", "argument of function", + "but its arguments considered equal according to constraints", "does not return a value of type UInt8", + "doesn't exist", // TODO: consecutive test runs can lead to dropped database + "in block. There are only columns:", // https://github.com/ClickHouse/ClickHouse/issues/42399 + "invalid character class range", "invalid escape sequence", + "is not under aggregate function and not in GROUP BY", "is not under aggregate function", + "is violated at row 1. Expression:", // TODO: check constraint on table creation + "is violated, because it is a constant expression returning 0. It is most likely an error in table definition", + "there are only columns", "there are columns", "in block. (NOT_FOUND_COLUMN_IN_BLOCK)", + "Missing columns", "Ambiguous column", + "Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)", + "Floating point partition key is not supported", "Cannot get JOIN keys from JOIN ON section", + "ILLEGAL_DIVISION", "DECIMAL_OVERFLOW", + "Cannot convert out of range floating point value to integer type", + "Unexpected inf or nan to integer conversion", "No such name in Block::erase", // https://github.com/ClickHouse/ClickHouse/issues/42769 + "EMPTY_LIST_OF_COLUMNS_QUERIED", // https://github.com/ClickHouse/ClickHouse/issues/43003 + "cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)", "AMBIGUOUS_IDENTIFIER", "CYCLIC_ALIASES", + "Positional argument numeric constant expression is not representable as", + "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", + "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)", + "Unexpected inf or nan to integer conversion", "Unsigned type must not contain", + "Unexpected inf or nan to integer conversion", - // The way we generate JOINs we can have ambiguous left table column without alias - // We may not count it as an issue, but it makes no sense to add more complex AST generation logic - errors.add("MULTIPLE_EXPRESSIONS_FOR_ALIAS"); - errors.add("AMBIGUOUS_IDENTIFIER"); // https://github.com/ClickHouse/ClickHouse/issues/45389 - errors.add("AMBIGUOUS_COLUMN_NAME"); // same https://github.com/ClickHouse/ClickHouse/issues/45389 - errors.add("Cannot parse number with multiple sign"); + // The way we generate JOINs we can have ambiguous left table column without + // alias + // We may not count it as an issue, but it makes no sense to add more complex + // AST generation logic + "MULTIPLE_EXPRESSIONS_FOR_ALIAS", "AMBIGUOUS_IDENTIFIER", // https://github.com/ClickHouse/ClickHouse/issues/45389 + "AMBIGUOUS_COLUMN_NAME", // same https://github.com/ClickHouse/ClickHouse/issues/45389 + "Cannot parse number with multiple sign"); + } + + public static void addExpectedExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpectedExpressionErrors()); } } diff --git a/src/sqlancer/cnosdb/CnosDBExpectedError.java b/src/sqlancer/cnosdb/CnosDBExpectedError.java index 516e44ce4..5932bfdac 100644 --- a/src/sqlancer/cnosdb/CnosDBExpectedError.java +++ b/src/sqlancer/cnosdb/CnosDBExpectedError.java @@ -6,74 +6,76 @@ import sqlancer.common.query.ExpectedErrors; public final class CnosDBExpectedError { - private static final List ERRORS = new ArrayList<>(); private CnosDBExpectedError() { } - static { - ERRORS.add("have the same name. Consider aliasing"); - ERRORS.add( + public static List getExpectedErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("have the same name. Consider aliasing"); + errors.add( "error: Optimizer rule 'projection_push_down' failed due to unexpected error: Schema error: Schema contains duplicate qualified field name"); - ERRORS.add("Projection references non-aggregate values:"); - ERRORS.add("External err: Schema error: No field named"); - ERRORS.add( + errors.add("Projection references non-aggregate values:"); + errors.add("External err: Schema error: No field named"); + errors.add( "Optimizer rule 'common_sub_expression_eliminate' failed due to unexpected error: Schema error: No field named"); - ERRORS.add("Binary"); - ERRORS.add("Invalid pattern in LIKE expression"); - ERRORS.add("If the projection contains the time column, it must contain the field column."); - ERRORS.add("Schema error: No field named"); - ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:"); - ERRORS.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error"); - ERRORS.add("Schema error: No field named "); - ERRORS.add("err: External err: Schema error: No field named"); - ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error"); - ERRORS.add("Csv error: CSV Writer does not support List"); - ERRORS.add("This feature is not implemented: cross join."); - ERRORS.add("Execution error: field position must be greater than zero"); - ERRORS.add("First argument of `DATE_PART` must be non-null scalar Utf8"); - ERRORS.add("Cannot create filter with non-boolean predicate 'NULL' returning Null"); - ERRORS.add("requested character too large for encoding."); - ERRORS.add("Can not find compatible types to compare Boolean with [Utf8]."); - ERRORS.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT"); - ERRORS.add("HAVING clause references non-aggregate values:"); - ERRORS.add("Cannot create filter with non-boolean predicate"); - ERRORS.add("negative substring length not allowed"); - ERRORS.add("The function Sum does not support inputs of type Boolean."); - ERRORS.add("The function Avg does not support inputs of type Boolean."); - ERRORS.add("Percentile value must be between 0.0 and 1.0 inclusive"); - ERRORS.add("Date part '' not supported"); - ERRORS.add("Min/Max accumulator not implemented for type Boolean."); - ERRORS.add("meta need get_series_id_by_filter"); - ERRORS.add("Arrow: Cast error:"); - ERRORS.add("Arrow error: Cast error:"); - ERRORS.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:"); - ERRORS.add("Arrow error: Divide by zero error"); - ERRORS.add("desired percentile argument must be float literal"); - ERRORS.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)"); - ERRORS.add("Execution error: Date part"); - ERRORS.add("Physical plan does not support logical expression MIN(Boolean"); - ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64"); - ERRORS.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64."); - ERRORS.add("Data type UInt64 not supported for binary operation '#' on dyn arrays."); - ERRORS.add("Arrow: Divide by zero error"); - ERRORS.add("The function ApproxPercentileCont does not support inputs of type Null."); - ERRORS.add("can't be evaluated because there isn't a common type to coerce the types to"); - ERRORS.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug"); - ERRORS.add("The function ApproxMedian does not support inputs of type Null."); - ERRORS.add("null character not permitted."); - ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Null."); - ERRORS.add("This feature is not implemented"); - ERRORS.add("The function Avg does not support inputs of type Null."); - ERRORS.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]"); - ERRORS.add( + errors.add("Binary"); + errors.add("Invalid pattern in LIKE expression"); + errors.add("If the projection contains the time column, it must contain the field column."); + errors.add("Schema error: No field named"); + errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:"); + errors.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error"); + errors.add("Schema error: No field named "); + errors.add("err: External err: Schema error: No field named"); + errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error"); + errors.add("Csv error: CSV Writer does not support List"); + errors.add("This feature is not implemented: cross join."); + errors.add("Execution error: field position must be greater than zero"); + errors.add("First argument of `DATE_PART` must be non-null scalar Utf8"); + errors.add("Cannot create filter with non-boolean predicate 'NULL' returning Null"); + errors.add("requested character too large for encoding."); + errors.add("Can not find compatible types to compare Boolean with [Utf8]."); + errors.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT"); + errors.add("HAVING clause references non-aggregate values:"); + errors.add("Cannot create filter with non-boolean predicate"); + errors.add("negative substring length not allowed"); + errors.add("The function Sum does not support inputs of type Boolean."); + errors.add("The function Avg does not support inputs of type Boolean."); + errors.add("Percentile value must be between 0.0 and 1.0 inclusive"); + errors.add("Date part '' not supported"); + errors.add("Min/Max accumulator not implemented for type Boolean."); + errors.add("meta need get_series_id_by_filter"); + errors.add("Arrow: Cast error:"); + errors.add("Arrow error: Cast error:"); + errors.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:"); + errors.add("Arrow error: Divide by zero error"); + errors.add("desired percentile argument must be float literal"); + errors.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)"); + errors.add("Execution error: Date part"); + errors.add("Physical plan does not support logical expression MIN(Boolean"); + errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64"); + errors.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64."); + errors.add("Data type UInt64 not supported for binary operation '#' on dyn arrays."); + errors.add("Arrow: Divide by zero error"); + errors.add("The function ApproxPercentileCont does not support inputs of type Null."); + errors.add("can't be evaluated because there isn't a common type to coerce the types to"); + errors.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug"); + errors.add("The function ApproxMedian does not support inputs of type Null."); + errors.add("null character not permitted."); + errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Null."); + errors.add("This feature is not implemented"); + errors.add("The function Avg does not support inputs of type Null."); + errors.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]"); + errors.add( "Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed."); - ERRORS.add("Coercion from"); + errors.add("Coercion from"); + return errors; } public static ExpectedErrors expectedErrors() { ExpectedErrors res = new ExpectedErrors(); - res.addAll(ERRORS); + res.addAll(getExpectedErrors()); return res; } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 7cb738ee8..63d0b30ce 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.cockroachdb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class CockroachDBErrors { @@ -7,7 +10,9 @@ public final class CockroachDBErrors { private CockroachDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add(" non-streaming operator encountered when vectorize=auto"); if (CockroachDBBugs.bug46915) { @@ -105,10 +110,10 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("as type time"); errors.add("as TimeTZ"); errors.add("as type decimal"); - addIntervalTypeErrors(errors); - addFunctionErrors(errors); - addGroupByErrors(errors); - addJoinTypes(errors); + errors.addAll(getIntervalTypeErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getGroupByErrors()); + errors.addAll(getJoinTypes()); errors.add("as int4, found type: decimal"); errors.add("to be of type int2, found type decimal"); errors.add("to be of type int, found type decimal"); // arithmetic overflows @@ -210,11 +215,19 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("argument of OFFSET must be type int, not type decimal"); errors.add("ERROR: for SELECT DISTINCT, ORDER BY expressions must appear in select list"); - addArrayErrors(errors); - addComputedColumnErrors(errors); + errors.addAll(getArrayErrors()); + errors.addAll(getComputedColumnErrors()); + + return errors; } - private static void addArrayErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + private static List getArrayErrors() { + ArrayList errors = new ArrayList<>(); + // arrays errors.add("cannot determine type of empty array"); errors.add("unknown signature: max(unknown[])"); @@ -260,18 +273,29 @@ private static void addArrayErrors(ExpectedErrors errors) { errors.add("to be of type int[], found type decimal[]"); errors.add("to be of type unknown[]"); // IF with null array + + return errors; } - private static void addIntervalTypeErrors(ExpectedErrors errors) { + private static List getIntervalTypeErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("overflow during Encode"); errors.add("type interval"); + + return errors; } - private static void addJoinTypes(ExpectedErrors errors) { + private static List getJoinTypes() { + ArrayList errors = new ArrayList<>(); + errors.add("JOIN/USING types"); + + return errors; } - private static void addGroupByErrors(ExpectedErrors errors) { + private static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); errors.add("non-integer constant in GROUP BY"); // https://github.com/cockroachdb/cockroach/pull/46649 -> aggregates on NULL are @@ -293,9 +317,11 @@ private static void addGroupByErrors(ExpectedErrors errors) { errors.add("unknown signature: abs(string)"); errors.add("unknown signature: acos(string)"); + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); // functions errors.add("abs of min integer value (-9223372036854775808) not defined"); // ABS errors.add("the input string must not be empty"); // ASCII @@ -311,15 +337,29 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("substring(): negative substring length"); // substring errors.add("negative substring length"); // substring errors.add("must be greater than zero"); // split_part + + return errors; } - public static void addTransactionErrors(ExpectedErrors errors) { + public static List getTransactionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("current transaction is aborted"); + + return errors; } - private static void addComputedColumnErrors(ExpectedErrors errors) { + public static void addTransactionErrors(ExpectedErrors errors) { + errors.addAll(getTransactionErrors()); + } + + private static List getComputedColumnErrors() { + ArrayList errors = new ArrayList<>(); + // computed columns errors.add("computed column expressions cannot reference computed columns"); + + return errors; } } diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index 1016969c9..348da179f 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -1,9 +1,8 @@ package sqlancer.common.query; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.regex.Pattern; @@ -14,8 +13,18 @@ */ public class ExpectedErrors { - private final Set errors = new HashSet<>(); - private final List regexes = new ArrayList<>(); + private final Set errors; + private final Set regexes; + + public ExpectedErrors() { + this.errors = new HashSet<>(); + this.regexes = new HashSet<>(); + } + + public ExpectedErrors(Collection errors, Collection regexErrors) { + this.errors = new HashSet<>(errors); + this.regexes = new HashSet<>(regexErrors); + } public ExpectedErrors add(String error) { if (error == null) { @@ -33,6 +42,11 @@ public ExpectedErrors addRegex(Pattern errorPattern) { return this; } + public ExpectedErrors addRegexString(String errorPattern) { + regexes.add(Pattern.compile(errorPattern)); + return this; + } + public ExpectedErrors addAll(Collection list) { errors.addAll(list); return this; @@ -46,12 +60,19 @@ public ExpectedErrors addAllRegexes(Collection list) { return this; } - public static ExpectedErrors from(String... errors) { - ExpectedErrors expectedErrors = new ExpectedErrors(); - for (String error : errors) { - expectedErrors.add(error); + public ExpectedErrors addAllRegexStrings(Collection list) { + for (String error : list) { + regexes.add(Pattern.compile(error)); } - return expectedErrors; + return this; + } + + public static ExpectedErrors from(String... errors) { + return newErrors().with(errors).build(); + } + + public static ExpectedErrorsBuilder newErrors() { + return new ExpectedErrorsBuilder(); } /** @@ -80,4 +101,29 @@ public boolean errorIsExpected(String error) { return false; } + public static class ExpectedErrorsBuilder { + private final Set errors = new HashSet<>(); + private final Set regexes = new HashSet<>(); + + public ExpectedErrorsBuilder with(String... list) { + errors.addAll(Arrays.asList(list)); + return this; + } + + public ExpectedErrorsBuilder with(Pattern... list) { + regexes.addAll(Arrays.asList(list)); + return this; + } + + public ExpectedErrorsBuilder withRegex(String... list) { + for (String error : list) { + regexes.add(Pattern.compile(error)); + } + return this; + } + + public ExpectedErrors build() { + return new ExpectedErrors(errors, regexes); + } + } } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 5e813dcbd..2f588480f 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -1,5 +1,8 @@ package sqlancer.databend; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class DatabendErrors { @@ -7,7 +10,9 @@ public final class DatabendErrors { private DatabendErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Division by zero"); errors.add("divided by zero"); errors.add("/ by zero"); @@ -45,20 +50,42 @@ public static void addExpressionErrors(ExpectedErrors errors) { * ('067596','19'), ('', '87'); */ errors.add("Can't cast column from null into non-nullable type"); + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Division by zero"); errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); errors.add("number overflowed while evaluating function `"); // 不能在int16类型column上插入int64的数据 + + return errors; } - public static void addGroupByErrors(ExpectedErrors errors) { + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Division by zero"); errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); errors.add("GROUP BY position"); + + return errors; + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); } } diff --git a/src/sqlancer/doris/DorisErrors.java b/src/sqlancer/doris/DorisErrors.java index 678601635..8e6fdfdc9 100644 --- a/src/sqlancer/doris/DorisErrors.java +++ b/src/sqlancer/doris/DorisErrors.java @@ -1,5 +1,8 @@ package sqlancer.doris; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class DorisErrors { @@ -7,7 +10,9 @@ public final class DorisErrors { private DorisErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + // SQL syntax error errors.add("Syntax error"); errors.add("Please check your sql, we meet an error when parsing"); @@ -52,13 +57,27 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (DorisBugs.bug19611) { errors.add("Duplicated inline view column alias"); } + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Insert has filtered data in strict mode"); errors.add("Only value columns of unique table could be updated"); errors.add("Only unique olap table could be updated"); errors.add("Number out of range"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 555bc2e88..5c2909f4d 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.duckdb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class DuckDBErrors { @@ -7,7 +10,9 @@ public final class DuckDBErrors { private DuckDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("with non-constant precision is not supported"); errors.add("Like pattern must not end with escape character"); errors.add("Could not convert string"); @@ -27,9 +32,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("GROUP BY clause cannot contain aggregates!"); // investigate - addRegexErrors(errors); + errors.addAll(getRegexErrors()); - addFunctionErrors(errors); + errors.addAll(getFunctionErrors()); errors.add("Overflow in multiplication"); errors.add("Out of Range"); @@ -52,9 +57,17 @@ public static void addExpressionErrors(ExpectedErrors errors) { // timestamp errors.add("Cannot subtract infinite timestamps"); errors.add("Timestamp difference is out of bounds"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); } - private static void addRegexErrors(ExpectedErrors errors) { + private static List getRegexErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("missing ]"); errors.add("missing )"); errors.add("invalid escape sequence"); @@ -64,9 +77,13 @@ private static void addRegexErrors(ExpectedErrors errors) { errors.add("invalid perl operator"); errors.add("invalid character class range"); errors.add("width is not integer"); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("SUBSTRING cannot handle negative lengths"); errors.add("is undefined outside [-1,1]"); // ACOS etc errors.add("invalid type specifier"); // PRINTF @@ -85,11 +102,15 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("Could not choose a best candidate function for the function call"); // monthname errors.add("expected a numeric precision field"); // ROUND errors.add("with non-constant precision is not supported"); // ROUND + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { - addRegexErrors(errors); - addFunctionErrors(errors); + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getRegexErrors()); + errors.addAll(getFunctionErrors()); errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); @@ -105,11 +126,25 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Could not cast value"); errors.add("create unique index, table contains duplicate data"); errors.add("Failed to cast"); + + return errors; } - public static void addGroupByErrors(ExpectedErrors errors) { + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY term out of range"); + + return errors; + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); } } diff --git a/src/sqlancer/h2/H2Errors.java b/src/sqlancer/h2/H2Errors.java index aa1cba9cd..5846c1a88 100644 --- a/src/sqlancer/h2/H2Errors.java +++ b/src/sqlancer/h2/H2Errors.java @@ -1,5 +1,8 @@ package sqlancer.h2; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class H2Errors { @@ -7,7 +10,8 @@ public final class H2Errors { private H2Errors() { } - public static void addInsertErrors(ExpectedErrors errors) { + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); errors.add("NULL not allowed for column"); errors.add("Unique index or primary key violation"); errors.add("Data conversion error"); @@ -16,9 +20,15 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Referential integrity constraint violation"); errors.add("Check constraint invalid"); errors.add("Check constraint violation"); + return errors; } - public static void addExpressionErrors(ExpectedErrors errors) { + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); errors.add("java.lang.ArithmeticException: BigInteger would overflow supported range"); errors.add("Value too long for column"); errors.add("Numeric value out of range"); @@ -41,12 +51,23 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add(/* precision */ "must be between"); // TRUNCATE_VALUE errors.add("Cannot parse \"TIMESTAMP\" constant"); // TRUNCATE errors.add("Invalid parameter count for \"TRUNC\", expected count: \"1\""); // TRUNCATE + return errors; } - public static void addDeleteErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getDeleteErrors() { + ArrayList errors = new ArrayList<>(); errors.add("No default value is set for column"); // referential actions errors.add("Referential integrity constraint violation"); errors.add("NULL not allowed for column"); + return errors; + } + + public static void addDeleteErrors(ExpectedErrors errors) { + errors.addAll(getDeleteErrors()); } } diff --git a/src/sqlancer/hsqldb/HSQLDBErrors.java b/src/sqlancer/hsqldb/HSQLDBErrors.java index 8e2a8cd2f..c02f8f2b9 100644 --- a/src/sqlancer/hsqldb/HSQLDBErrors.java +++ b/src/sqlancer/hsqldb/HSQLDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.hsqldb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class HSQLDBErrors { @@ -7,21 +10,43 @@ public final class HSQLDBErrors { private HSQLDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); errors.add(""); + return errors; } - private static void addRegexErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + private static List getRegexErrors() { + ArrayList errors = new ArrayList<>(); + errors.add(""); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add(""); + + return errors; + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getRegexErrors()); + errors.addAll(getFunctionErrors()); + + return errors; } public static void addInsertErrors(ExpectedErrors errors) { - addRegexErrors(errors); - addFunctionErrors(errors); + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/mariadb/MariaDBErrors.java b/src/sqlancer/mariadb/MariaDBErrors.java index 4918accfb..b84deba02 100644 --- a/src/sqlancer/mariadb/MariaDBErrors.java +++ b/src/sqlancer/mariadb/MariaDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.mariadb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class MariaDBErrors { @@ -7,6 +10,27 @@ public final class MariaDBErrors { private MariaDBErrors() { } + public static List getCommonErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("is out of range"); + // regex + errors.add("unmatched parentheses"); + errors.add("nothing to repeat at offset"); + errors.add("missing )"); + errors.add("missing terminating ]"); + errors.add("range out of order in character class"); + errors.add("unrecognized character after "); + errors.add("Got error '(*VERB) not recognized or malformed"); + errors.add("must be followed by"); + errors.add("malformed number or name after"); + errors.add("digit expected after"); + errors.add("Regex error"); + errors.add("Lock wait timeout exceeded"); + + return errors; + } + public static void addCommonErrors(ExpectedErrors errors) { errors.add("is out of range"); // regex @@ -24,7 +48,8 @@ public static void addCommonErrors(ExpectedErrors errors) { errors.add("Lock wait timeout exceeded"); } - public static void addInsertErrors(ExpectedErrors errors) { + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); errors.add("Out of range"); errors.add("Duplicate entry"); // violates UNIQUE constraint errors.add("cannot be null"); // violates NOT NULL constraint @@ -34,6 +59,11 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("The value specified for generated column"); // trying to insert into a generated column errors.add("Incorrect double value"); errors.add("Incorrect string value"); + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/materialize/gen/MaterializeCommon.java b/src/sqlancer/materialize/gen/MaterializeCommon.java index 7dee348f4..d21c8f81c 100644 --- a/src/sqlancer/materialize/gen/MaterializeCommon.java +++ b/src/sqlancer/materialize/gen/MaterializeCommon.java @@ -1,5 +1,6 @@ package sqlancer.materialize.gen; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; @@ -21,7 +22,9 @@ public final class MaterializeCommon { private MaterializeCommon() { } - public static void addCommonFetchErrors(ExpectedErrors errors) { + public static List getCommonFetchErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); errors.add("but it cannot be referenced from this part of the query"); errors.add("missing FROM-clause entry for table"); @@ -36,14 +39,30 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("does not exist"); errors.add("aggregate functions are not allowed in"); errors.add("is only defined for finite arguments"); + + return errors; } - public static void addCommonTableErrors(ExpectedErrors errors) { + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.addAll(getCommonFetchErrors()); + } + + public static List getCommonTableErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not commutative"); // exclude errors.add("operator requires run-time type coercion"); // exclude + + return errors; } - public static void addCommonExpressionErrors(ExpectedErrors errors) { + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.addAll(getCommonTableErrors()); + } + + public static List getCommonExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); @@ -77,14 +96,22 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("aggregate functions are not allowed in"); errors.add("only defined for finite arguments"); errors.add("unable to parse column reference in GROUP BY clause"); // TODO - addToCharFunctionErrors(errors); - addBitStringOperationErrors(errors); - addFunctionErrors(errors); - addCommonRangeExpressionErrors(errors); - addCommonRegexExpressionErrors(errors); + errors.addAll(getToCharFunctionErrors()); + errors.addAll(getBitStringOperationErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getCommonRangeExpressionErrors()); + errors.addAll(getCommonRegexExpressionErrors()); + + return errors; } - private static void addToCharFunctionErrors(ExpectedErrors errors) { + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + private static List getToCharFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("multiple decimal points"); errors.add("and decimal point together"); errors.add("multiple decimal points"); @@ -96,16 +123,24 @@ private static void addToCharFunctionErrors(ExpectedErrors errors) { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + + return errors; } - private static void addBitStringOperationErrors(ExpectedErrors errors) { + private static List getBitStringOperationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("cannot XOR bit strings of different sizes"); errors.add("cannot AND bit strings of different sizes"); errors.add("cannot OR bit strings of different sizes"); errors.add("must be type boolean, not type text"); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("out of valid range"); // get_bit/get_byte errors.add("cannot take logarithm of a negative number"); errors.add("cannot take logarithm of zero"); @@ -117,24 +152,62 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii errors.add("negative substring length not allowed"); // substr errors.add("invalid mask length"); // set_masklen + + return errors; } - private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + private static List getCommonRegexExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not a valid hexadecimal digit"); + + return errors; } - public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + public static List getCommonRangeExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("range lower bound must be less than or equal to range upper bound"); errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); errors.add("result of range union would not be contiguous"); + + return errors; } - public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + public static List getCommonInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("value too long for type character"); errors.add("not found in view targetlist"); errors.add("CAST does not support casting from"); + + return errors; + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + public static List getGroupingErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("unable to parse column reference in GROUP BY clause"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in"); + + return errors; + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.addAll(getGroupingErrors()); } public static boolean appendDataType(MaterializeDataType type, StringBuilder sb, boolean allowSerial, @@ -332,14 +405,6 @@ private static void deleteOrUpdateAction(StringBuilder sb) { sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); } - public static void addGroupingErrors(ExpectedErrors errors) { - errors.add("non-integer constant in GROUP BY"); // TODO - errors.add("unable to parse column reference in GROUP BY clause"); // TODO - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("is not in select list"); - errors.add("aggregate functions are not allowed in"); - } - public static String getFreeIndexName(MaterializeSchema s) { List indexNames = s.getIndexNames(); String candidateName; diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 9a8f076ce..3f681acb5 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -1,5 +1,8 @@ package sqlancer.mysql; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class MySQLErrors { @@ -7,16 +10,26 @@ public final class MySQLErrors { private MySQLErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) errors.add("is not valid for CHARACTER SET"); if (MySQLBugs.bug111471) { errors.add("Memory capacity exceeded"); } + + return errors; } - public static void addInsertUpdateErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("doesn't have a default value"); errors.add("Data truncation"); errors.add("Incorrect integer value"); @@ -25,6 +38,12 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { errors.add("Data truncated for functional index"); errors.add("cannot be null"); errors.add("Incorrect decimal value"); + + return errors; + } + + public static void addInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getInsertUpdateErrors()); } } diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index 893490c34..822c5497d 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -1,5 +1,8 @@ package sqlancer.oceanbase; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class OceanBaseErrors { @@ -7,7 +10,9 @@ public final class OceanBaseErrors { private OceanBaseErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) errors.add("is not valid for CHARACTER SET"); errors.add("The observer or zone is not the master"); @@ -15,9 +20,17 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Truncated incorrect DOUBLE value"); errors.add("Invalid numeric"); errors.add("Data truncated for argument"); + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Duplicate entry"); errors.add("cannot be null"); errors.add("doesn't have a default value"); @@ -42,5 +55,10 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Invalid numeric"); errors.add("Miss column"); + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 497ad6d0b..a0cc21ae2 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -23,7 +23,9 @@ public final class PostgresCommon { private PostgresCommon() { } - public static void addCommonFetchErrors(ExpectedErrors errors) { + public static List getCommonFetchErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); errors.add("but it cannot be referenced from this part of the query"); errors.add("missing FROM-clause entry for table"); @@ -33,14 +35,30 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("non-integer constant in GROUP BY"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY position"); + + return errors; } - public static void addCommonTableErrors(ExpectedErrors errors) { + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.addAll(getCommonFetchErrors()); + } + + public static List getCommonTableErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not commutative"); // exclude errors.add("operator requires run-time type coercion"); // exclude + + return errors; } - public static void addCommonExpressionErrors(ExpectedErrors errors) { + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.addAll(getCommonTableErrors()); + } + + public static List getCommonExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); @@ -54,7 +72,6 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("invalid hexadecimal digit"); errors.add("invalid hexadecimal data: odd number of digits"); errors.add("zero raised to a negative power is undefined"); - errors.addRegex(Pattern.compile("cannot convert infinity to \\w+")); errors.add("division by zero"); errors.add("invalid input syntax for type money"); errors.add("invalid input syntax for type"); @@ -65,14 +82,32 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("a negative number raised to a non-integer power yields a complex result"); errors.add("could not determine polymorphic type because input has type unknown"); errors.add("character number must be positive"); - addToCharFunctionErrors(errors); - addBitStringOperationErrors(errors); - addFunctionErrors(errors); - addCommonRangeExpressionErrors(errors); - addCommonRegexExpressionErrors(errors); + errors.addAll(getToCharFunctionErrors()); + errors.addAll(getBitStringOperationErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getCommonRangeExpressionErrors()); + errors.addAll(getCommonRegexExpressionErrors()); + + return errors; + } + + public static List getCommonExpressionRegexErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add(Pattern.compile("cannot convert infinity to \\w+")); + errors.addAll(getFunctionRegexErrors()); + + return errors; + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + errors.addAllRegexes(getCommonExpressionRegexErrors()); } - private static void addToCharFunctionErrors(ExpectedErrors errors) { + private static List getToCharFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("multiple decimal points"); errors.add("and decimal point together"); errors.add("multiple decimal points"); @@ -84,16 +119,24 @@ private static void addToCharFunctionErrors(ExpectedErrors errors) { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + + return errors; } - private static void addBitStringOperationErrors(ExpectedErrors errors) { + private static List getBitStringOperationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("cannot XOR bit strings of different sizes"); errors.add("cannot AND bit strings of different sizes"); errors.add("cannot OR bit strings of different sizes"); errors.add("must be type boolean, not type text"); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("out of valid range"); // get_bit/get_byte errors.add("cannot take logarithm of a negative number"); errors.add("cannot take logarithm of zero"); @@ -103,39 +146,81 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("requested length too large"); // repeat errors.add("invalid memory alloc request size"); // repeat + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + + return errors; + } + + private static List getFunctionRegexErrors() { + ArrayList errors = new ArrayList<>(); /* * PostgreSQL support only a few conversion variants to ASCII: LATIN1, LATIN2, LATIN9 and WINDOWS1250. So, it is * better to skip this error at all. */ - errors.addRegex(Pattern.compile("encoding conversion from \\w+ to ASCII not supported")); + errors.add(Pattern.compile("encoding conversion from \\w+ to ASCII not supported")); /* * In accordance with PostgreSQL code, commit 0ab1a2e, conversions to or from SQL_ASCII is meaningless. So * disable errors on such an attempt. */ - errors.addRegex(Pattern.compile("encoding conversion from SQL_ASCII to \\w+ not supported")); - errors.addRegex(Pattern.compile("encoding conversion from \\w+ to SQL_ASCII not supported")); + errors.add(Pattern.compile("encoding conversion from SQL_ASCII to \\w+ not supported")); + errors.add(Pattern.compile("encoding conversion from \\w+ to SQL_ASCII not supported")); - errors.add("negative substring length not allowed"); // substr - errors.add("invalid mask length"); // set_masklen + return errors; } - private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + private static List getCommonRegexExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not a valid hexadecimal digit"); + + return errors; } - public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + public static List getCommonRangeExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("range lower bound must be less than or equal to range upper bound"); errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); errors.add("result of range union would not be contiguous"); + + return errors; } - public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonRangeExpressionErrors()); + } + + public static List getCommonInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("value too long for type character"); errors.add("cannot insert a non-DEFAULT value into column"); errors.add("not found in view targetlist"); + + return errors; + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getCommonInsertUpdateErrors()); + } + + public static List getGroupingErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in GROUP BY"); + + return errors; + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.addAll(getGroupingErrors()); } public static boolean appendDataType(PostgresDataType type, StringBuilder sb, boolean allowSerial, @@ -424,12 +509,4 @@ private static void appendExcludeElement(StringBuilder sb, PostgresGlobalState g private static void deleteOrUpdateAction(StringBuilder sb) { sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); } - - public static void addGroupingErrors(ExpectedErrors errors) { - errors.add("non-integer constant in GROUP BY"); // TODO - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("is not in select list"); - errors.add("aggregate functions are not allowed in GROUP BY"); - } - } diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 603addb5c..68a99ef9f 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -1,5 +1,8 @@ package sqlancer.presto; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class PrestoErrors { @@ -7,7 +10,9 @@ public final class PrestoErrors { private PrestoErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + // Presto errors errors.add("cannot be applied to"); errors.add("LIKE expression must evaluate to a varchar"); @@ -57,9 +62,17 @@ public static void addExpressionErrors(ExpectedErrors errors) { // ARRAY errors.add("Unknown type: ARRAY"); + + return errors; } - private static void addRegexErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + private static List getRegexErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("missing ]"); errors.add("missing )"); errors.add("invalid escape sequence"); @@ -69,9 +82,13 @@ private static void addRegexErrors(ExpectedErrors errors) { errors.add("invalid perl operator"); errors.add("invalid character class range"); errors.add("width is not integer"); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("SUBSTRING cannot handle negative lengths"); errors.add("is undefined outside [-1,1]"); // ACOS etc errors.add("invalid type specifier"); // PRINTF @@ -90,12 +107,16 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("Could not choose a best candidate function for the function call"); // monthname errors.add("expected a numeric precision field"); // ROUND errors.add("with non-constant precision is not supported"); // ROUND + + return errors; } // TODO: cover presto error - public static void addInsertErrors(ExpectedErrors errors) { - addRegexErrors(errors); - addFunctionErrors(errors); + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getRegexErrors()); + errors.addAll(getFunctionErrors()); errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); @@ -118,10 +139,23 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Values rows have mismatched types"); errors.add("Invalid numeric literal"); + return errors; } - public static void addGroupByErrors(ExpectedErrors errors) { + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("must be an aggregate expression or appear in GROUP BY clause"); + + return errors; + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); } } diff --git a/src/sqlancer/questdb/QuestDBErrors.java b/src/sqlancer/questdb/QuestDBErrors.java index 83c74fdc1..213f06ad4 100644 --- a/src/sqlancer/questdb/QuestDBErrors.java +++ b/src/sqlancer/questdb/QuestDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.questdb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class QuestDBErrors { @@ -7,7 +10,9 @@ public final class QuestDBErrors { private QuestDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + // TODO (anxing) errors.add("unexpected argument for function: "); errors.add("unexpected token:"); // SELECT FROM multiple tables without WHERE/ JOIN clause @@ -16,16 +21,36 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("too few arguments for 'in'"); errors.add("cannot compare TIMESTAMP with type"); // WHERE column IN with nonTIMESTAMP arg errors.add("constant expected"); + + return errors; } - public static void addGroupByErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getGroupByErrors() { // TODO (anxing) + + return new ArrayList<>(); } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + // TODO (anxing) errors.add("Invalid column"); errors.add("inconvertible types:"); errors.add("inconvertible value:"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 6827116af..406729bfb 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -1,6 +1,8 @@ package sqlancer.sqlite3; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import sqlancer.common.query.ExpectedErrors; @@ -9,14 +11,24 @@ public final class SQLite3Errors { private SQLite3Errors() { } - public static void addDeleteErrors(ExpectedErrors errors) { + public static List getDeleteErrors() { + ArrayList errors = new ArrayList<>(); + // DELETE trigger for a view/table to which colomns were added or deleted errors.add("columns but"); // trigger with on conflict clause errors.add("ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint"); + + return errors; } - public static void addExpectedExpressionErrors(ExpectedErrors errors) { + public static void addDeleteErrors(ExpectedErrors errors) { + errors.addAll(getDeleteErrors()); + } + + public static List getExpectedExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("[SQLITE_BUSY] The database file is locked"); errors.add("FTS expression tree is too large"); errors.add("String or BLOB exceeds size limit"); @@ -67,9 +79,16 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("circularly defined"); errors.add("[SQLITE_ERROR] SQL error or missing database"); // A possible delay in the execution of DROP TABLE // statement. + return errors; } - public static void addMatchQueryErrors(ExpectedErrors errors) { + public static void addExpectedExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpectedExpressionErrors()); + } + + public static List getMatchQueryErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("unable to use function MATCH in the requested context"); errors.add("malformed MATCH expression"); errors.add("fts5: syntax error near"); @@ -78,28 +97,60 @@ public static void addMatchQueryErrors(ExpectedErrors errors) { errors.add("fts5: column queries are not supported"); // vt0.c0 MATCH '2016456922' errors.add("fts5: phrase queries are not supported"); errors.add("unterminated string"); + + return errors; } - public static void addTableManipulationErrors(ExpectedErrors errors) { + public static void addMatchQueryErrors(ExpectedErrors errors) { + errors.addAll(getMatchQueryErrors()); + } + + public static List getTableManipulationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("unsupported frame specification"); errors.add("non-deterministic functions prohibited in CHECK constraints"); errors.addAll(Arrays.asList("subqueries prohibited in CHECK constraints", "generated columns cannot be part of the PRIMARY KEY", "must have at least one non-generated column")); + + return errors; } - public static void addQueryErrors(ExpectedErrors errors) { + public static void addTableManipulationErrors(ExpectedErrors errors) { + errors.addAll(getTableManipulationErrors()); + } + + public static List getQueryErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("ON clause references tables to its right"); + + return errors; } - public static void addInsertNowErrors(ExpectedErrors errors) { + public static void addQueryErrors(ExpectedErrors errors) { + errors.addAll(getQueryErrors()); + } + + public static List getInsertNowErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("non-deterministic use of strftime()"); errors.add("non-deterministic use of time()"); errors.add("non-deterministic use of datetime()"); errors.add("non-deterministic use of julianday()"); errors.add("non-deterministic use of date()"); + + return errors; } - public static void addInsertUpdateErrors(ExpectedErrors errors) { + public static void addInsertNowErrors(ExpectedErrors errors) { + errors.addAll(getInsertNowErrors()); + } + + public static List getInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("String or BLOB exceeds size limit"); errors.add("[SQLITE_CONSTRAINT_CHECK]"); errors.add("[SQLITE_CONSTRAINT_PRIMARYKEY]"); @@ -114,6 +165,12 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { errors.add("[SQLITE_ERROR] SQL error or missing database (no such table:"); errors.add("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch"); errors.add("no such column"); // trigger + + return errors; + } + + public static void addInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getInsertUpdateErrors()); } } diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java index 152c5bfc9..d8b0fd416 100644 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.stonedb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; @@ -7,36 +10,54 @@ public final class StoneDBErrors { private StoneDBErrors() { } - public static void addExpectedExpressionErrors(StoneDBGlobalState globalState, ExpectedErrors errors) { + public static List getExpectedExpressionErrors(StoneDBGlobalState globalState) { + ArrayList errors = new ArrayList<>(); + // java.sql.SQLException: Incorrect DATE value: '292269055-12-02' errors.add("Incorrect DATE value: "); - // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 + // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for + // column 'c1' at row 1 errors.add("Incorrect string value: "); // java.sql.SQLException: Incorrect integer value: 'ST' for column 'c1' at row 1 errors.add("Incorrect integer value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Truncated incorrect INTEGER value: '#Q' + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Truncated + // incorrect INTEGER value: '#Q' errors.add("Data truncation: Truncated incorrect INTEGER value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT value is out of range in + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT + // value is out of range in // '-((`database0`.`t0`.`c1` >> (not(`database0`.`t0`.`c1`))))' errors.add("Data truncation: BIGINT value is out of range in "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT UNSIGNED value is out of range in + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT + // UNSIGNED value is out of range in // '(`database10`.`t0`.`c0` + (`database10`.`t0`.`c0` & (not(0.5))))' errors.add("Data truncation: BIGINT UNSIGNED value is out of range in "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect time value: '0Sly6xqF0' for + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect + // time value: '0Sly6xqF0' for // column 'c1' at row 1 errors.add("Data truncation: Incorrect time value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: DOUBLE value is out of range in + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: DOUBLE + // value is out of range in // '(`database0`.`t0`.`c0` * `database0`.`t0`.`c0`)' errors.add("Data truncation: DOUBLE value is out of range in "); - // java.sql.SQLException: Numeric result of an expression is too large and cannot be handled by tianmu. + // java.sql.SQLException: Numeric result of an expression is too large and + // cannot be handled by tianmu. errors.add("Numeric result of an expression is too large and cannot be handled by tianmu."); - // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order clause' + // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order + // clause' errors.add("Unknown column "); if (globalState.getDbmsSpecificOptions().test80Version) { - // Caused by: java.sql.SQLException: Incorrect DATETIME value: '292269055-12-02 16:47:04' + // Caused by: java.sql.SQLException: Incorrect DATETIME value: '292269055-12-02 + // 16:47:04' errors.add("Incorrect DATETIME value: "); - // Caused by: java.sql.SQLException: Incorrect TIMESTAMP value: '292269055-12-02 16:47:04' + // Caused by: java.sql.SQLException: Incorrect TIMESTAMP value: '292269055-12-02 + // 16:47:04' errors.add("Incorrect TIMESTAMP value: "); } + + return errors; + } + + public static void addExpectedExpressionErrors(StoneDBGlobalState globalState, ExpectedErrors errors) { + errors.addAll(getExpectedExpressionErrors(globalState)); } } diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index fd4d23d3d..f10587a8b 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.tidb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class TiDBErrors { @@ -7,7 +10,9 @@ public final class TiDBErrors { private TiDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("DECIMAL value is out of range"); errors.add("error parsing regexp"); errors.add("BIGINT UNSIGNED value is out of range"); @@ -58,14 +63,30 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (TiDBBugs.bug44747) { errors.add("index out of range"); } + + return errors; } - public static void addExpressionHavingErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getExpressionHavingErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not in GROUP BY clause and contains nonaggregated column"); errors.add("Unknown column"); + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addExpressionHavingErrors(ExpectedErrors errors) { + errors.addAll(getExpressionHavingErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Duplicate entry"); errors.add("cannot be null"); errors.add("doesn't have a default value"); @@ -84,6 +105,12 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Incorrect decimal value"); errors.add("error parsing regexp"); errors.add("is not valid for CHARACTER SET"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java index 1a13b9cb4..69c4148de 100644 --- a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java +++ b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java @@ -1,5 +1,8 @@ package sqlancer.timescaledb.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class TimescaleDBCommon { @@ -7,7 +10,11 @@ private TimescaleDBCommon() { } - public static void addTimescaleDBErrors(ExpectedErrors errors) { + public static List getTimescaleDBErrors() { + return new ArrayList<>(); + } + public static void addTimescaleDBErrors(ExpectedErrors errors) { + errors.addAll(getTimescaleDBErrors()); } } diff --git a/src/sqlancer/yugabyte/ycql/YCQLErrors.java b/src/sqlancer/yugabyte/ycql/YCQLErrors.java index 2bcddb71c..010b2111a 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLErrors.java +++ b/src/sqlancer/yugabyte/ycql/YCQLErrors.java @@ -1,5 +1,8 @@ package sqlancer.yugabyte.ycql; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class YCQLErrors { @@ -7,7 +10,9 @@ public final class YCQLErrors { private YCQLErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Signature mismatch in call to builtin function"); errors.add("Qualified name not allowed for column reference"); errors.add("Datatype Mismatch"); @@ -16,6 +21,12 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Invalid SQL Statement"); errors.add("Order by clause contains invalid expression"); errors.add("Invalid Function Call"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); } } diff --git a/src/sqlancer/yugabyte/ysql/YSQLErrors.java b/src/sqlancer/yugabyte/ysql/YSQLErrors.java index a2cee1add..de692b352 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLErrors.java +++ b/src/sqlancer/yugabyte/ysql/YSQLErrors.java @@ -1,5 +1,8 @@ package sqlancer.yugabyte.ysql; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class YSQLErrors { @@ -7,7 +10,9 @@ public final class YSQLErrors { private YSQLErrors() { } - public static void addCommonFetchErrors(ExpectedErrors errors) { + public static List getCommonFetchErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("An I/O error occurred while sending to the backend"); errors.add("Conflicts with committed transaction"); errors.add("cannot be changed"); @@ -22,9 +27,17 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("non-integer constant in"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY position"); + + return errors; } - public static void addCommonTableErrors(ExpectedErrors errors) { + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.addAll(getCommonFetchErrors()); + } + + public static List getCommonTableErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("PRIMARY KEY containing column of type 'INET' not yet supported"); errors.add("PRIMARY KEY containing column of type 'VARBIT' not yet supported"); errors.add("PRIMARY KEY containing column of type 'INT4RANGE' not yet supported"); @@ -34,9 +47,17 @@ public static void addCommonTableErrors(ExpectedErrors errors) { errors.add("is not commutative"); // exclude errors.add("cannot be changed"); errors.add("operator requires run-time type coercion"); // exclude + + return errors; } - public static void addCommonExpressionErrors(ExpectedErrors errors) { + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.addAll(getCommonTableErrors()); + } + + public static List getCommonExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("syntax error at or near \"(\""); errors.add("does not exist"); errors.add("is not unique"); @@ -69,14 +90,22 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("a negative number raised to a non-integer power yields a complex result"); errors.add("could not determine polymorphic type because input has type unknown"); - addToCharFunctionErrors(errors); - addBitStringOperationErrors(errors); - addFunctionErrors(errors); - addCommonRangeExpressionErrors(errors); - addCommonRegexExpressionErrors(errors); + errors.addAll(getToCharFunctionErrors()); + errors.addAll(getBitStringOperationErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getCommonRangeExpressionErrors()); + errors.addAll(getCommonRegexExpressionErrors()); + + return errors; } - public static void addToCharFunctionErrors(ExpectedErrors errors) { + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + public static List getToCharFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("multiple decimal points"); errors.add("and decimal point together"); errors.add("multiple decimal points"); @@ -88,16 +117,32 @@ public static void addToCharFunctionErrors(ExpectedErrors errors) { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + + return errors; } - public static void addBitStringOperationErrors(ExpectedErrors errors) { + public static void addToCharFunctionErrors(ExpectedErrors errors) { + errors.addAll(getToCharFunctionErrors()); + } + + public static List getBitStringOperationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("cannot XOR bit strings of different sizes"); errors.add("cannot AND bit strings of different sizes"); errors.add("cannot OR bit strings of different sizes"); errors.add("must be type boolean, not type text"); + + return errors; } - public static void addFunctionErrors(ExpectedErrors errors) { + public static void addBitStringOperationErrors(ExpectedErrors errors) { + errors.addAll(getBitStringOperationErrors()); + } + + public static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("out of valid range"); // get_bit/get_byte errors.add("cannot take logarithm of a negative number"); errors.add("cannot take logarithm of zero"); @@ -109,18 +154,40 @@ public static void addFunctionErrors(ExpectedErrors errors) { errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii errors.add("negative substring length not allowed"); // substr errors.add("invalid mask length"); // set_masklen + + return errors; } - public static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + public static void addFunctionErrors(ExpectedErrors errors) { + errors.addAll(getFunctionErrors()); + } + + public static List getCommonRegexExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not a valid hexadecimal digit"); + + return errors; } - public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + public static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonRangeExpressionErrors()); + } + + public static List getCommonRangeExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("range lower bound must be less than or equal to range upper bound"); errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); errors.add("result of range union would not be contiguous"); + + return errors; + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonRangeExpressionErrors()); } public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { @@ -128,14 +195,24 @@ public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { errors.add("not found in view targetlist"); } - public static void addGroupingErrors(ExpectedErrors errors) { + public static List getGroupingErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("non-integer constant in GROUP BY"); // TODO errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("is not in select list"); errors.add("aggregate functions are not allowed in GROUP BY"); + + return errors; } - public static void addViewErrors(ExpectedErrors errors) { + public static void addGroupingErrors(ExpectedErrors errors) { + errors.addAll(getGroupingErrors()); + } + + public static List getViewErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("already exists"); errors.add("cannot drop columns from view"); errors.add("non-integer constant in ORDER BY"); // TODO @@ -147,5 +224,11 @@ public static void addViewErrors(ExpectedErrors errors) { errors.add("is not a view"); errors.add("non-integer constant in DISTINCT ON"); errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + + return errors; + } + + public static void addViewErrors(ExpectedErrors errors) { + errors.addAll(getViewErrors()); } } diff --git a/test/sqlancer/TestExpectedErrors.java b/test/sqlancer/TestExpectedErrors.java index 2781df89b..2cb938824 100644 --- a/test/sqlancer/TestExpectedErrors.java +++ b/test/sqlancer/TestExpectedErrors.java @@ -1,5 +1,6 @@ package sqlancer; +import java.util.List; import java.util.regex.Pattern; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -31,6 +32,18 @@ public void testStringSimple() { } + @Test + public void testStringsSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addAll(List.of("a", "b", "c")); + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + } + @Test public void testRegexSimple() { ExpectedErrors errors = new ExpectedErrors(); @@ -44,6 +57,41 @@ public void testRegexSimple() { } + @Test + public void testRegexesSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addAllRegexes(List.of(Pattern.compile("a\\d"), Pattern.compile("b\\D"))); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } + + @Test + public void testRegexStringSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addRegexString("a\\d"); + errors.addRegexString("b\\D"); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + } + + @Test + public void testRegexStrings() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addAllRegexStrings(List.of("a\\d", "b\\D")); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } + @Test public void testStringRealistic() { ExpectedErrors errors = new ExpectedErrors(); @@ -60,4 +108,28 @@ public void testRegexRealistic() { assertTrue(errors.errorIsExpected("PRIMARY KEY constraint was violated!")); } + @Test + public void testBuilder() { + ExpectedErrors errors = ExpectedErrors.newErrors().with("a", "b", "c").build(); + + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + errors = ExpectedErrors.newErrors().withRegex("a\\d", "b\\D").with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + errors = ExpectedErrors.newErrors().with(Pattern.compile("a\\d"), Pattern.compile("b\\D")).with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } } From 8cd8246e4cadc3e5e1c30524563424420b5c3313 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 25 Mar 2024 12:06:18 +0800 Subject: [PATCH 0841/1284] Rename `xxxExpressions()` to `xxxClauses()` for select --- src/sqlancer/clickhouse/ClickHouseToStringVisitor.java | 4 ++-- src/sqlancer/clickhouse/ast/ClickHouseSelect.java | 4 ++-- .../oracle/tlp/ClickHouseTLPAggregateOracle.java | 4 ++-- .../clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java | 4 ++-- src/sqlancer/cnosdb/CnosDBToStringVisitor.java | 4 ++-- src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java | 2 +- .../cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java | 2 +- src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java | 2 +- src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java | 2 +- src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java | 4 ++-- src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java | 4 ++-- .../gen/CockroachDBRandomQuerySynthesizer.java | 2 +- .../cockroachdb/oracle/CockroachDBNoRECOracle.java | 2 +- .../oracle/tlp/CockroachDBTLPAggregateOracle.java | 8 +++++--- .../oracle/tlp/CockroachDBTLPExtendedWhereOracle.java | 2 +- .../oracle/tlp/CockroachDBTLPHavingOracle.java | 2 +- .../cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java | 2 +- src/sqlancer/common/ast/SelectBase.java | 4 ++-- src/sqlancer/common/ast/newast/Select.java | 4 ++-- src/sqlancer/databend/DatabendToStringVisitor.java | 4 ++-- .../databend/gen/DatabendRandomQuerySynthesizer.java | 5 +++-- src/sqlancer/databend/test/DatabendNoRECOracle.java | 4 ++-- .../test/DatabendPivotedQuerySynthesisOracle.java | 2 +- .../tlp/DatabendQueryPartitioningAggregateTester.java | 2 +- .../test/tlp/DatabendQueryPartitioningHavingTester.java | 4 ++-- .../test/tlp/DatabendQueryPartitioningWhereTester.java | 2 +- src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java | 2 +- src/sqlancer/doris/oracle/DorisNoRECOracle.java | 2 +- .../doris/oracle/DorisPivotedQuerySynthesisOracle.java | 2 +- .../oracle/tlp/DorisQueryPartitioningAggregateTester.java | 2 +- .../oracle/tlp/DorisQueryPartitioningHavingTester.java | 2 +- .../oracle/tlp/DorisQueryPartitioningWhereTester.java | 2 +- src/sqlancer/doris/visitor/DorisToStringVisitor.java | 4 ++-- src/sqlancer/duckdb/DuckDBToStringVisitor.java | 4 ++-- src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java | 2 +- src/sqlancer/duckdb/test/DuckDBNoRECOracle.java | 2 +- .../test/DuckDBQueryPartitioningAggregateTester.java | 2 +- .../duckdb/test/DuckDBQueryPartitioningHavingTester.java | 2 +- .../duckdb/test/DuckDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/h2/H2QueryPartitioningWhereTester.java | 2 +- src/sqlancer/h2/H2RandomQuerySynthesizer.java | 2 +- src/sqlancer/h2/H2ToStringVisitor.java | 4 ++-- src/sqlancer/hsqldb/HSQLDBToStringVisitor.java | 4 ++-- src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java | 2 +- .../hsqldb/test/HSQLDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/materialize/MaterializeToStringVisitor.java | 4 ++-- .../materialize/gen/MaterializeRandomQueryGenerator.java | 2 +- .../materialize/oracle/MaterializeNoRECOracle.java | 3 +-- .../oracle/MaterializePivotedQuerySynthesisOracle.java | 2 +- .../oracle/tlp/MaterializeTLPAggregateOracle.java | 2 +- .../materialize/oracle/tlp/MaterializeTLPBase.java | 2 +- .../oracle/tlp/MaterializeTLPHavingOracle.java | 5 +++-- .../materialize/oracle/tlp/MaterializeTLPWhereOracle.java | 4 ++-- src/sqlancer/mysql/MySQLToStringVisitor.java | 6 +++--- src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java | 2 +- .../mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 2 +- src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java | 4 ++-- src/sqlancer/oceanbase/OceanBaseToStringVisitor.java | 6 +++--- .../oracle/OceanBasePivotedQuerySynthesisOracle.java | 2 +- .../oceanbase/oracle/OceanBaseTLPWhereOracle.java | 4 ++-- src/sqlancer/postgres/PostgresToStringVisitor.java | 4 ++-- .../postgres/gen/PostgresRandomQueryGenerator.java | 2 +- src/sqlancer/postgres/oracle/PostgresNoRECOracle.java | 2 +- .../oracle/PostgresPivotedQuerySynthesisOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPAggregateOracle.java | 2 +- src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 2 +- .../postgres/oracle/tlp/PostgresTLPHavingOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPWhereOracle.java | 4 ++-- src/sqlancer/presto/PrestoToStringVisitor.java | 4 ++-- src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java | 5 +++-- .../presto/gen/PrestoTypedExpressionGenerator.java | 5 +++-- src/sqlancer/presto/test/PrestoNoRECOracle.java | 3 +-- .../test/PrestoQueryPartitioningAggregateTester.java | 2 +- .../presto/test/PrestoQueryPartitioningHavingTester.java | 2 +- .../presto/test/PrestoQueryPartitioningWhereTester.java | 2 +- src/sqlancer/questdb/QuestDBToStringVisitor.java | 4 ++-- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 4 ++-- src/sqlancer/sqlite3/ast/SQLite3Select.java | 4 ++-- src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java | 2 +- .../oracle/SQLite3PivotedQuerySynthesisOracle.java | 2 +- .../sqlite3/oracle/SQLite3RandomQuerySynthesizer.java | 5 +++-- .../sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java | 4 ++-- .../sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java | 2 +- src/sqlancer/stonedb/StoneDBToStringVisitor.java | 4 ++-- src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java | 2 +- .../oracle/StoneDBQueryPartitioningHavingTester.java | 5 +++-- .../oracle/StoneDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java | 2 +- src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 5 +++-- src/sqlancer/tidb/oracle/TiDBDQPOracle.java | 2 +- src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java | 2 +- src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java | 2 +- src/sqlancer/tidb/visitor/TiDBToStringVisitor.java | 4 ++-- src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java | 4 ++-- .../yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java | 2 +- src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java | 4 ++-- .../yugabyte/ysql/gen/YSQLRandomQueryGenerator.java | 2 +- src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java | 2 +- .../ysql/oracle/YSQLPivotedQuerySynthesisOracle.java | 2 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java | 2 +- src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 2 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java | 2 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java | 4 ++-- 106 files changed, 160 insertions(+), 153 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index 37f4d3ab6..cebce8fc2 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -97,9 +97,9 @@ public void visit(ClickHouseSelect select, boolean inner) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByClause().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByClause()); + visit(select.getOrderByClauses()); } if (inner) { sb.append(")"); diff --git a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java index 24d489982..3904b767b 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java @@ -64,11 +64,11 @@ public ClickHouseExpression getLimitClause() { return limitClause; } - public List getOrderByClause() { + public List getOrderByClauses() { return orderByClause; } - public void setOrderByExpressions(List orderBy) { + public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index d14356a57..cedac9c51 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -25,7 +25,7 @@ public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState sta public void check() throws SQLException { super.check(); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } @@ -49,7 +49,7 @@ public void check() throws SQLException { .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java index 92195bedf..be2bd71fc 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java @@ -23,7 +23,7 @@ public ClickHouseTLPWhereOracle(ClickHouseProvider.ClickHouseGlobalState state) public void check() throws SQLException { super.check(); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } String originalQueryString = ClickHouseVisitor.asString(select); @@ -31,7 +31,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } diff --git a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java index 64f2fd8b8..44d3bf509 100644 --- a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java +++ b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java @@ -146,9 +146,9 @@ public void visit(CnosDBSelect s) { visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByExpressions()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java index df91e9a58..0c817c655 100644 --- a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java +++ b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java @@ -141,7 +141,7 @@ private int getOptimizedQueryCount(List randomTables, List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, CnosDBExpectedError.expectedErrors(), state); - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = CnosDBVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java index 1c6fe0f36..67abfdbbe 100644 --- a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java @@ -102,9 +102,9 @@ public void visit(CockroachDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java index d3679b015..09f5b165d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java @@ -56,7 +56,7 @@ public static CockroachDBSelect generateSelect(CockroachDBGlobalState globalStat select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.getOrderingTerms()); + select.setOrderByClauses(gen.getOrderingTerms()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index c627b9151..2bc1873a0 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -89,7 +89,7 @@ private int getOptimizedResult(CockroachDBExpression whereCondition, List mapped(CockroachDBAggregate aggregate) { case MIN: return aliasArgs(Arrays.asList(aggregate)); case AVG: - // List arg = Arrays.asList(new CockroachDBCast(aggregate.getExpr().get(0), + // List arg = Arrays.asList(new + // CockroachDBCast(aggregate.getExpr().get(0), // CockroachDBDataType.DECIMAL.get())); CockroachDBAggregate sum = new CockroachDBAggregate(CockroachDBAggregateFunction.SUM, aggregate.getExpr()); CockroachDBCast count = new CockroachDBCast( new CockroachDBAggregate(CockroachDBAggregateFunction.COUNT, aggregate.getExpr()), CockroachDBDataType.DECIMAL.get()); - // CockroachDBBinaryArithmeticOperation avg = new CockroachDBBinaryArithmeticOperation(sum, count, + // CockroachDBBinaryArithmeticOperation avg = new + // CockroachDBBinaryArithmeticOperation(sum, count, // CockroachDBBinaryArithmeticOperator.DIV); return aliasArgs(Arrays.asList(sum, count)); default: diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java index 7463e767f..863f056ab 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java @@ -38,7 +38,7 @@ public void check() throws SQLException { boolean allowOrderBy = Randomly.getBoolean(); if (allowOrderBy) { - select.setOrderByExpressions(gen.getOrderingTerms()); + select.setOrderByClauses(gen.getOrderingTerms()); } select.setWhereClause(combinePredicate(predicate)); String firstQueryString = CockroachDBVisitor.asString(select); diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java index d5a96d80b..7a7dd7779 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java @@ -28,7 +28,7 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java index db0ea6162..0fe935d18 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java @@ -32,7 +32,7 @@ public void check() throws SQLException { boolean allowOrderBy = Randomly.getBoolean(); if (allowOrderBy) { - select.setOrderByExpressions(gen.getOrderingTerms()); + select.setOrderByClauses(gen.getOrderingTerms()); } CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); select.setWhereClause(predicate); diff --git a/src/sqlancer/common/ast/SelectBase.java b/src/sqlancer/common/ast/SelectBase.java index 9e02a7f61..ebc3ed5c3 100644 --- a/src/sqlancer/common/ast/SelectBase.java +++ b/src/sqlancer/common/ast/SelectBase.java @@ -59,14 +59,14 @@ public List getGroupByExpressions() { return groupByExpressions; } - public void setOrderByExpressions(List orderByExpressions) { + public void setOrderByClauses(List orderByExpressions) { if (orderByExpressions == null) { throw new IllegalArgumentException(); } this.orderByExpressions = orderByExpressions; } - public List getOrderByExpressions() { + public List getOrderByClauses() { assert orderByExpressions != null; return orderByExpressions; } diff --git a/src/sqlancer/common/ast/newast/Select.java b/src/sqlancer/common/ast/newast/Select.java index 0953f289a..53520648d 100644 --- a/src/sqlancer/common/ast/newast/Select.java +++ b/src/sqlancer/common/ast/newast/Select.java @@ -24,9 +24,9 @@ public interface Select, E extends Expression, T exte Expression getLimitClause(); - List getOrderByClause(); + List getOrderByClauses(); - void setOrderByExpressions(List orderBy); + void setOrderByClauses(List orderBy); void setOffsetClause(E offsetClause); diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java index b3669e4b7..87c94d074 100644 --- a/src/sqlancer/databend/DatabendToStringVisitor.java +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -77,9 +77,9 @@ private void visit(DatabendSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index cd2427c2f..4f0b2fd00 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -64,8 +64,9 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int List> noExprColumns = new ArrayList<>(columnOfLeafNode); if (Randomly.getBoolean() && !noExprColumns.isEmpty() && !isDistinct) { - select.setOrderByExpressions(Randomly.nonEmptySubset(noExprColumns)); - // TODO (for SELECT DISTINCT, ORDER BY expressions must appear in select list) isDistinct + select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); + // TODO (for SELECT DISTINCT, ORDER BY expressions must appear in select list) + // isDistinct // 需要orderby输入每个select list，可以用数字代替比如：1,2,3... } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index d8cf09a4a..50a8c4de4 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -113,8 +113,8 @@ private int getOptimizedQueryCount(SQLConnection con, List getRectifiedQuery() throws Exception { DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); if (!isDistinct) { List> orderBys = gen.generateOrderBy(); - selectStatement.setOrderByExpressions(orderBys); + selectStatement.setOrderByClauses(orderBys); } return new SQLQueryAdapter(DatabendToStringVisitor.asString(selectStatement), errors); } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index b8463bd53..043e93963 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -58,7 +58,7 @@ public void check() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); // if (Randomly.getBooleanWithRatherLowProbability()) { - // select.setOrderByExpressions(gen.generateOrderBys()); + // select.setOrderByClauses(gen.generateOrderBys()); // } originalQuery = DatabendToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java index 2eda63ab5..dfde7755f 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java @@ -33,11 +33,11 @@ public void check() throws SQLException { // boolean orderBy = Randomly.getBoolean(); boolean orderBy = false; // 关闭order by if (orderBy) { // TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select - // select.setOrderByExpressions(gen.generateOrderBys()); + // select.setOrderByClauses(gen.generateOrderBys()); List> constants = new ArrayList<>(); constants.add(new DatabendConstant.DatabendIntConstant( Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } select.setGroupByExpressions(groupByExpression); select.setHavingClause(null); diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index a9ac6a6c3..d6fb7f0e6 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -28,7 +28,7 @@ public void check() throws SQLException { // boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); boolean orderBy = false; // if (orderBy) { //TODO 待开启 - // select.setOrderByExpressions(gen.generateOrderBys()); + // select.setOrderByClauses(gen.generateOrderBys()); // } select.setWhereClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); diff --git a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java index 3193594df..1b85d4b34 100644 --- a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java +++ b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java @@ -62,7 +62,7 @@ public static DorisSelect generateSelect(DorisGlobalState globalState, int nrCol List> noExprColumns = new ArrayList<>(columnOfLeafNode); if (Randomly.getBoolean()) { - select.setOrderByExpressions(Randomly.nonEmptySubset(noExprColumns)); + select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); } if (Randomly.getBoolean()) { select.setGroupByExpressions(noExprColumns); diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java index dab6b53ae..01fa11f2f 100644 --- a/src/sqlancer/doris/oracle/DorisNoRECOracle.java +++ b/src/sqlancer/doris/oracle/DorisNoRECOracle.java @@ -114,7 +114,7 @@ private int getOptimizedQueryCount(SQLConnection con, List List> constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } select.setJoinList(joins); int firstCount = 0; diff --git a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java index 95c79393b..4d8f3c012 100644 --- a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java @@ -71,7 +71,7 @@ protected Query getRectifiedQuery() throws Exception { List> constants = new ArrayList<>(); constants.add(new DorisConstant.DorisIntConstant( Randomly.smallNumber() % selectStatement.getFetchColumns().size() + 1)); - selectStatement.setOrderByExpressions(constants); + selectStatement.setOrderByClauses(constants); } return new SQLQueryAdapter(DorisToStringVisitor.asString(selectStatement), errors); } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java index 2e886afbe..d8076d3e5 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -65,7 +65,7 @@ public void check() throws SQLException { List> constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } originalQuery = DorisToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java index 2151e1da3..676065616 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -36,7 +36,7 @@ public void check() throws SQLException { List> constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } select.setGroupByExpressions(groupByExpression); select.setHavingClause(null); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java index e373e98b9..34ade1d3a 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -35,7 +35,7 @@ public void check() throws SQLException { List> constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } select.setWhereClause(DorisExprToNode.cast(predicate)); String firstQueryString = DorisToStringVisitor.asString(select); diff --git a/src/sqlancer/doris/visitor/DorisToStringVisitor.java b/src/sqlancer/doris/visitor/DorisToStringVisitor.java index 846955c83..f7770058f 100644 --- a/src/sqlancer/doris/visitor/DorisToStringVisitor.java +++ b/src/sqlancer/doris/visitor/DorisToStringVisitor.java @@ -147,9 +147,9 @@ private void visit(DorisSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index f7c97f3e5..37493d0f6 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -68,9 +68,9 @@ private void visit(DuckDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java index 29af42e10..aef88ef13 100644 --- a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java +++ b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java @@ -48,7 +48,7 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC select.setWhereClause(gen.generateExpression()); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index de89bfa7d..2ce6a8ad2 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -113,7 +113,7 @@ private int getFirstQueryCount(SQLConnection con, List> t select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions(new DuckDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); + select.setOrderByClauses(new DuckDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); } // select.setSelectType(SelectType.ALL); select.setJoinList(joins); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index 8d7835d29..de8bd96a0 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -58,7 +58,7 @@ public void check() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = DuckDBToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index e8988d867..b75bf02a5 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -30,7 +30,7 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java index ca9cde670..7e20e605c 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java @@ -27,7 +27,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = DuckDBToStringVisitor.asString(select); diff --git a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java index 8de4ecd37..cc09dea7a 100644 --- a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java +++ b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java @@ -24,7 +24,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = H2ToStringVisitor.asString(select); diff --git a/src/sqlancer/h2/H2RandomQuerySynthesizer.java b/src/sqlancer/h2/H2RandomQuerySynthesizer.java index 37942f0e5..dafa7835c 100644 --- a/src/sqlancer/h2/H2RandomQuerySynthesizer.java +++ b/src/sqlancer/h2/H2RandomQuerySynthesizer.java @@ -36,7 +36,7 @@ public static H2Select generateSelect(H2GlobalState globalState, int nrColumns) select.setWhereClause(gen.generateExpression()); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/h2/H2ToStringVisitor.java b/src/sqlancer/h2/H2ToStringVisitor.java index cb86ede30..ad45c66f2 100644 --- a/src/sqlancer/h2/H2ToStringVisitor.java +++ b/src/sqlancer/h2/H2ToStringVisitor.java @@ -67,9 +67,9 @@ public void visit(H2Select select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java index 722e241fa..8caf7b867 100644 --- a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java +++ b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java @@ -74,9 +74,9 @@ private void visit(HSQLDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java index ca376e348..eb08d061d 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -101,7 +101,7 @@ private int getFirstQueryCount(SQLConnection con, List> t select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions(new HSQLDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); + select.setOrderByClauses(new HSQLDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); int firstCount = 0; diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java index 0b1706583..2b793383e 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java @@ -23,7 +23,7 @@ public void check() throws Exception { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(expressionGenerator.generateOrderBys()); + select.setOrderByClauses(expressionGenerator.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = HSQLDBToStringVisitor.asString(select); diff --git a/src/sqlancer/materialize/MaterializeToStringVisitor.java b/src/sqlancer/materialize/MaterializeToStringVisitor.java index 6a3f06445..b39179cae 100644 --- a/src/sqlancer/materialize/MaterializeToStringVisitor.java +++ b/src/sqlancer/materialize/MaterializeToStringVisitor.java @@ -148,9 +148,9 @@ public void visit(MaterializeSelect s) { visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByExpressions()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java index 16f4f1be8..c8b1441fc 100644 --- a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java @@ -46,7 +46,7 @@ public static MaterializeSelect createRandomQuery(int nrColumns, MaterializeGlob } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java index fbfbcb132..bf9cc43fd 100644 --- a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java +++ b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java @@ -147,8 +147,7 @@ private int getOptimizedQueryCount(List randomTables, Lis select.setFromList(randomTables); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( - new MaterializeExpressionGenerator(state).setColumns(columns).generateOrderBy()); + select.setOrderByClauses(new MaterializeExpressionGenerator(state).setColumns(columns).generateOrderBy()); } select.setSelectType(SelectType.ALL); select.setJoinClauses(joinStatements); diff --git a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java index 07bfb470f..f4b2a1cc6 100644 --- a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java +++ b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java @@ -64,7 +64,7 @@ public SQLQueryAdapter getRectifiedQuery() throws SQLException { } List orderBy = new MaterializeExpressionGenerator(globalState).setColumns(columns) .generateOrderBy(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(MaterializeVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java index 4f5f67294..e8154bb9a 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java @@ -63,7 +63,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } originalQuery = MaterializeVisitor.asString(select); generatedQueryString = originalQuery; diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java index 4aa2196d1..dcf16dfad 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java @@ -106,7 +106,7 @@ public static MaterializeSubquery createSubquery(MaterializeGlobalState globalSt select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java index 46d6c6d74..e4328ce40 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java @@ -36,7 +36,8 @@ protected void havingCheck() throws SQLException { generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - // See https://github.com/MaterializeInc/materialize/issues/18346, have to check if predicate errors by putting + // See https://github.com/MaterializeInc/materialize/issues/18346, have to check + // if predicate errors by putting // it in SELECT first List originalColumns = select.getFetchColumns(); List checkColumns = new ArrayList<>(); @@ -48,7 +49,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } select.setHavingClause(predicate); String firstQueryString = MaterializeVisitor.asString(select); diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java index 6af0ac2cc..6cec373a9 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java @@ -25,13 +25,13 @@ public void check() throws SQLException { protected void whereCheck() throws SQLException { if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } String originalQueryString = MaterializeVisitor.asString(select); generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = MaterializeVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 757232406..c4bce3559 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -101,14 +101,14 @@ public void visit(MySQLSelect s) { visit(groupBys.get(i)); } } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - List orderBys = s.getOrderByExpressions(); + List orderBys = s.getOrderByClauses(); for (int i = 0; i < orderBys.size(); i++) { if (i != 0) { sb.append(", "); } - visit(s.getOrderByExpressions().get(i)); + visit(s.getOrderByClauses().get(i)); } } if (s.getLimitClause() != null) { diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java index 8ebaff3b6..e1a7da75f 100644 --- a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -33,7 +33,7 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) select.setWhereClause(gen.generateExpression()); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 6e50fbb33..28665c328 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -68,7 +68,7 @@ public Query getRectifiedQuery() throws SQLException { selectStatement.setModifiers(modifiers); List orderBy = new MySQLExpressionGenerator(globalState).setColumns(columns) .generateOrderBys(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(MySQLVisitor.asString(selectStatement), errors); } diff --git a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java index 4b578a24d..3d4de3873 100644 --- a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java @@ -25,9 +25,9 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = MySQLVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java index 56fef98f1..a65ae8ee3 100644 --- a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java @@ -100,14 +100,14 @@ public void visit(OceanBaseSelect s) { sb.append(" HAVING "); visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - List orderBys = s.getOrderByExpressions(); + List orderBys = s.getOrderByClauses(); for (int i = 0; i < orderBys.size(); i++) { if (i != 0) { sb.append(", "); } - visit(s.getOrderByExpressions().get(i)); + visit(s.getOrderByClauses().get(i)); } } if (s.getLimitClause() != null) { diff --git a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java index 48fe1002e..8fa0a4bc0 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java @@ -71,7 +71,7 @@ public Query getRectifiedQuery() throws SQLException { selectStatement.setOffsetClause(offsetClause); } List orderBy = generateOrderBy(columns); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(OceanBaseVisitor.asString(selectStatement), errors); } diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java index 73a0c3a6f..7c8c64e3c 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java @@ -25,9 +25,9 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = OceanBaseVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index d1766e6c8..17327ab9c 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -170,9 +170,9 @@ public void visit(PostgresSelect s) { visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByExpressions()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java index d1527ca8b..7fc5049cd 100644 --- a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java @@ -45,7 +45,7 @@ public static PostgresSelect createRandomQuery(int nrColumns, PostgresGlobalStat } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index f4bc10e74..2ad7cc2b7 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -144,7 +144,7 @@ private int getOptimizedQueryCount(List randomTables, List

orderBy = new PostgresExpressionGenerator(globalState).setColumns(columns) .generateOrderBy(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(PostgresVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index a1eaa43c7..a1ae5597e 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -61,7 +61,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } originalQuery = PostgresVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 5e349b5dc..2e34b1f87 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -103,7 +103,7 @@ public static PostgresSubquery createSubquery(PostgresGlobalState globalState, S select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java index 3a1275299..6af683903 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java @@ -36,7 +36,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } select.setHavingClause(predicate); String firstQueryString = PostgresVisitor.asString(select); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java index b043f6981..d8aeb4938 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java @@ -24,12 +24,12 @@ public void check() throws SQLException { protected void whereCheck() throws SQLException { if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } String originalQueryString = PostgresVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = PostgresVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/presto/PrestoToStringVisitor.java b/src/sqlancer/presto/PrestoToStringVisitor.java index 1342ff635..f840cf850 100644 --- a/src/sqlancer/presto/PrestoToStringVisitor.java +++ b/src/sqlancer/presto/PrestoToStringVisitor.java @@ -99,9 +99,9 @@ private void visit(PrestoSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java index 167c1f222..2d18311b6 100644 --- a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java +++ b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java @@ -50,7 +50,7 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC select.setWhereClause(gen.generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull())); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); @@ -61,7 +61,8 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC } // if (Randomly.getBoolean()) { // select.setOffsetClause( - // PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + // PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, + // Integer.MAX_VALUE))); // } if (Randomly.getBoolean()) { select.setHavingClause(gen.generateHavingClause()); diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java index 7afb8da29..79e657fda 100644 --- a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -342,7 +342,7 @@ private PrestoSelect generateSubquery(List columns) { Node predicate = typedExpressionGenerator.generatePredicate(); select.setWhereClause(predicate); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions(typedExpressionGenerator.generateOrderBys()); + select.setOrderByClauses(typedExpressionGenerator.generateOrderBys()); } List> joins = PrestoJoin.getJoins(tableList, globalState); select.setJoinList(joins); @@ -741,7 +741,8 @@ public static BinaryOperatorNode.Operator getRandomForType(PrestoSchema.PrestoCo case ARRAY: case INTERVAL_YEAR_TO_MONTH: case INTERVAL_DAY_TO_SECOND: - // return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, IS_DISTINCT_FROM, + // return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, + // IS_DISTINCT_FROM, // IS_NOT_DISTINCT_FROM); default: return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, IS_DISTINCT_FROM, IS_NOT_DISTINCT_FROM); diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java index a63d912b0..da09b5a01 100644 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -114,8 +114,7 @@ private int getFirstQueryCount(SQLConnection con, List> t select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( - new PrestoTypedExpressionGenerator(state).setColumns(columns).generateOrderBys()); + select.setOrderByClauses(new PrestoTypedExpressionGenerator(state).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); int firstCount = 0; diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java index 527d5d648..0f10a22d4 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java @@ -54,7 +54,7 @@ public void check() throws SQLException { aggregateFunction); select.setFetchColumns(List.of(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = PrestoToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java index b3ff3beaa..ad11806bf 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java @@ -31,7 +31,7 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java index 093c27108..cfff823c5 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java @@ -28,7 +28,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = PrestoToStringVisitor.asString(select); diff --git a/src/sqlancer/questdb/QuestDBToStringVisitor.java b/src/sqlancer/questdb/QuestDBToStringVisitor.java index 295a6d910..7b2c889d2 100644 --- a/src/sqlancer/questdb/QuestDBToStringVisitor.java +++ b/src/sqlancer/questdb/QuestDBToStringVisitor.java @@ -49,9 +49,9 @@ private void visit(QuestDBSelect select) { // sb.append(" HAVING "); // visit(select.getHavingClause()); // } - // if (!select.getOrderByExpressions().isEmpty()) { + // if (!select.getOrderByClauses().isEmpty()) { // sb.append(" ORDER BY "); - // visit(select.getOrderByExpressions()); + // visit(select.getOrderByClauses()); // } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 475c16335..b8b860943 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -148,9 +148,9 @@ public void visit(SQLite3Select s, boolean inner) { sb.append(" HAVING "); visit(s.getHavingClause()); } - if (!s.getOrderByClause().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByClause()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 5ff008709..36beb5fd8 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -86,11 +86,11 @@ public SQLite3Expression getLimitClause() { return limitClause; } - public List getOrderByClause() { + public List getOrderByClauses() { return orderByClause; } - public void setOrderByExpressions(List orderBy) { + public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 45ec6631d..2ec2e392f 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -127,7 +127,7 @@ private Function getOptimizedQuery(SQLite3Select se SQLite3Expression randomWhereCondition) throws SQLException { boolean useAggregate = Randomly.getBoolean(); if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (useAggregate) { select.setFetchColumns(Arrays.asList(new SQLite3Aggregate(Collections.emptyList(), diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 4021c88e9..98a30dc23 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -99,7 +99,7 @@ public SQLite3Select getQuery() throws SQLException { } /* PQS does not check for ordering, so we can generate any ORDER BY clause */ List orderBy = new SQLite3ExpressionGenerator(globalState).generateOrderBys(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); if (!groupByClause.isEmpty() && Randomly.getBoolean()) { selectStatement.setHavingClause(generateRectifiedExpression(columns, pivotRow, true)); } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java index 4447ea2fb..d3d7ded31 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java @@ -102,7 +102,8 @@ public static SQLite3Expression generate(SQLite3GlobalState globalState, int siz select.setFromList(SQLite3Common.getTableRefs(tables, s)); // TODO: no values are referenced from this sub query yet // if (Randomly.getBooleanWithSmallProbability()) { - // select.getFromList().add(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)); + // select.getFromList().add(SQLite3RandomQuerySynthesizer.generate(globalState, + // Randomly.smallNumber() + 1)); // } // WHERE @@ -121,7 +122,7 @@ public static SQLite3Expression generate(SQLite3GlobalState globalState, int siz boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { // ORDER BY - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBooleanWithRatherLowProbability()) { // LIMIT diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 849a8640f..4d1dd8529 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -54,7 +54,7 @@ public void check() throws SQLException { List from = SQLite3Common.getTableRefs(targetTables.getTables(), s); select.setFromList(from); if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } String originalQuery = SQLite3Visitor.asString(select); generatedQueryString = originalQuery; @@ -120,7 +120,7 @@ private SQLite3Select getSelect(SQLite3Aggregate aggregate, List(joins)); select.setFromList(new ArrayList<>(tableReferenceNodeList)); select.setWhereClause(gen.generateExpression()); - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); } diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java index 2465494ef..eb1482d7f 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -47,7 +47,7 @@ public void check() throws Exception { select.setFetchColumns(fetchColumns); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } String originalQuery = StoneDBToStringVisitor.asString(select); diff --git a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java index 6cd5b3bc9..ebf1f9967 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java @@ -55,7 +55,7 @@ public void check() throws Exception { select.setJoinList(joins); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( + select.setOrderByClauses( new StoneDBExpressionGenerator(globalState).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 744d52f55..d6efe30d1 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -112,7 +112,7 @@ private int getOptimizedQueryCount(SQLConnection con, List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - // specific to combined query string, will produce the same result as original query string in logic + // specific to combined query string, will produce the same result as original + // query string in logic select.setHavingClause(predicate); String firstQueryString = StoneDBToStringVisitor.asString(select); select.setHavingClause(negatedPredicate); diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java index 2a8f2b4d4..4c9267f68 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java @@ -23,7 +23,7 @@ public void check() throws Exception { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = StoneDBToStringVisitor.asString(select); diff --git a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java index f0609ad56..1b1599ac2 100644 --- a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java +++ b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java @@ -41,7 +41,7 @@ public static TiDBSelect generateSelect(TiDBGlobalState globalState, int nrColum select.setWhereClause(gen.generateExpression()); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java index 4e56dd5fa..4fd5c8c54 100644 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -63,7 +63,7 @@ public void check() throws SQLException { select.setWhereClause(gen.generateExpression()); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(select.getFetchColumns()); @@ -115,7 +115,8 @@ protected boolean mutateJoin() { TiDBJoin join = (TiDBJoin) Randomly.fromList(select.getJoinList()); // CROSS does not need ON Condition, while other joins do - // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + // To avoid Null pointer, generating a new new condition when mutating CROSS to + // other joins if (join.getJoinType() == JoinType.CROSS) { List columns = new ArrayList<>(); columns.addAll(((TiDBTableReference) join.getLeftTable()).getTable().getColumns()); diff --git a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java index e35e4ee70..17b288b7d 100644 --- a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java @@ -54,7 +54,7 @@ public void check() throws SQLException { select.setWhereClause(gen.generateExpression()); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(gen.generateExpression()); diff --git a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java index 23b3cb035..76e1a3c46 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java @@ -29,7 +29,7 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java index f3640800e..e437b2856 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java @@ -66,7 +66,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = TiDBVisitor.asString(select); diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index e9c199407..67c7cda31 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -74,9 +74,9 @@ public void visit(TiDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } } diff --git a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java index ddc7e6e8a..59adf3e6b 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java +++ b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java @@ -41,9 +41,9 @@ private void visit(YCQLSelect select) { sb.append(" WHERE "); visit(select.getWhereClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java index 7a10009ad..05f9ec2dd 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java @@ -38,7 +38,7 @@ public static YCQLSelect generateSelect(YCQLGlobalState globalState, int nrColum select.setWhereClause(gen.generateExpression()); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(Randomly.nonEmptySubset(select.getFetchColumns())); diff --git a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java index 39732b728..eef4c7d9f 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java +++ b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java @@ -136,9 +136,9 @@ public void visit(YSQLSelect s) { visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByExpressions()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java index 03109227d..02e87dffd 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java @@ -45,7 +45,7 @@ public static YSQLSelect createRandomQuery(int nrColumns, YSQLGlobalState global } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java index 3ae27fd73..37a6491ad 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -141,7 +141,7 @@ private int getOptimizedQueryCount(List randomTables, List orderBy = new YSQLExpressionGenerator(globalState).setColumns(columns).generateOrderBy(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(YSQLVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java index 537a074ae..6e9b74f87 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -61,7 +61,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } originalQuery = YSQLVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java index 863e85968..fa63ffc1f 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -53,7 +53,7 @@ public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java index 2302143c6..d4a398cf2 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java @@ -43,7 +43,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } select.setHavingClause(predicate); String firstQueryString = YSQLVisitor.asString(select); diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java index 0af8095d0..0d61dd8c5 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java @@ -24,12 +24,12 @@ public void check() throws SQLException { protected void whereCheck() throws SQLException { if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } String originalQueryString = YSQLVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = YSQLVisitor.asString(select); select.setWhereClause(negatedPredicate); From d8f3899167b7b9683b638f2b115a2edfb1edd14e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 26 Mar 2024 17:41:21 +0800 Subject: [PATCH 0842/1284] Downgrade DuckDB from 5.1 to 4.0 --- .github/workflows/main.yml | 2 ++ pom.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af92103d7..b063ce6d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,12 +7,14 @@ on: paths: - 'src/**' - '.github/workflows/*.yml' + - 'pom.xml' pull_request: branches: - main paths: - 'src/**' - '.github/workflows/*.yml' + - 'pom.xml' jobs: misc: diff --git a/pom.xml b/pom.xml index 292378933..2b34d6c27 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 0.5.1 + 0.4.0 com.facebook.presto From b68bc6e6f788b21b3b383fecb0c7fba9bf24f10d Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 26 Mar 2024 17:42:07 +0800 Subject: [PATCH 0843/1284] Include NoREC in DuckDB unit tests --- .github/workflows/main.yml | 4 +++- test/sqlancer/dbms/TestDuckDBNoREC.java | 16 ++++++++++++++++ .../dbms/{TestDuckDB.java => TestDuckDBTLP.java} | 4 ++-- 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 test/sqlancer/dbms/TestDuckDBNoREC.java rename test/sqlancer/dbms/{TestDuckDB.java => TestDuckDBTLP.java} (90%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b063ce6d8..71425b0e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -214,7 +214,9 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: DuckDB Tests - run: mvn -Dtest=TestDuckDB test + run: | + mvn -Dtest=TestDuckDBTLP test + mvn -Dtest=TestDuckDBNoREC test h2: name: DBMS Tests (H2) diff --git a/test/sqlancer/dbms/TestDuckDBNoREC.java b/test/sqlancer/dbms/TestDuckDBNoREC.java new file mode 100644 index 000000000..dea1c70dd --- /dev/null +++ b/test/sqlancer/dbms/TestDuckDBNoREC.java @@ -0,0 +1,16 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestDuckDBNoREC { + @Test + public void testDuckDBNoREC() { + // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "duckdb", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDBTLP.java similarity index 90% rename from test/sqlancer/dbms/TestDuckDB.java rename to test/sqlancer/dbms/TestDuckDBTLP.java index d36d8610f..322e3eeb4 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDBTLP.java @@ -6,10 +6,10 @@ import sqlancer.Main; -public class TestDuckDB { +public class TestDuckDBTLP { @Test - public void testDuckDB() { + public void testDuckDBTLP() { // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, From a5b25f700c24e60f7f7ee408475613005a9be9f8 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Tue, 26 Mar 2024 23:01:39 +0800 Subject: [PATCH 0844/1284] Update SQLite3 tests (#917) * Update SQLite3 tests * Update CI to run when tests/ is updated --- .github/workflows/main.yml | 5 ++++- ...{TestSQLite3.java => TestSQLiteNoREC.java} | 7 +++---- test/sqlancer/dbms/TestSQLiteTLP.java | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) rename test/sqlancer/dbms/{TestSQLite3.java => TestSQLiteNoREC.java} (53%) create mode 100644 test/sqlancer/dbms/TestSQLiteTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af92103d7..25016aa96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,12 +6,14 @@ on: - main paths: - 'src/**' + - 'test/**' - '.github/workflows/*.yml' pull_request: branches: - main paths: - 'src/**' + - 'test/**' - '.github/workflows/*.yml' jobs: @@ -383,7 +385,8 @@ jobs: - name: SQLite Tests run: | mvn -Dtest=TestSQLitePQS test - mvn -Dtest=TestSQLite3 test + mvn -Dtest=TestSQLiteTLP test + mvn -Dtest=TestSQLiteNoREC test sqlite-qpg: name: QPG Tests (SQLite) diff --git a/test/sqlancer/dbms/TestSQLite3.java b/test/sqlancer/dbms/TestSQLiteNoREC.java similarity index 53% rename from test/sqlancer/dbms/TestSQLite3.java rename to test/sqlancer/dbms/TestSQLiteNoREC.java index f26b449c1..aa7741659 100644 --- a/test/sqlancer/dbms/TestSQLite3.java +++ b/test/sqlancer/dbms/TestSQLiteNoREC.java @@ -6,13 +6,12 @@ import sqlancer.Main; -public class TestSQLite3 { +public class TestSQLiteNoREC { @Test - public void testSqlite() { - // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 + public void testSqliteNoREC() { assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", "0", "sqlite3" })); + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", "NoREC" })); } } diff --git a/test/sqlancer/dbms/TestSQLiteTLP.java b/test/sqlancer/dbms/TestSQLiteTLP.java new file mode 100644 index 000000000..dfeb83193 --- /dev/null +++ b/test/sqlancer/dbms/TestSQLiteTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLiteTLP { + + @Test + public void testSqliteTLP() { + // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 + assertEquals(0, + Main.executeMain(new String[] {"--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", + "QUERY_PARTITIONING"})); + } +} From 82bb0c30510bf1ef7e4fcb6215fda5a9d9add2b0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 26 Mar 2024 23:53:14 +0800 Subject: [PATCH 0845/1284] Fix a formatting error (#930) --- test/sqlancer/dbms/TestSQLiteTLP.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sqlancer/dbms/TestSQLiteTLP.java b/test/sqlancer/dbms/TestSQLiteTLP.java index dfeb83193..7d90fd93c 100644 --- a/test/sqlancer/dbms/TestSQLiteTLP.java +++ b/test/sqlancer/dbms/TestSQLiteTLP.java @@ -12,8 +12,8 @@ public class TestSQLiteTLP { public void testSqliteTLP() { // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 assertEquals(0, - Main.executeMain(new String[] {"--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", - "QUERY_PARTITIONING"})); + "QUERY_PARTITIONING" })); } } From 4e33b7e1e2b69c880fa122b34938b1d28149767b Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:24:44 +0800 Subject: [PATCH 0846/1284] [WIP] Generic NoREC oracle (#928) * Add NoRECGenerator interface * Add `toSum` method for Select interface Used in the NoREC oracle to aggregate unoptimized query results * Add `getRandomTableNonEmptyTables` function * Add generic NoREC oracle * Check for null arguments in NoREC oracle constructor * Update NoREC oracle implementation and add comments for generator --- src/sqlancer/common/gen/NoRECGenerator.java | 49 ++++++ src/sqlancer/common/oracle/NoRECOracle.java | 141 ++++++++++++++++++ .../common/oracle/TestOracleUtils.java | 22 +++ 3 files changed, 212 insertions(+) create mode 100644 src/sqlancer/common/gen/NoRECGenerator.java create mode 100644 src/sqlancer/common/oracle/NoRECOracle.java create mode 100644 src/sqlancer/common/oracle/TestOracleUtils.java diff --git a/src/sqlancer/common/gen/NoRECGenerator.java b/src/sqlancer/common/gen/NoRECGenerator.java new file mode 100644 index 000000000..f57f7c20f --- /dev/null +++ b/src/sqlancer/common/gen/NoRECGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface NoRECGenerator, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + NoRECGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + Select generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + /** + * Generates a query string that is likely to be optimized by the DBMS. + * + * @param select + * the base select expression used to generate the query + * @param whereCondition + * a condition where records will be checked with + * @param shouldUseAggregate + * whether to aggregate the record counts (`true`) or display records as is (`false`) + * + * @return a query string to be executed + */ + String generateOptimizedQueryString(Select select, E whereCondition, boolean shouldUseAggregate); + + /** + * Generates a query string that is unlikely to be optimized by the DBMS. + * + * @param select + * the base select expression used to generate the query + * @param whereCondition + * the condition each record will be checked with + * + * @return a query string to be executed + */ + String generateUnoptimizedQueryString(Select select, E whereCondition); +} diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java new file mode 100644 index 000000000..1fbad671a --- /dev/null +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -0,0 +1,141 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class NoRECOracle, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + + private NoRECGenerator gen; + private final ExpectedErrors errors; + + private String lastQueryString; + + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + Select select = gen.generateSelect(); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + + E randomWhereCondition = gen.generateBooleanExpression(); + + boolean shouldUseAggregate = Randomly.getBoolean(); + String optimizedQueryString = gen.generateOptimizedQueryString(select, randomWhereCondition, + shouldUseAggregate); + lastQueryString = optimizedQueryString; + + String unoptimizedQueryString = gen.generateUnoptimizedQueryString(select, randomWhereCondition); + + int optimizedCount = shouldUseAggregate ? extractCounts(optimizedQueryString, errors, state) + : countRows(optimizedQueryString, errors, state); + int unoptimizedCount = extractCounts(optimizedQueryString, errors, state); + + if (optimizedCount == -1 || unoptimizedCount == -1) { + throw new IgnoreMeException(); + } + + if (unoptimizedCount != optimizedCount) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, + unoptimizedCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", optimizedCount, + unoptimizedCount, firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + @Override + public String getLastQueryString() { + return lastQueryString; + } + + private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(queryString); + } + + int count = 0; + try (SQLancerResultSet rs = q.executeAndGet(state)) { + if (rs == null) { + return -1; + } else { + try { + while (rs.next()) { + count++; + } + } catch (SQLException e) { + count = -1; + } + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw (IgnoreMeException) e; + } + throw new AssertionError(q.getQueryString(), e); + } + return count; + } + + private int extractCounts(String queryString, ExpectedErrors errors, SQLGlobalState state) { + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(queryString); + } + + int count = 0; + try (SQLancerResultSet rs = q.executeAndGet(state)) { + if (rs == null) { + return -1; + } else { + try { + while (rs.next()) { + count += rs.getInt(1); + } + } catch (SQLException e) { + count = -1; + } + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw (IgnoreMeException) e; + } + throw new AssertionError(q.getQueryString(), e); + } + return count; + } + +} diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java new file mode 100644 index 000000000..b22cc89f5 --- /dev/null +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -0,0 +1,22 @@ +package sqlancer.common.oracle; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public final class TestOracleUtils { + + private TestOracleUtils() { + } + + public static , C extends AbstractTableColumn> AbstractTables getRandomTableNonEmptyTables( + AbstractSchema schema) { + if (schema.getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables())); + } +} From 0c6dcbd3a3f209cb1f52f5c5013b7a45547a7c2a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 27 Mar 2024 11:47:52 +0800 Subject: [PATCH 0847/1284] Remove the TiDB incremental ANALYZE feature (#931) This feature currently fails the CI: "the incremental analyze feature has already been removed in TiDB v7.5.0, so this will have no effect." --- src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 8bcea30fb..f78878ac0 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -17,11 +17,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx ExpectedErrors errors = new ExpectedErrors(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); - StringBuilder sb = new StringBuilder("ANALYZE "); - if (analyzeIndex && Randomly.getBoolean()) { - sb.append("INCREMENTAL "); - } - sb.append("TABLE "); + StringBuilder sb = new StringBuilder("ANALYZE TABLE "); sb.append(table.getName()); if (analyzeIndex) { sb.append(" INDEX "); From 419038a7f634d136feffb2c8e0ea599ac5fce7da Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 27 Mar 2024 11:10:21 +0800 Subject: [PATCH 0848/1284] Add expected insert errors for TiDB v7.5.1 --- src/sqlancer/tidb/TiDBErrors.java | 3 +++ src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java | 1 + 2 files changed, 4 insertions(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index f10587a8b..4f45debce 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -29,6 +29,7 @@ public static List getExpressionErrors() { errors.add("doesn't have a default value"); // default errors.add("is not valid for CHARACTER SET"); errors.add("DOUBLE value is out of range"); + errors.add("Result of space() was larger than max_allowed_packet"); errors.add("Data truncat"); errors.add("Truncated incorrect FLOAT value"); @@ -105,6 +106,8 @@ public static List getInsertErrors() { errors.add("Incorrect decimal value"); errors.add("error parsing regexp"); errors.add("is not valid for CHARACTER SET"); + errors.add("for function inet_aton"); + errors.add("'Empty pattern is invalid' from regexp"); return errors; } diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 535cf973f..4d0807d4b 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -33,6 +33,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("A PRIMARY must include all columns in the table's partitioning function"); errors.add("key was too long"); errors.add("Duplicate entry"); + errors.add("has a partitioning function dependency and cannot be dropped or renamed"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); From 1c47e40725ec598bd0753abe86981b8a26e25d7f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 27 Mar 2024 11:11:04 +0800 Subject: [PATCH 0849/1284] Pin TiDB version in CI to v7.5.1 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f618ab1ca..786593077 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -459,7 +459,7 @@ jobs: - name: Set up TiDB run: | docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v7.5.1 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" From 56c5e98ca20da7bf91f28553b4d8b694237cacd8 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Thu, 28 Mar 2024 08:15:15 +0800 Subject: [PATCH 0850/1284] [SQLite3] Use generic NoREC implementation (#933) * Implement Join interface for SQlite3 Join * Implement Select interface for SQLite3 Select * Fix typo in NoREC oracle * Add logging to NoREC oracle * Add reproducer to NoREC oracle * Implement NoRECGenerator interface for SQLite3 * Use generic NoREC oracle for SQLite3 --- src/sqlancer/common/ast/newast/Join.java | 2 - src/sqlancer/common/oracle/NoRECOracle.java | 41 +++- .../sqlite3/ast/SQLite3Expression.java | 9 +- src/sqlancer/sqlite3/ast/SQLite3Select.java | 30 ++- .../gen/SQLite3ExpressionGenerator.java | 77 +++++++- .../sqlite3/oracle/SQLite3NoRECOracle.java | 179 ++---------------- 6 files changed, 168 insertions(+), 170 deletions(-) diff --git a/src/sqlancer/common/ast/newast/Join.java b/src/sqlancer/common/ast/newast/Join.java index b1d1ed280..90935b338 100644 --- a/src/sqlancer/common/ast/newast/Join.java +++ b/src/sqlancer/common/ast/newast/Join.java @@ -6,8 +6,6 @@ public interface Join, T extends AbstractTable, C extends AbstractTableColumn> extends Expression { - T getTable(); - Expression getOnClause(); void setOnClause(E onClause); diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 1fbad671a..aac047b2c 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -1,9 +1,12 @@ package sqlancer.common.oracle; import java.sql.SQLException; +import java.util.Objects; +import java.util.function.Function; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.SQLGlobalState; import sqlancer.common.ast.newast.Expression; import sqlancer.common.ast.newast.Join; @@ -25,8 +28,24 @@ public class NoRECOracle, E extends Expression, S ext private NoRECGenerator gen; private final ExpectedErrors errors; + private Reproducer reproducer; private String lastQueryString; + private static class NoRECReproducer> implements Reproducer { + private final Function optimizedQuery; + private final Function unoptimizedQuery; + + NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery) { + this.optimizedQuery = optimizedQuery; + this.unoptimizedQuery = unoptimizedQuery; + } + + @Override + public boolean bugStillTriggers(G globalState) { + return !Objects.equals(optimizedQuery.apply(globalState), unoptimizedQuery.apply(globalState)); + } + } + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -34,10 +53,12 @@ public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expec this.state = state; this.gen = gen; this.errors = expectedErrors; + this.reproducer = null; } @Override public void check() throws SQLException { + reproducer = null; S schema = state.getSchema(); AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); gen = gen.setTablesAndColumns(targetTables); @@ -52,18 +73,31 @@ public void check() throws SQLException { String optimizedQueryString = gen.generateOptimizedQueryString(select, randomWhereCondition, shouldUseAggregate); lastQueryString = optimizedQueryString; + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(optimizedQueryString); + } String unoptimizedQueryString = gen.generateUnoptimizedQueryString(select, randomWhereCondition); + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(unoptimizedQueryString); + } int optimizedCount = shouldUseAggregate ? extractCounts(optimizedQueryString, errors, state) : countRows(optimizedQueryString, errors, state); - int unoptimizedCount = extractCounts(optimizedQueryString, errors, state); + int unoptimizedCount = extractCounts(unoptimizedQueryString, errors, state); if (optimizedCount == -1 || unoptimizedCount == -1) { throw new IgnoreMeException(); } if (unoptimizedCount != optimizedCount) { + Function optimizedQuery = state -> shouldUseAggregate + ? extractCounts(optimizedQueryString, errors, state) + : countRows(optimizedQueryString, errors, state); + + Function unoptimizedQuery = state -> extractCounts(unoptimizedQueryString, errors, state); + reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery); + String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, @@ -81,6 +115,11 @@ public String getLastQueryString() { return lastQueryString; } + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 8f99da8e5..d84f399a4 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -6,6 +6,7 @@ import sqlancer.IgnoreMeException; import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; import sqlancer.common.visitor.BinaryOperation; import sqlancer.common.visitor.UnaryOperation; import sqlancer.sqlite3.SQLite3CollateHelper; @@ -18,7 +19,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public abstract class SQLite3Expression { +public abstract class SQLite3Expression implements Expression { public static class SQLite3TableReference extends SQLite3Expression { @@ -128,7 +129,8 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } - public static class Join extends SQLite3Expression { + public static class Join extends SQLite3Expression + implements sqlancer.common.ast.newast.Join { public enum JoinType { INNER, CROSS, OUTER, NATURAL, RIGHT, FULL; @@ -163,6 +165,7 @@ public SQLite3Table getTable() { return table; } + @Override public SQLite3Expression getOnClause() { return onClause; } @@ -176,6 +179,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { return null; } + @Override public void setOnClause(SQLite3Expression onClause) { this.onClause = onClause; } @@ -183,7 +187,6 @@ public void setOnClause(SQLite3Expression onClause) { public void setType(JoinType type) { this.type = type; } - } public static class Subquery extends SQLite3Expression { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 36beb5fd8..57eb91b0b 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -4,9 +4,15 @@ import java.util.Collections; import java.util.List; +import sqlancer.common.ast.newast.Select; +import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3Select extends SQLite3Expression { +public class SQLite3Select extends SQLite3Expression + implements Select { private SelectType fromOptions = SelectType.ALL; private List fromList = Collections.emptyList(); @@ -54,66 +60,82 @@ public void setFromOptions(SelectType fromOptions) { this.fromOptions = fromOptions; } + @Override public List getFromList() { return fromList; } + @Override public void setFromList(List fromList) { this.fromList = fromList; } + @Override public SQLite3Expression getWhereClause() { return whereClause; } + @Override public void setWhereClause(SQLite3Expression whereClause) { this.whereClause = whereClause; } + @Override public void setGroupByClause(List groupByClause) { this.groupByClause = groupByClause; } + @Override public List getGroupByClause() { return groupByClause; } + @Override public void setLimitClause(SQLite3Expression limitClause) { this.limitClause = limitClause; } + @Override public SQLite3Expression getLimitClause() { return limitClause; } + @Override public List getOrderByClauses() { return orderByClause; } + @Override public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } + @Override public void setOffsetClause(SQLite3Expression offsetClause) { this.offsetClause = offsetClause; } + @Override public SQLite3Expression getOffsetClause() { return offsetClause; } + @Override public void setFetchColumns(List fetchColumns) { this.fetchColumns = fetchColumns; } + @Override public List getFetchColumns() { return fetchColumns; } + @Override public void setJoinClauses(List joinStatements) { this.joinStatements = joinStatements; } + @Override public List getJoinClauses() { return joinStatements; } @@ -124,13 +146,19 @@ public SQLite3CollateSequence getExplicitCollateSequence() { return null; } + @Override public void setHavingClause(SQLite3Expression havingClause) { this.havingClause = havingClause; } + @Override public SQLite3Expression getHavingClause() { assert orderByClause != null; return havingClause; } + @Override + public String asString() { + return SQLite3Visitor.asString(this); + } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 89e7e5860..7b2b164e6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -7,7 +7,10 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Select; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; @@ -31,12 +34,14 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; import sqlancer.sqlite3.ast.SQLite3Function.ComputableFunction; import sqlancer.sqlite3.ast.SQLite3RowValueExpression; +import sqlancer.sqlite3.ast.SQLite3Select; import sqlancer.sqlite3.ast.SQLite3UnaryOperation; import sqlancer.sqlite3.ast.SQLite3UnaryOperation.UnaryOperator; import sqlancer.sqlite3.oracle.SQLite3RandomQuerySynthesizer; @@ -45,12 +50,14 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3RowValue; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3ExpressionGenerator implements ExpressionGenerator { +public class SQLite3ExpressionGenerator implements ExpressionGenerator, + NoRECGenerator { private SQLite3RowValue rw; private final SQLite3GlobalState globalState; private boolean tryToGenerateKnownResult; private List columns = Collections.emptyList(); + private List targetTables; private final Randomly r; private boolean deterministicOnly; private boolean allowMatchClause; @@ -63,6 +70,7 @@ public SQLite3ExpressionGenerator(SQLite3ExpressionGenerator other) { this.globalState = other.globalState; this.tryToGenerateKnownResult = other.tryToGenerateKnownResult; this.columns = new ArrayList<>(other.columns); + this.targetTables = other.targetTables; this.r = other.r; this.deterministicOnly = other.deterministicOnly; this.allowMatchClause = other.allowMatchClause; @@ -702,4 +710,71 @@ public SQLite3Expression generateResultKnownExpression() { return expr; } + @Override + public SQLite3ExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.targetTables = targetTables.getTables(); + gen.columns = targetTables.getColumns(); + return gen; + } + + @Override + public SQLite3Expression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public SQLite3Select generateSelect() { + return new SQLite3Select(); + } + + @Override + public List getRandomJoinClauses() { + return getRandomJoinClauses(targetTables); + } + + @Override + public List getTableRefs() { + List tableRefs = new ArrayList<>(); + for (SQLite3Table t : targetTables) { + SQLite3TableReference tableRef; + if (Randomly.getBooleanWithSmallProbability() && !globalState.getSchema().getIndexNames().isEmpty()) { + tableRef = new SQLite3TableReference(globalState.getSchema().getRandomIndexOrBailout(), t); + } else { + tableRef = new SQLite3TableReference(t); + } + tableRefs.add(tableRef); + } + return tableRefs; + } + + @Override + public String generateOptimizedQueryString(Select select, + SQLite3Expression whereCondition, boolean shouldUseAggregate) { + if (Randomly.getBoolean()) { + select.setOrderByClauses(generateOrderBys()); + } + if (shouldUseAggregate) { + select.setFetchColumns(Arrays.asList(new SQLite3Aggregate(Collections.emptyList(), + SQLite3Aggregate.SQLite3AggregateFunction.COUNT_ALL))); + } else { + SQLite3ColumnName aggr = new SQLite3ColumnName(SQLite3Column.createDummy("*"), null); + select.setFetchColumns(Arrays.asList(aggr)); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(Select select, + SQLite3Expression whereCondition) { + SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, + whereCondition); + SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); + select.setFetchColumns(Arrays.asList(asText)); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ")"; + } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 2ec2e392f..27e0229a5 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -1,194 +1,49 @@ package sqlancer.sqlite3.oracle; import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.function.Function; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Visitor; -import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.Join; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; -import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3NoRECOracle extends NoRECBase implements TestOracle { +public class SQLite3NoRECOracle implements TestOracle { - private static final int NO_VALID_RESULT = -1; - private final SQLite3Schema s; - private SQLite3ExpressionGenerator gen; - private Reproducer reproducer; - - private static class SQLite3NoRECReproducer implements Reproducer { - private final Function optimizedQuery; - private final Function unoptimizedQuery; - - SQLite3NoRECReproducer(Function optimizedQuery, - Function unoptimizedQuery) { - this.optimizedQuery = optimizedQuery; - this.unoptimizedQuery = unoptimizedQuery; - } - - @Override - public boolean bugStillTriggers(SQLite3GlobalState globalState) { - return !Objects.equals(optimizedQuery.apply(globalState), unoptimizedQuery.apply(globalState)); - } - } + NoRECOracle oracle; public SQLite3NoRECOracle(SQLite3GlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - SQLite3Errors.addExpectedExpressionErrors(errors); - SQLite3Errors.addMatchQueryErrors(errors); - SQLite3Errors.addQueryErrors(errors); - errors.add("misuse of aggregate"); - errors.add("misuse of window function"); - errors.add("second argument to nth_value must be a positive integer"); - errors.add("no such table"); - errors.add("no query solution"); - errors.add("unable to use function MATCH in the requested context"); + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors() + .with(SQLite3Errors.getExpectedExpressionErrors().toArray(new String[0])) + .with(SQLite3Errors.getMatchQueryErrors().toArray(new String[0])) + .with(SQLite3Errors.getQueryErrors().toArray(new String[0])) + .with("misuse of aggregate", "misuse of window function", + "second argument to nth_value must be a positive integer", "no such table", "no query solution", + "unable to use function MATCH in the requested context") + .build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - reproducer = null; - SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - gen = new SQLite3ExpressionGenerator(state).setColumns(columns); - SQLite3Expression randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List joinStatements = gen.getRandomJoinClauses(tables); - List tableRefs = SQLite3Common.getTableRefs(tables, s); - SQLite3Select select = new SQLite3Select(); - select.setFromList(tableRefs); - select.setJoinClauses(joinStatements); - - Function optimizedQuery = getOptimizedQuery(select, randomWhereCondition); - Function unoptimizedQuery = getUnoptimizedQuery(select, randomWhereCondition); - int optimizedCount = optimizedQuery.apply(state); - int unoptimizedCount = unoptimizedQuery.apply(state); - if (optimizedCount == NO_VALID_RESULT || unoptimizedCount == NO_VALID_RESULT) { - throw new IgnoreMeException(); - } - if (optimizedCount != unoptimizedCount) { - reproducer = new SQLite3NoRECReproducer(optimizedQuery, unoptimizedQuery); - state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); - throw new AssertionError(optimizedCount + " " + unoptimizedCount); - } - + oracle.check(); } @Override public Reproducer getLastReproducer() { - return reproducer; + return oracle.getLastReproducer(); } @Override public String getLastQueryString() { - return optimizedQueryString; - } - - private Function getUnoptimizedQuery(SQLite3Select select, - SQLite3Expression randomWhereCondition) throws SQLException { - SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, - randomWhereCondition); - SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); - select.setFetchColumns(Arrays.asList(asText)); - select.setWhereClause(null); - unoptimizedQueryString = "SELECT SUM(count) FROM (" + SQLite3Visitor.asString(select) + ")"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - return state -> extractCounts(q, state); + return oracle.getLastQueryString(); } - - private Function getOptimizedQuery(SQLite3Select select, - SQLite3Expression randomWhereCondition) throws SQLException { - boolean useAggregate = Randomly.getBoolean(); - if (Randomly.getBoolean()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - if (useAggregate) { - select.setFetchColumns(Arrays.asList(new SQLite3Aggregate(Collections.emptyList(), - SQLite3Aggregate.SQLite3AggregateFunction.COUNT_ALL))); - } else { - SQLite3ColumnName aggr = new SQLite3ColumnName(SQLite3Column.createDummy("*"), null); - select.setFetchColumns(Arrays.asList(aggr)); - } - select.setWhereClause(randomWhereCondition); - optimizedQueryString = SQLite3Visitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - return state -> useAggregate ? extractCounts(q, state) : countRows(q, state); - } - - private int countRows(SQLQueryAdapter q, SQLite3GlobalState globalState) { - int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return NO_VALID_RESULT; - } else { - try { - while (rs.next()) { - count++; - } - } catch (SQLException e) { - count = NO_VALID_RESULT; - } - } - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw (IgnoreMeException) e; - } - throw new AssertionError(unoptimizedQueryString, e); - } - return count; - } - - private int extractCounts(SQLQueryAdapter q, SQLite3GlobalState globalState) { - int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return NO_VALID_RESULT; - } else { - try { - while (rs.next()) { - count += rs.getInt(1); - } - } catch (SQLException e) { - count = NO_VALID_RESULT; - } - } - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw (IgnoreMeException) e; - } - throw new AssertionError(unoptimizedQueryString, e); - } - return count; - } - } From 8dbfa2887823b16ebeb4bd57d12403f11fa91fc9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 28 Mar 2024 17:19:16 +0800 Subject: [PATCH 0851/1284] Add NoREC test for Postgres --- .github/workflows/main.yml | 5 +++- test/sqlancer/dbms/TestPostgresNoREC.java | 23 ++++++++++++++++++ test/sqlancer/dbms/TestPostgresPQS.java | 24 +++++++++++++++++++ ...TestPostgres.java => TestPostgresTLP.java} | 16 ++----------- 4 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 test/sqlancer/dbms/TestPostgresNoREC.java create mode 100644 test/sqlancer/dbms/TestPostgresPQS.java rename test/sqlancer/dbms/{TestPostgres.java => TestPostgresTLP.java} (53%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f618ab1ca..8a848716a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -368,7 +368,10 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test + run: | + POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresPQS test + POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresTLP test + POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresNoREC test sqlite: name: DBMS Tests (SQLite) diff --git a/test/sqlancer/dbms/TestPostgresNoREC.java b/test/sqlancer/dbms/TestPostgresNoREC.java new file mode 100644 index 000000000..53100e3c3 --- /dev/null +++ b/test/sqlancer/dbms/TestPostgresNoREC.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPostgresNoREC { + + String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); + boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); + + @Test + public void testNoREC() { + assumeTrue(postgresIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", + "false", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestPostgresPQS.java b/test/sqlancer/dbms/TestPostgresPQS.java new file mode 100644 index 000000000..ebf4be1aa --- /dev/null +++ b/test/sqlancer/dbms/TestPostgresPQS.java @@ -0,0 +1,24 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPostgresPQS { + + String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); + boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); + + @Test + public void testPQS() { + assumeTrue(postgresIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "postgres", "--test-collations", "false", "--oracle", "pqs" })); + } + +} diff --git a/test/sqlancer/dbms/TestPostgres.java b/test/sqlancer/dbms/TestPostgresTLP.java similarity index 53% rename from test/sqlancer/dbms/TestPostgres.java rename to test/sqlancer/dbms/TestPostgresTLP.java index 5eeb1f858..e704724b8 100644 --- a/test/sqlancer/dbms/TestPostgres.java +++ b/test/sqlancer/dbms/TestPostgresTLP.java @@ -7,29 +7,17 @@ import sqlancer.Main; -public class TestPostgres { +public class TestPostgresTLP { String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); @Test - public void testPostgres() { + public void testTLP() { assumeTrue(postgresIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", "false" })); } - - @Test - public void testPQS() { - assumeTrue(postgresIsAvailable); - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", - "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", - "pqsdb" /* Workaround for connections not being closed */, "postgres", "--test-collations", - "false", "--oracle", "pqs" })); - } - } From 73e72eceecfda951ca3807d1603f612792b15075 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 29 Mar 2024 09:55:19 +0800 Subject: [PATCH 0852/1284] Refactor environment check in tests --- test/sqlancer/dbms/TestConfig.java | 7 +++++++ test/sqlancer/dbms/TestPostgresNoREC.java | 5 +---- test/sqlancer/dbms/TestPostgresPQS.java | 5 +---- test/sqlancer/dbms/TestPostgresTLP.java | 5 +---- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 9fac06920..8cea624c7 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -3,4 +3,11 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + + public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; + + public static boolean isEnvironmentTrue(String key) { + String value = System.getenv(key); + return value != null && value.equalsIgnoreCase("true"); + } } diff --git a/test/sqlancer/dbms/TestPostgresNoREC.java b/test/sqlancer/dbms/TestPostgresNoREC.java index 53100e3c3..8b9f00d48 100644 --- a/test/sqlancer/dbms/TestPostgresNoREC.java +++ b/test/sqlancer/dbms/TestPostgresNoREC.java @@ -9,12 +9,9 @@ public class TestPostgresNoREC { - String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); - boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); - @Test public void testNoREC() { - assumeTrue(postgresIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", diff --git a/test/sqlancer/dbms/TestPostgresPQS.java b/test/sqlancer/dbms/TestPostgresPQS.java index ebf4be1aa..f6a37f533 100644 --- a/test/sqlancer/dbms/TestPostgresPQS.java +++ b/test/sqlancer/dbms/TestPostgresPQS.java @@ -9,12 +9,9 @@ public class TestPostgresPQS { - String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); - boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); - @Test public void testPQS() { - assumeTrue(postgresIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", diff --git a/test/sqlancer/dbms/TestPostgresTLP.java b/test/sqlancer/dbms/TestPostgresTLP.java index e704724b8..5bd722991 100644 --- a/test/sqlancer/dbms/TestPostgresTLP.java +++ b/test/sqlancer/dbms/TestPostgresTLP.java @@ -9,12 +9,9 @@ public class TestPostgresTLP { - String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); - boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); - @Test public void testTLP() { - assumeTrue(postgresIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", From b6fb5ad5ac86f1c7b7ff87474d401545034a1ac3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 28 Mar 2024 15:31:48 +0800 Subject: [PATCH 0853/1284] Implement expression interface for Postgres expressions --- src/sqlancer/common/ast/SelectBase.java | 11 +++++++++++ src/sqlancer/mariadb/ast/MariaDBSelectStatement.java | 1 + src/sqlancer/oceanbase/ast/OceanBaseSelect.java | 2 ++ src/sqlancer/postgres/ast/PostgresExpression.java | 4 +++- src/sqlancer/postgres/ast/PostgresJoin.java | 6 +++++- src/sqlancer/postgres/ast/PostgresSelect.java | 12 +++++++++++- 6 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/common/ast/SelectBase.java b/src/sqlancer/common/ast/SelectBase.java index ebc3ed5c3..e79a1a87d 100644 --- a/src/sqlancer/common/ast/SelectBase.java +++ b/src/sqlancer/common/ast/SelectBase.java @@ -36,6 +36,10 @@ public void setFromList(List fromList) { this.fromList = fromList; } + public void setFromTables(List tables) { + setFromList(tables); + } + public List getFromList() { if (fromList == null) { throw new IllegalStateException(); @@ -115,4 +119,11 @@ public void setJoinList(List joinList) { this.joinList = joinList; } + public List getGroupByClause() { + return getGroupByExpressions(); + } + + public void setGroupByClause(List groupByExpressions) { + setGroupByExpressions(groupByExpressions); + } } diff --git a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java index a690276d1..813b92337 100644 --- a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java +++ b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java @@ -16,6 +16,7 @@ public enum MariaDBSelectType { private MariaDBSelectType selectType = MariaDBSelectType.ALL; private MariaDBExpression whereCondition; + @Override public void setGroupByClause(List groupBys) { this.groupBys = groupBys; } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java index a87ff5c27..a673492c2 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java @@ -29,10 +29,12 @@ public void setFromOptions(SelectType fromOptions) { this.fromOptions = fromOptions; } + @Override public void setGroupByClause(List groupBys) { this.groupBys = groupBys; } + @Override public List getGroupByClause() { return this.groupBys; } diff --git a/src/sqlancer/postgres/ast/PostgresExpression.java b/src/sqlancer/postgres/ast/PostgresExpression.java index 96ddfa433..433e2207c 100644 --- a/src/sqlancer/postgres/ast/PostgresExpression.java +++ b/src/sqlancer/postgres/ast/PostgresExpression.java @@ -1,8 +1,10 @@ package sqlancer.postgres.ast; +import sqlancer.common.ast.newast.Expression; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; -public interface PostgresExpression { +public interface PostgresExpression extends Expression { default PostgresDataType getExpressionType() { return null; diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index d1a076bbe..7bfef3f6f 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -5,12 +5,14 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class PostgresJoin implements PostgresExpression { +public class PostgresJoin implements PostgresExpression, Join { public enum PostgresJoinType { INNER, LEFT, RIGHT, FULL, CROSS; @@ -77,6 +79,7 @@ public static List getJoins(List tableLi return joinExpressions; } + @Override public void setOnClause(PostgresExpression clause) { this.onClause = clause; } @@ -97,6 +100,7 @@ public PostgresExpression getRightTable() { return rightTable; } + @Override public PostgresExpression getOnClause() { return onClause; } diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index 70a42650f..c864a56e8 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -5,10 +5,14 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresVisitor; -public class PostgresSelect extends SelectBase implements PostgresExpression { +public class PostgresSelect extends SelectBase + implements PostgresExpression, Select { private SelectType selectOption = SelectType.ALL; private List joinClauses = Collections.emptyList(); @@ -111,11 +115,13 @@ public PostgresDataType getExpressionType() { return null; } + @Override public void setJoinClauses(List joinStatements) { this.joinClauses = joinStatements; } + @Override public List getJoinClauses() { return joinClauses; } @@ -132,4 +138,8 @@ public ForClause getForClause() { return forClause; } + @Override + public String asString() { + return PostgresVisitor.asString(this); + } } From d9def9379ca8951cb69bf578fa17dda4390fdc73 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 28 Mar 2024 17:08:29 +0800 Subject: [PATCH 0854/1284] Specify select type for NoREC generator --- src/sqlancer/common/gen/NoRECGenerator.java | 10 +++++----- src/sqlancer/common/oracle/NoRECOracle.java | 8 ++++---- .../sqlite3/gen/SQLite3ExpressionGenerator.java | 10 ++++------ src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java | 3 ++- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/sqlancer/common/gen/NoRECGenerator.java b/src/sqlancer/common/gen/NoRECGenerator.java index f57f7c20f..185e8286b 100644 --- a/src/sqlancer/common/gen/NoRECGenerator.java +++ b/src/sqlancer/common/gen/NoRECGenerator.java @@ -9,13 +9,13 @@ import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; -public interface NoRECGenerator, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { +public interface NoRECGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { - NoRECGenerator setTablesAndColumns(AbstractTables tables); + NoRECGenerator setTablesAndColumns(AbstractTables tables); E generateBooleanExpression(); - Select generateSelect(); + S generateSelect(); List getRandomJoinClauses(); @@ -33,7 +33,7 @@ public interface NoRECGenerator, E extends Expression * * @return a query string to be executed */ - String generateOptimizedQueryString(Select select, E whereCondition, boolean shouldUseAggregate); + String generateOptimizedQueryString(S select, E whereCondition, boolean shouldUseAggregate); /** * Generates a query string that is unlikely to be optimized by the DBMS. @@ -45,5 +45,5 @@ public interface NoRECGenerator, E extends Expression * * @return a query string to be executed */ - String generateUnoptimizedQueryString(Select select, E whereCondition); + String generateUnoptimizedQueryString(S select, E whereCondition); } diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index aac047b2c..acc7cffac 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -20,12 +20,12 @@ import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; -public class NoRECOracle, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> +public class NoRECOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> implements TestOracle { private final G state; - private NoRECGenerator gen; + private NoRECGenerator gen; private final ExpectedErrors errors; private Reproducer reproducer; @@ -46,7 +46,7 @@ public boolean bugStillTriggers(G globalState) { } } - public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); } @@ -63,7 +63,7 @@ public void check() throws SQLException { AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); gen = gen.setTablesAndColumns(targetTables); - Select select = gen.generateSelect(); + Z select = gen.generateSelect(); select.setJoinClauses(gen.getRandomJoinClauses()); select.setFromList(gen.getTableRefs()); diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 7b2b164e6..0ac0cd272 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -7,7 +7,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Select; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.schema.AbstractTables; @@ -51,7 +50,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; public class SQLite3ExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator { private SQLite3RowValue rw; private final SQLite3GlobalState globalState; @@ -749,8 +748,8 @@ public List getTableRefs() { } @Override - public String generateOptimizedQueryString(Select select, - SQLite3Expression whereCondition, boolean shouldUseAggregate) { + public String generateOptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition, + boolean shouldUseAggregate) { if (Randomly.getBoolean()) { select.setOrderByClauses(generateOrderBys()); } @@ -767,8 +766,7 @@ public String generateOptimizedQueryString(Select select, - SQLite3Expression whereCondition) { + public String generateUnoptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition) { SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, whereCondition); SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 27e0229a5..7393ec8d5 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -10,6 +10,7 @@ import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.ast.SQLite3Select; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; @@ -17,7 +18,7 @@ public class SQLite3NoRECOracle implements TestOracle { - NoRECOracle oracle; + NoRECOracle oracle; public SQLite3NoRECOracle(SQLite3GlobalState globalState) { SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); From 85c22fe3179ef68e9abd898bc8eac97de6558fbb Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 28 Mar 2024 17:09:22 +0800 Subject: [PATCH 0855/1284] Implement NoREC generator for Postgres --- .../gen/PostgresExpressionGenerator.java | 122 +++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index f979e97da..5b5851671 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -11,12 +11,16 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresRowValue; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresSchema.PostgresTables; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresAggregate.PostgresAggregateFunction; import sqlancer.postgres.ast.PostgresBetweenOperation; @@ -40,6 +44,8 @@ import sqlancer.postgres.ast.PostgresFunction.PostgresFunctionWithResult; import sqlancer.postgres.ast.PostgresFunctionWithUnknownResult; import sqlancer.postgres.ast.PostgresInOperation; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresOrderByTerm.PostgresOrder; @@ -47,11 +53,18 @@ import sqlancer.postgres.ast.PostgresPOSIXRegularExpression.POSIXRegex; import sqlancer.postgres.ast.PostgresPostfixOperation; import sqlancer.postgres.ast.PostgresPostfixOperation.PostfixOperator; +import sqlancer.postgres.ast.PostgresPostfixText; import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresPrefixOperation.PrefixOperator; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.ast.PostgresSelect.ForClause; +import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; +import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.ast.PostgresSimilarTo; -public class PostgresExpressionGenerator implements ExpressionGenerator { +public class PostgresExpressionGenerator implements ExpressionGenerator, + NoRECGenerator { private final int maxDepth; @@ -59,6 +72,8 @@ public class PostgresExpressionGenerator implements ExpressionGenerator columns; + private List targetTables; + private PostgresRowValue rw; private boolean expectedResult; @@ -586,6 +601,35 @@ public PostgresExpressionGenerator allowAggregates(boolean value) { return this; } + public static PostgresSubquery createSubquery(PostgresGlobalState globalState, String name, PostgresTables tables) { + List columns = new ArrayList<>(); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + PostgresSelect select = new PostgresSelect(); + select.setFromList(tables.getTables().stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return new PostgresSubquery(select, name); + } + @Override public PostgresExpression generatePredicate() { return generateExpression(PostgresDataType.BOOLEAN); @@ -601,4 +645,80 @@ public PostgresExpression isNull(PostgresExpression expr) { return new PostgresPostfixOperation(expr, PostfixOperator.IS_NULL); } + @Override + public PostgresExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + this.targetTables = targetTables.getTables(); + this.columns = targetTables.getColumns(); + return this; + } + + @Override + public PostgresExpression generateBooleanExpression() { + return generateExpression(PostgresDataType.BOOLEAN); + } + + @Override + public PostgresSelect generateSelect() { + return new PostgresSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + for (int i = 1; i < targetTables.size(); i++) { + PostgresExpression joinClause = generateExpression(PostgresDataType.BOOLEAN); + PostgresTable table = Randomly.fromList(targetTables); + targetTables.remove(table); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + PostgresTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PostgresSubquery subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); + PostgresExpression joinClause = generateExpression(PostgresDataType.BOOLEAN); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public List getTableRefs() { + return targetTables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(PostgresSelect select, PostgresExpression whereCondition, + boolean shouldUseAggregate) { + PostgresColumnValue allColumns = new PostgresColumnValue(PostgresColumn.createDummy("*"), null); + if (shouldUseAggregate) { + select.setFetchColumns( + Arrays.asList(new PostgresAggregate(List.of(allColumns), PostgresAggregateFunction.COUNT))); + } else { + select.setFetchColumns(Arrays.asList(allColumns)); + } + select.setWhereClause(whereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBy()); + } + select.setSelectType(SelectType.ALL); + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(PostgresSelect select, PostgresExpression whereCondition) { + PostgresCastOperation isTrue = new PostgresCastOperation(whereCondition, + PostgresCompoundDataType.create(PostgresDataType.INT)); + PostgresPostfixText asText = new PostgresPostfixText(isTrue, " as count", null, PostgresDataType.INT); + select.setFetchColumns(Arrays.asList(asText)); + select.setWhereClause(null); + select.setOrderByClauses(List.of()); + select.setSelectType(SelectType.ALL); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } From 0931c3feb96c054d1138989394056e2361449dbf Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 29 Mar 2024 10:38:58 +0800 Subject: [PATCH 0856/1284] Use generic NoREC oracle for Postgres --- .../postgres/oracle/PostgresNoRECOracle.java | 131 +++--------------- 1 file changed, 22 insertions(+), 109 deletions(-) diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 2ad7cc2b7..d1553c35f 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -1,77 +1,57 @@ package sqlancer.postgres.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; +import java.util.regex.Pattern; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.PostgresCompoundDataType; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresSchema.PostgresTables; -import sqlancer.postgres.PostgresVisitor; -import sqlancer.postgres.ast.PostgresCastOperation; -import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; -import sqlancer.postgres.ast.PostgresPostfixText; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; -import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; import sqlancer.postgres.oracle.tlp.PostgresTLPBase; -public class PostgresNoRECOracle extends NoRECBase implements TestOracle { +public class PostgresNoRECOracle implements TestOracle { - private final PostgresSchema s; + private final NoRECOracle oracle; public PostgresNoRECOracle(PostgresGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - PostgresCommon.addCommonExpressionErrors(errors); - PostgresCommon.addCommonFetchErrors(errors); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors() + .with(PostgresCommon.getCommonExpressionErrors().toArray(new String[0])) + .with(PostgresCommon.getCommonFetchErrors().toArray(new String[0])) + .with(PostgresCommon.getCommonExpressionRegexErrors().toArray(new Pattern[0])).build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - PostgresTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - PostgresExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); + oracle.check(); + } - List joinStatements = getJoinStatements(state, columns, tables); - List fromTables = tables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) - .collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } public static List getJoinStatements(PostgresGlobalState globalState, List columns, @@ -98,71 +78,4 @@ public static List getJoinStatements(PostgresGlobalState globalSta } return joinStatements; } - - private PostgresExpression getRandomWhereCondition(List columns) { - return new PostgresExpressionGenerator(state).setColumns(columns).generateExpression(PostgresDataType.BOOLEAN); - } - - private int getUnoptimizedQueryCount(List fromTables, PostgresExpression randomWhereCondition, - List joinStatements) throws SQLException { - PostgresSelect select = new PostgresSelect(); - PostgresCastOperation isTrue = new PostgresCastOperation(randomWhereCondition, - PostgresCompoundDataType.create(PostgresDataType.INT)); - PostgresPostfixText asText = new PostgresPostfixText(isTrue, " as count", null, PostgresDataType.INT); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(fromTables); - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + PostgresVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(List randomTables, List columns, - PostgresExpression randomWhereCondition, List joinStatements) throws SQLException { - PostgresSelect select = new PostgresSelect(); - PostgresColumnValue allColumns = new PostgresColumnValue(Randomly.fromList(columns), null); - select.setFetchColumns(Arrays.asList(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new PostgresExpressionGenerator(state).setColumns(columns).generateOrderBy()); - } - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = PostgresVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; - } - } From 1e01d0a5dae53cdacecff107bbe4a82a8eaf27c8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 29 Mar 2024 10:39:24 +0800 Subject: [PATCH 0857/1284] Use generic NoREC oracle for Citus --- .../citus/oracle/CitusNoRECOracle.java | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/citus/oracle/CitusNoRECOracle.java b/src/sqlancer/citus/oracle/CitusNoRECOracle.java index 88ec3391a..a59207ec4 100644 --- a/src/sqlancer/citus/oracle/CitusNoRECOracle.java +++ b/src/sqlancer/citus/oracle/CitusNoRECOracle.java @@ -1,14 +1,49 @@ package sqlancer.citus.oracle; +import java.sql.SQLException; +import java.util.regex.Pattern; + +import sqlancer.Reproducer; import sqlancer.citus.gen.CitusCommon; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.oracle.PostgresNoRECOracle; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class CitusNoRECOracle extends PostgresNoRECOracle { +public class CitusNoRECOracle implements TestOracle { + private final NoRECOracle oracle; public CitusNoRECOracle(PostgresGlobalState globalState) { - super(globalState); - CitusCommon.addCitusErrors(errors); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors() + .with(PostgresCommon.getCommonExpressionErrors().toArray(new String[0])) + .with(PostgresCommon.getCommonFetchErrors().toArray(new String[0])) + .with(PostgresCommon.getCommonExpressionRegexErrors().toArray(new Pattern[0])) + .with(CitusCommon.getCitusErrors().toArray(new String[0])).build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); + } + + @Override + public void check() throws SQLException { + oracle.check(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From b7ac21ab075a9b4111c02af62765630735fd1ec9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sun, 31 Mar 2024 10:18:29 +0800 Subject: [PATCH 0858/1284] Rename `with` method in ExpectedErrorsBuilder for regex & regex string --- .../citus/oracle/CitusNoRECOracle.java | 7 ++--- src/sqlancer/common/query/ExpectedErrors.java | 16 +++++++++-- .../postgres/oracle/PostgresNoRECOracle.java | 8 ++---- .../sqlite3/oracle/SQLite3NoRECOracle.java | 6 ++-- test/sqlancer/TestExpectedErrors.java | 28 +++++++++++++++++-- 5 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/citus/oracle/CitusNoRECOracle.java b/src/sqlancer/citus/oracle/CitusNoRECOracle.java index a59207ec4..ffc7c13fb 100644 --- a/src/sqlancer/citus/oracle/CitusNoRECOracle.java +++ b/src/sqlancer/citus/oracle/CitusNoRECOracle.java @@ -1,7 +1,6 @@ package sqlancer.citus.oracle; import java.sql.SQLException; -import java.util.regex.Pattern; import sqlancer.Reproducer; import sqlancer.citus.gen.CitusCommon; @@ -23,10 +22,8 @@ public class CitusNoRECOracle implements TestOracle { public CitusNoRECOracle(PostgresGlobalState globalState) { PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors() - .with(PostgresCommon.getCommonExpressionErrors().toArray(new String[0])) - .with(PostgresCommon.getCommonFetchErrors().toArray(new String[0])) - .with(PostgresCommon.getCommonExpressionRegexErrors().toArray(new Pattern[0])) + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) .with(CitusCommon.getCitusErrors().toArray(new String[0])).build(); this.oracle = new NoRECOracle<>(globalState, gen, errors); } diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index 348da179f..f71d185e1 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -110,18 +110,30 @@ public ExpectedErrorsBuilder with(String... list) { return this; } - public ExpectedErrorsBuilder with(Pattern... list) { + public ExpectedErrorsBuilder with(Collection list) { + return with(list.toArray(new String[0])); + } + + public ExpectedErrorsBuilder withRegex(Pattern... list) { regexes.addAll(Arrays.asList(list)); return this; } - public ExpectedErrorsBuilder withRegex(String... list) { + public ExpectedErrorsBuilder withRegex(Collection list) { + return withRegex(list.toArray(new Pattern[0])); + } + + public ExpectedErrorsBuilder withRegexString(String... list) { for (String error : list) { regexes.add(Pattern.compile(error)); } return this; } + public ExpectedErrorsBuilder withRegexString(Collection list) { + return withRegexString(list.toArray(new String[0])); + } + public ExpectedErrors build() { return new ExpectedErrors(errors, regexes); } diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index d1553c35f..9243b4870 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -3,7 +3,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.regex.Pattern; import sqlancer.Randomly; import sqlancer.Reproducer; @@ -32,10 +31,9 @@ public class PostgresNoRECOracle implements TestOracle { public PostgresNoRECOracle(PostgresGlobalState globalState) { PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors() - .with(PostgresCommon.getCommonExpressionErrors().toArray(new String[0])) - .with(PostgresCommon.getCommonFetchErrors().toArray(new String[0])) - .with(PostgresCommon.getCommonExpressionRegexErrors().toArray(new Pattern[0])).build(); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .build(); this.oracle = new NoRECOracle<>(globalState, gen, errors); } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 7393ec8d5..8f6e2d276 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -22,10 +22,8 @@ public class SQLite3NoRECOracle implements TestOracle { public SQLite3NoRECOracle(SQLite3GlobalState globalState) { SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors() - .with(SQLite3Errors.getExpectedExpressionErrors().toArray(new String[0])) - .with(SQLite3Errors.getMatchQueryErrors().toArray(new String[0])) - .with(SQLite3Errors.getQueryErrors().toArray(new String[0])) + ExpectedErrors errors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .with(SQLite3Errors.getMatchQueryErrors()).with(SQLite3Errors.getQueryErrors()) .with("misuse of aggregate", "misuse of window function", "second argument to nth_value must be a positive integer", "no such table", "no query solution", "unable to use function MATCH in the requested context") diff --git a/test/sqlancer/TestExpectedErrors.java b/test/sqlancer/TestExpectedErrors.java index 2cb938824..2a3b1a67e 100644 --- a/test/sqlancer/TestExpectedErrors.java +++ b/test/sqlancer/TestExpectedErrors.java @@ -118,14 +118,38 @@ public void testBuilder() { assertTrue(errors.errorIsExpected("aa")); assertFalse(errors.errorIsExpected("d")); - errors = ExpectedErrors.newErrors().withRegex("a\\d", "b\\D").with("c").build(); + errors = ExpectedErrors.newErrors().with(List.of("a", "b", "c")).build(); + + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + errors = ExpectedErrors.newErrors().withRegexString("a\\d", "b\\D").with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + errors = ExpectedErrors.newErrors().withRegexString(List.of("a\\d", "b\\D")).with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + errors = ExpectedErrors.newErrors().withRegex(Pattern.compile("a\\d"), Pattern.compile("b\\D")).with("c") + .build(); assertTrue(errors.errorIsExpected("a0")); assertTrue(errors.errorIsExpected("bb")); assertTrue(errors.errorIsExpected("c")); assertFalse(errors.errorIsExpected("aa")); - errors = ExpectedErrors.newErrors().with(Pattern.compile("a\\d"), Pattern.compile("b\\D")).with("c").build(); + errors = ExpectedErrors.newErrors().withRegex(List.of(Pattern.compile("a\\d"), Pattern.compile("b\\D"))) + .with("c").build(); assertTrue(errors.errorIsExpected("a0")); assertTrue(errors.errorIsExpected("bb")); From bd511f09846cc585e8f7f4a620a65e08482e1528 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 1 Apr 2024 14:17:58 +0800 Subject: [PATCH 0859/1284] Pin Clickhouse version to 23.1.2672 lts --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f0adb6e4..766aac668 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,8 +109,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up ClickHouse run: | - docker pull clickhouse/clickhouse-server:head - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:head + docker pull clickhouse/clickhouse-server:24.3.1.2672 + docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:24.3.1.2672 sleep 5 - name: Run Tests run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test From 27638c3be1543094b97288ddfb9516572d39c0f8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 1 Apr 2024 14:48:39 +0800 Subject: [PATCH 0860/1284] Update ExpectedErrors to match change in error --- src/sqlancer/clickhouse/ClickHouseErrors.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 23c27c8bf..c4ccd7e44 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -34,9 +34,8 @@ public static List getExpectedExpressionErrors() { "is not under aggregate function and not in GROUP BY", "is not under aggregate function", "is violated at row 1. Expression:", // TODO: check constraint on table creation "is violated, because it is a constant expression returning 0. It is most likely an error in table definition", - "there are only columns", "there are columns", "in block. (NOT_FOUND_COLUMN_IN_BLOCK)", - "Missing columns", "Ambiguous column", - "Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)", + "there are only columns", "there are columns", "(NOT_FOUND_COLUMN_IN_BLOCK)", "Missing columns", + "Ambiguous column", "Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)", "Floating point partition key is not supported", "Cannot get JOIN keys from JOIN ON section", "ILLEGAL_DIVISION", "DECIMAL_OVERFLOW", "Cannot convert out of range floating point value to integer type", From d6ab9965e746d69b67d63f7aaa2738f9370f82b8 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Wed, 3 Apr 2024 21:11:46 +0800 Subject: [PATCH 0861/1284] [MySQL] Add bugs 114533 and 114544 (#938) * Pass exception from ResultSet up as SQLException * [MySQL] Add bugs 114533 and 114534 --- src/sqlancer/common/query/SQLancerResultSet.java | 6 +++++- src/sqlancer/mysql/MySQLBugs.java | 6 ++++++ src/sqlancer/mysql/MySQLErrors.java | 13 +++++++++++++ .../mysql/gen/datadef/MySQLIndexGenerator.java | 7 ++++++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/query/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java index d6f2a67cb..d1221a7f7 100644 --- a/src/sqlancer/common/query/SQLancerResultSet.java +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -35,7 +35,11 @@ public int getInt(int i) throws SQLException { } public String getString(int i) throws SQLException { - return rs.getString(i); + try { + return rs.getString(i); + } catch (NumberFormatException e) { + throw new SQLException(e); + } } public boolean isClosed() throws SQLException { diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 00a7a0ddf..8cb8a3391 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -31,6 +31,12 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=112264 public static boolean bug112264 = true; + // https://bugs.mysql.com/bug.php?id=114533 + public static boolean bug114533 = true; + + // https://bugs.mysql.com/bug.php?id=114534 + public static boolean bug114534 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 3f681acb5..9dcc0a5c3 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; import sqlancer.common.query.ExpectedErrors; @@ -23,8 +24,20 @@ public static List getExpressionErrors() { return errors; } + public static List getExpressionRegexErrors() { + ArrayList errors = new ArrayList<>(); + + if (MySQLBugs.bug114533) { + errors.add(Pattern.compile("For input string: \"0+-0\"")); // match: For input string: + // "00000000000000000000-0" + } + + return errors; + } + public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); + errors.addAllRegexes(getExpressionRegexErrors()); } public static List getInsertUpdateErrors() { diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index 3f418385d..550893db5 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -5,6 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -78,7 +79,11 @@ public SQLQueryAdapter create() { if (Randomly.getBoolean() && c.getType() == MySQLDataType.VARCHAR) { sb.append("("); // TODO for string - sb.append(r.getInteger(1, 5)); + if (MySQLBugs.bug114534) { + sb.append(r.getInteger(2, 5)); + } else { + sb.append(r.getInteger(1, 5)); + } sb.append(")"); } if (Randomly.getBoolean()) { From 3c4d04d78b6d5cf0e4807c91028208345417aef2 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Sun, 7 Apr 2024 22:21:13 +0800 Subject: [PATCH 0862/1284] [MySQL] Pin version to 8.0.36 (#940) --- .github/workflows/main.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 766aac668..dcbb1af99 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -323,6 +323,14 @@ jobs: mysql: name: DBMS Tests (MySQL) runs-on: ubuntu-latest + services: + mysql: + image: mysql:8.0.36 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - uses: actions/checkout@v3 with: @@ -335,11 +343,8 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up MySQL - run: | - sudo systemctl start mysql.service # MySQL 8.0.32-0ubuntu0.22.04.2 - name: Create SQLancer user - run: mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" + run: mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS From 3095ec5f2a5b2005f885ae2d3b95513446767043 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 11:40:49 +0800 Subject: [PATCH 0863/1284] Add TLPWhereGenerator interface --- .../common/gen/PartitionGenerator.java | 27 ++++++++++++++++++ .../common/gen/TLPWhereGenerator.java | 28 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 src/sqlancer/common/gen/PartitionGenerator.java create mode 100644 src/sqlancer/common/gen/TLPWhereGenerator.java diff --git a/src/sqlancer/common/gen/PartitionGenerator.java b/src/sqlancer/common/gen/PartitionGenerator.java new file mode 100644 index 000000000..affc62c42 --- /dev/null +++ b/src/sqlancer/common/gen/PartitionGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.common.gen; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.schema.AbstractTableColumn; + +public interface PartitionGenerator, C extends AbstractTableColumn> { + + /** + * Negates a predicate (i.e., uses a NOT operator). + * + * @param predicate + * the boolean predicate. + * + * @return the negated predicate. + */ + E negatePredicate(E predicate); + + /** + * Checks if an expression evaluates to NULL (i.e., implements the IS NULL operator). + * + * @param expr + * the expression + * + * @return an expression that checks whether the expression evaluates to NULL. + */ + E isNull(E expr); +} diff --git a/src/sqlancer/common/gen/TLPWhereGenerator.java b/src/sqlancer/common/gen/TLPWhereGenerator.java new file mode 100644 index 000000000..095a878f9 --- /dev/null +++ b/src/sqlancer/common/gen/TLPWhereGenerator.java @@ -0,0 +1,28 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface TLPWhereGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> + extends PartitionGenerator { + + TLPWhereGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + List generateOrderBys(); +} From 307713ac59675bae6286e86eda8290db63e1ddb8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 11:41:04 +0800 Subject: [PATCH 0864/1284] Add generic TLPWhere oracle --- .../common/oracle/TLPWhereOracle.java | 84 +++++++++++++++++++ .../common/oracle/TestOracleUtils.java | 33 ++++++++ 2 files changed, 117 insertions(+) create mode 100644 src/sqlancer/common/oracle/TLPWhereOracle.java diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java new file mode 100644 index 000000000..5f7b3feb6 --- /dev/null +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -0,0 +1,84 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class TLPWhereOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + + private TLPWhereGenerator gen; + private final ExpectedErrors errors; + + private String generatedQueryString; + + public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + S s = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(s); + gen = gen.setTablesAndColumns(targetTables); + + Select select = gen.generateSelect(); + + boolean shouldCreateDummy = true; + select.setFetchColumns(gen.generateFetchColumns(shouldCreateDummy)); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + select.setWhereClause(null); + + String originalQueryString = select.asString(); + generatedQueryString = originalQueryString; + List firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + boolean orderBy = Randomly.getBooleanWithSmallProbability(); + if (orderBy) { + select.setOrderByClauses(gen.generateOrderBys()); + } + + TestOracleUtils.PredicateVariants predicates = TestOracleUtils.initializeTernaryPredicateVariants(gen, + gen.generateBooleanExpression()); + select.setWhereClause(predicates.predicate); + String firstQueryString = select.asString(); + select.setWhereClause(predicates.negatedPredicate); + String secondQueryString = select.asString(); + select.setWhereClause(predicates.isNullPredicate); + String thirdQueryString = select.asString(); + + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + + ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java index b22cc89f5..bab2e26c9 100644 --- a/src/sqlancer/common/oracle/TestOracleUtils.java +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -2,6 +2,8 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.gen.PartitionGenerator; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; @@ -12,6 +14,18 @@ public final class TestOracleUtils { private TestOracleUtils() { } + public static final class PredicateVariants, C extends AbstractTableColumn> { + public E predicate; + public E negatedPredicate; + public E isNullPredicate; + + PredicateVariants(E predicate, E negatedPredicate, E isNullPredicate) { + this.predicate = predicate; + this.negatedPredicate = negatedPredicate; + this.isNullPredicate = isNullPredicate; + } + } + public static , C extends AbstractTableColumn> AbstractTables getRandomTableNonEmptyTables( AbstractSchema schema) { if (schema.getDatabaseTables().isEmpty()) { @@ -19,4 +33,23 @@ private TestOracleUtils() { } return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables())); } + + public static , T extends AbstractTable, C extends AbstractTableColumn> PredicateVariants initializeTernaryPredicateVariants( + PartitionGenerator gen, E predicate) { + if (gen == null) { + throw new IllegalStateException(); + } + if (predicate == null) { + throw new IllegalStateException(); + } + E negatedPredicate = gen.negatePredicate(predicate); + if (negatedPredicate == null) { + throw new IllegalStateException(); + } + E isNullPredicate = gen.isNull(predicate); + if (isNullPredicate == null) { + throw new IllegalStateException(); + } + return new PredicateVariants<>(predicate, negatedPredicate, isNullPredicate); + } } From 6e4204830d61292333e3ac58621f3dfbc294fde9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 11:59:25 +0800 Subject: [PATCH 0865/1284] Implement TLPWhereGenerator interfaces for SQLite3 --- .../sqlite3/gen/SQLite3ExpressionGenerator.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 0ac0cd272..e4f0741f9 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Aggregate; @@ -50,7 +51,8 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; public class SQLite3ExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private SQLite3RowValue rw; private final SQLite3GlobalState globalState; @@ -133,6 +135,7 @@ public static SQLite3Expression getRandomLiteralValue(SQLite3GlobalState globalS return new SQLite3ExpressionGenerator(globalState).getRandomLiteralValueInternal(globalState.getRandomly()); } + @Override public List generateOrderBys() { List expressions = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { @@ -747,6 +750,18 @@ public List getTableRefs() { return tableRefs; } + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + List columns = new ArrayList<>(); + if (shouldCreateDummy && Randomly.getBoolean()) { + columns.add(new SQLite3ColumnName(SQLite3Column.createDummy("*"), null)); + } else { + columns = Randomly.nonEmptySubset(this.columns).stream().map(c -> new SQLite3ColumnName(c, null)) + .collect(Collectors.toList()); + } + return columns; + } + @Override public String generateOptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition, boolean shouldUseAggregate) { From 17f7d3ee09f267267c08cbdd1120482afe4d7715 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 12:00:14 +0800 Subject: [PATCH 0866/1284] Use generic TLPWhere oracle for SQLite3 --- .../oracle/tlp/SQLite3TLPWhereOracle.java | 49 +++++++------------ 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java index b9580ddd8..8d850fdcb 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java @@ -1,50 +1,37 @@ package sqlancer.sqlite3.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Expression; +import sqlancer.sqlite3.ast.SQLite3Select; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.schema.SQLite3Schema; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3TLPWhereOracle extends SQLite3TLPBase { +public class SQLite3TLPWhereOracle implements TestOracle { - private String generatedQueryString; + private final TLPWhereOracle oracle; public SQLite3TLPWhereOracle(SQLite3GlobalState state) { - super(state); + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .build(); + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = SQLite3Visitor.asString(select); - generatedQueryString = originalQueryString; - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithSmallProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = SQLite3Visitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = SQLite3Visitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = SQLite3Visitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } @Override public String getLastQueryString() { - return generatedQueryString; + return oracle.getLastQueryString(); } - } From fe7f7939343bbbc019560e3c7f80ffdc855da9d1 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 14:58:42 +0800 Subject: [PATCH 0867/1284] [MariaDB] Pin version to 11.3.2 --- .github/workflows/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dcbb1af99..a2d21b67d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -241,6 +241,14 @@ jobs: mariadb: name: DBMS Tests (MariaDB) runs-on: ubuntu-latest + services: + mysql: + image: mariadb:11.3.2 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 3306:3306 + options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - uses: actions/checkout@v3 @@ -254,15 +262,8 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Install MariaDB - run: | - sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 - sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main' - sudo apt update - sudo apt install mariadb-server - sudo systemctl start mariadb - name: Create SQLancer User - run: sudo mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" + run: sudo mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test From a2f2bb08fbf06c43c0d0663e8932e4a4879bc594 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 16:24:47 +0800 Subject: [PATCH 0868/1284] [MariaDB] Remove `engine_condition_pushdown=off` optimizer switch --- .../mariadb/gen/MariaDBSetGenerator.java | 23 +++++++++---------- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 11 ++++----- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index fc1f1d5a6..860f60c4f 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -122,15 +122,14 @@ private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", - "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "engine_condition_pushdown", - "exists_to_in", "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", - "index_merge", "index_merge_intersection", "index_merge_sort_intersection", - "index_merge_sort_union", "index_merge_union", "in_to_exists", "join_cache_bka", - "join_cache_hashed", "join_cache_incremental", "loosescan", "materialization", "mrr", - "mrr_cost_based", "mrr_sort_keys", "not_null_range_scan", "optimize_join_buffer_size", - "orderby_uses_equalities", "outer_join_with_cache", "partial_match_rowid_merge", - "partial_match_table_scan", "rowid_filter", "semijoin", "semijoin_with_cache", "split_materialized", - "subquery_cache", "table_elimination" }; + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "exists_to_in", + "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", "index_merge", + "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", + "index_merge_union", "in_to_exists", "join_cache_bka", "join_cache_hashed", + "join_cache_incremental", "loosescan", "materialization", "mrr", "mrr_cost_based", "mrr_sort_keys", + "not_null_range_scan", "optimize_join_buffer_size", "orderby_uses_equalities", + "outer_join_with_cache", "partial_match_rowid_merge", "partial_match_table_scan", "rowid_filter", + "semijoin", "semijoin_with_cache", "split_materialized", "subquery_cache", "table_elimination" }; List optionSubset = Arrays.asList(Randomly.fromOptions(options)); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); @@ -191,9 +190,9 @@ public static SQLQueryAdapter resetOptimizer() { public static List getAllOptimizer(MariaDBGlobalState globalState) { List result = new ArrayList<>(); String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", - "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "engine_condition_pushdown", - "exists_to_in", "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", - "index_merge", "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "exists_to_in", "extended_keys", + "firstmatch", "index_condition_pushdown", "hash_join_cardinality", "index_merge", + "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", "index_merge_union", "in_to_exists", "join_cache_bka", "join_cache_hashed", "join_cache_incremental", "loosescan", "materialization", "mrr", "mrr_cost_based", "mrr_sort_keys", "not_null_range_scan", "optimize_join_buffer_size", "orderby_uses_equalities", "outer_join_with_cache", diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 94db1fab7..79333eb36 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -135,12 +135,11 @@ private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); String[] options = { "index_merge", "index_merge_union", "index_merge_sort_union", - "index_merge_intersection", "engine_condition_pushdown", "index_condition_pushdown", "mrr", - "mrr_cost_based", "block_nested_loop", "batched_key_access", "materialization", "semijoin", - "loosescan", "firstmatch", "duplicateweedout", "subquery_materialization_cost_based", - "use_index_extensions", "condition_fanout_filter", "derived_merge", "use_invisible_indexes", - "skip_scan", "hash_join", "subquery_to_derived", "prefer_ordering_index", - "derived_condition_pushdown" }; + "index_merge_intersection", "index_condition_pushdown", "mrr", "mrr_cost_based", + "block_nested_loop", "batched_key_access", "materialization", "semijoin", "loosescan", "firstmatch", + "duplicateweedout", "subquery_materialization_cost_based", "use_index_extensions", + "condition_fanout_filter", "derived_merge", "use_invisible_indexes", "skip_scan", "hash_join", + "subquery_to_derived", "prefer_ordering_index", "derived_condition_pushdown" }; List optionSubset = Randomly.nonEmptySubset(options); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); From b7bbc1dc95e087267f28330cb4942ca48c1bcfe3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 16:26:08 +0800 Subject: [PATCH 0869/1284] [MariaDB] Add key length restriction to expected errors --- src/sqlancer/mariadb/gen/MariaDBTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java index 4376b62db..23f09a420 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java @@ -142,6 +142,7 @@ private void createOrReplaceTable() { sb.append("IF NOT EXISTS "); } sb.append(tableName); + errors.add("Specified key was too long; max key length is"); } } From acd0b46bd04390190f717aa5ab1d6c4b7308a63b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 18 May 2024 09:55:28 +0800 Subject: [PATCH 0870/1284] [MariaDB] Add bug MDEV-33893 --- src/sqlancer/mariadb/MariaDBBugs.java | 3 +++ .../mariadb/gen/MariaDBTableAdminCommandGenerator.java | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mariadb/MariaDBBugs.java b/src/sqlancer/mariadb/MariaDBBugs.java index fb1770763..5a1bfa5cc 100644 --- a/src/sqlancer/mariadb/MariaDBBugs.java +++ b/src/sqlancer/mariadb/MariaDBBugs.java @@ -26,6 +26,9 @@ public final class MariaDBBugs { // https://jira.mariadb.org/browse/MDEV-32143 public static boolean bug32143 = true; + // https://jira.mariadb.org/browse/MDEV-33893 + public static boolean bug33893 = true; + private MariaDBBugs() { } diff --git a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java index a141ca58d..dd40739f4 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java @@ -8,6 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryResultCheckAdapter; +import sqlancer.mariadb.MariaDBBugs; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; @@ -54,7 +55,9 @@ public static SQLQueryAdapter checkTable(MariaDBSchema newSchema) { public static SQLQueryAdapter optimizeTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "OPTIMIZE TABLE"); - MariaDBCommon.addWaitClause(sb); + if (!MariaDBBugs.bug33893) { + MariaDBCommon.addWaitClause(sb); + } return checkForMsgText(sb, s -> s.equals("OK") || s.equals("Table does not support optimize, doing recreate + analyze instead") || s.contentEquals("Table is already up to date") || s.contains("Lock wait timeout") From eba08c4d11cb8a696c379c692ac879896123ab3a Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 21 May 2024 11:34:24 -0400 Subject: [PATCH 0871/1284] Upgrade CockroachDB to v24.1.0-rc.2 --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dcbb1af99..89e447dc3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -138,11 +138,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.2.0-beta.1.linux-amd64.tgz | tar xvz - cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.1.0-rc.2.linux-amd64.tgz | tar xvz + cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -163,11 +163,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.2.0-beta.1.linux-amd64.tgz | tar xvz - cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.1.0-rc.2.linux-amd64.tgz | tar xvz + cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test From 59470daff17e4de74079bce33dce3ef878970873 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 21 May 2024 13:07:29 -0400 Subject: [PATCH 0872/1284] Consolidate "unsupported binary operator" errors for CockroachDB --- src/sqlancer/cockroachdb/CockroachDBErrors.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 63d0b30ce..1f3e1a587 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -100,11 +100,7 @@ public static List getExpressionErrors() { errors.add("LOOKUP can only be used with INNER or LEFT joins"); // TODO errors.add("ambiguous binary operator: || "); - errors.add(" ERROR: unsupported binary operator: || (desired )"); - errors.add("unsupported binary operator: || (desired )"); - errors.add("incompatible value type: unsupported binary operator: || (desired )"); - errors.add("unsupported binary operator: || (desired )"); - errors.add("unsupported binary operator: || (desired )"); + errors.add("unsupported binary operator"); errors.add("parsing as type timestamp: empty or blank input"); errors.add("parsing as type timestamp: field"); errors.add("as type time"); @@ -121,7 +117,6 @@ public static List getExpressionErrors() { errors.add("numeric constant out of int64 range"); errors.add("unknown signature: overlay"); errors.add("unknown signature: substring"); - errors.add("unsupported binary operator: + (desired )"); errors.add("unsupported comparison operator"); errors.add("unknown signature: chr(decimal) (desired )"); errors.add("unknown signature: to_english(decimal) (desired )"); @@ -136,7 +131,6 @@ public static List getExpressionErrors() { errors.add("has type decimal"); errors.add("to be of type decimal, found type int"); errors.add("value type decimal doesn't match type int"); - errors.add("unsupported binary operator: / (desired )"); errors.add("(desired )"); errors.add("(desired )"); errors.add("(desired )"); From 3151b5c49392748b73c3e32700d8de816790b2ed Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 21 May 2024 13:09:43 -0400 Subject: [PATCH 0873/1284] Consolidate "could not parse JSON" errors for CockroachDB --- src/sqlancer/cockroachdb/CockroachDBErrors.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 1f3e1a587..b52df11bd 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -146,10 +146,7 @@ public static List getExpressionErrors() { errors.add("exists but is not a directory"); // TODO - errors.add("could not parse JSON: trailing characters after JSON document"); - errors.add("could not parse JSON: unable to decode JSON: invalid character"); - errors.add("could not parse JSON: unable to decode JSON: EOF"); - errors.add("could not parse JSON: unable to decode JSON: unexpected EOF"); + errors.add("could not parse JSON"); errors.add("can't order by column type jsonb"); errors.add("odd length hex string"); From 43a373a1feeac8a1f5cf5bfc205db05e71c2e302 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Sun, 16 Jun 2024 13:15:31 +0800 Subject: [PATCH 0874/1284] [Doris] Updated Doris to 2.1.4 (#949) * [Doris] Update deployment in CI * [Doris] Update group by columns to match fetch columns * [Doris] Prevent generation of null for some expressions * [Doris] Add bug 36070 * [Doris] Add new error message * [Doris] Update CI script and pin to version 2.1.3 * [Doris] Add bug 36072 * [Doris] Fix bugs in TLP Aggregate oracle * [Doris] Add bug 34342 * [Doris] Add bug 36343 * [Doris] Add bug 36346 * [Doris] Update bugs list * [Doris] Update to version 2.1.4 * [Doris] Refactor tests * [Doris] Add bug 36351 --- .github/workflows/main.yml | 30 +++++------ src/sqlancer/doris/DorisBugs.java | 48 ++++++++--------- src/sqlancer/doris/DorisErrors.java | 2 + src/sqlancer/doris/DorisSchema.java | 10 +++- .../gen/DorisNewExpressionGenerator.java | 39 +++++++++++--- ...DorisQueryPartitioningAggregateTester.java | 26 ++++++--- .../tlp/DorisQueryPartitioningBase.java | 7 +-- .../DorisQueryPartitioningGroupByTester.java | 2 +- .../DorisQueryPartitioningHavingTester.java | 1 + test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestDoris.java | 54 ------------------- test/sqlancer/dbms/TestDorisNoREC.java | 26 +++++++++ test/sqlancer/dbms/TestDorisPQS.java | 26 +++++++++ test/sqlancer/dbms/TestDorisTLP.java | 26 +++++++++ 14 files changed, 188 insertions(+), 110 deletions(-) delete mode 100644 test/sqlancer/dbms/TestDoris.java create mode 100644 test/sqlancer/dbms/TestDorisNoREC.java create mode 100644 test/sqlancer/dbms/TestDorisPQS.java create mode 100644 test/sqlancer/dbms/TestDorisTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 20a591cb8..9fb049f88 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -544,26 +544,26 @@ jobs: - name: Set up Apache Doris run: | sudo sysctl -w vm.max_map_count=2000000 - LASTEST_TAG=$(curl -s GET https://api.github.com/repos/apache/doris/releases | jq -r '.[].tag_name' | sed -n 1p) - LASTEST_TAG_BIG_VERSION=$(echo ${LASTEST_TAG} | awk '{split($1, arr, "."); print arr[1]"."arr[2]}') - curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz" - curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz" - curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz" - mkdir ./doris - tar xf apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-fe-${LASTEST_TAG}-bin-x86_64 doris/fe - tar xf apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-be-${LASTEST_TAG}-bin-x86_64 doris/be - tar xf apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64 doris/dependencies - cp doris/dependencies/*.jar doris/be/lib/ - doris/fe/bin/start_fe.sh --daemon - doris/be/bin/start_be.sh --daemon - sudo apt install libnet-ifconfig-wrapper-perl --assume-yes - IP=$(ifconfig eth0 | grep inet | grep -v inet6 | awk '{print $2}') + wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz + tar zxf apache-doris-2.1.4-bin-x64.tar.gz + mv apache-doris-2.1.4-bin-x64 apache-doris + sudo swapoff -a + cd apache-doris/fe + ./bin/start_fe.sh --daemon + cd ../be + ./bin/start_be.sh --daemon + + sleep 30 + IP=$(hostname -I | awk '{print $1}') mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';" mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: DORIS_AVAILABLE=true mvn -Dtest=TestDoris test + run: | + DORIS_AVAILABLE=true mvn -Dtest=TestDorisNoREC test + DORIS_AVAILABLE=true mvn -Dtest=TestDorisPQS test + DORIS_AVAILABLE=true mvn -Dtest=TestDorisTLP test reducer: name: Reducer Tests diff --git a/src/sqlancer/doris/DorisBugs.java b/src/sqlancer/doris/DorisBugs.java index 080aa0450..956be1683 100644 --- a/src/sqlancer/doris/DorisBugs.java +++ b/src/sqlancer/doris/DorisBugs.java @@ -1,43 +1,43 @@ package sqlancer.doris; public final class DorisBugs { - // https://github.com/apache/doris/issues/17697 - // Logical bug about where true not in (columns) - public static boolean bug17697 = true; - - // https://github.com/apache/doris/issues/17700 - // Cannot use between and in boolean column - public static boolean bug17700 = true; - - // https://github.com/apache/doris/issues/17701 - // Wrong result of `where column not in (values)` - public static boolean bug17701 = true; - - // https://github.com/apache/doris/issues/17705 - // Different result caused by `where` split and union all - public static boolean bug17705 = true; - // https://github.com/apache/doris/issues/19370 // Internal Error occur in GroupBy&Having sql // fixed by https://github.com/apache/doris/pull/19559 - public static boolean bug19370 = true; + public static boolean bug19370; // https://github.com/apache/doris/issues/19374 // Different result of having not ($value in column) and having ($value not in column) // fixed by https://github.com/apache/doris/pull/19471 - public static boolean bug19374 = true; + public static boolean bug19374; // https://github.com/apache/doris/issues/19611 // ERROR occur in nested subqueries with same column name and union public static boolean bug19611 = true; - // https://github.com/apache/doris/issues/19613 - // Wrong result when right outer join and where false - public static boolean bug19613 = true; + // https://github.com/apache/doris/issues/36070 + // Expression evaluate to NULL but is treated as FALSE in where clause + public static boolean bug36070 = true; + + // https://github.com/apache/doris/issues/36072 + // SELECT DISTINCT does not work with aggregate key column + public static boolean bug36072 = true; + + // https://github.com/apache/doris/issues/36342 + // Wrong result with INNER JOIN and CURRENT_TIMESTAMP + public static boolean bug36342 = true; + + // https://github.com/apache/doris/issues/36343 + // Wrong result with SELECT DISTINCT and UNIQUE model + public static boolean bug36343 = true; + + // https://github.com/apache/doris/issues/36346 + // Wrong result with LEFT JOIN SELECT DISTINCT and IN operation + public static boolean bug36346 = true; - // https://github.com/apache/doris/issues/19614 - // Wrong result when value like column from table_join - public static boolean bug19614 = true; + // https://github.com/apache/doris/issues/36351 + // Wrong result with TINYINT column with value -1049190528 + public static boolean bug36351 = true; private DorisBugs() { diff --git a/src/sqlancer/doris/DorisErrors.java b/src/sqlancer/doris/DorisErrors.java index 8e6fdfdc9..28e93b80a 100644 --- a/src/sqlancer/doris/DorisErrors.java +++ b/src/sqlancer/doris/DorisErrors.java @@ -57,6 +57,7 @@ public static List getExpressionErrors() { if (DorisBugs.bug19611) { errors.add("Duplicated inline view column alias"); } + errors.add("Arithmetic overflow"); return errors; } @@ -72,6 +73,7 @@ public static List getInsertErrors() { errors.add("Only value columns of unique table could be updated"); errors.add("Only unique olap table could be updated"); errors.add("Number out of range"); + errors.add("Arithmetic overflow"); return errors; } diff --git a/src/sqlancer/doris/DorisSchema.java b/src/sqlancer/doris/DorisSchema.java index d452f784a..76697fad7 100644 --- a/src/sqlancer/doris/DorisSchema.java +++ b/src/sqlancer/doris/DorisSchema.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -30,7 +31,14 @@ public enum DorisTableDataModel { UNIQUE, AGGREGATE, DUPLICATE; public static DorisTableDataModel getRandom() { - return Randomly.fromOptions(values()); + List validOptions = new ArrayList<>(Arrays.asList(values())); + if (DorisBugs.bug36072) { + validOptions.remove(AGGREGATE); + } + if (DorisBugs.bug36343) { + validOptions.remove(UNIQUE); + } + return Randomly.fromList(validOptions); } } diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java index 5dcb82e8f..f283d0c39 100644 --- a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -7,10 +7,12 @@ import java.util.Set; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.doris.DorisBugs; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisDataType; @@ -117,10 +119,12 @@ public DorisExpression generateExpression(DorisDataType type, int depth) { return function.getCall(type, this, depth + 1); } } - if (globalState.getDbmsSpecificOptions().testCasts && Randomly.getBooleanWithRatherLowProbability()) { + if (!DorisBugs.bug36070 && type != DorisDataType.NULL && globalState.getDbmsSpecificOptions().testCasts + && Randomly.getBooleanWithRatherLowProbability()) { return new DorisCastOperation(DorisExprToNode.cast(generateExpression(getRandomType(), depth + 1)), type); } - if (globalState.getDbmsSpecificOptions().testCase && Randomly.getBooleanWithRatherLowProbability()) { + if (!DorisBugs.bug36070 && globalState.getDbmsSpecificOptions().testCase + && Randomly.getBooleanWithRatherLowProbability()) { DorisExpression expr = generateExpression(DorisDataType.BOOLEAN, depth + 1); List conditions = new ArrayList<>(); List cases = new ArrayList<>(); @@ -192,7 +196,7 @@ DorisExpression generateBooleanExpression(int depth) { allowAggregateFunctions = false; } List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); - if (!globalState.getDbmsSpecificOptions().testIn) { + if (DorisBugs.bug36346 || !globalState.getDbmsSpecificOptions().testIn) { validOptions.remove(BooleanExpression.IN_OPERATION); } if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { @@ -201,7 +205,7 @@ DorisExpression generateBooleanExpression(int depth) { if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { validOptions.remove(BooleanExpression.BINARY_COMPARISON); } - if (!globalState.getDbmsSpecificOptions().testBetween) { + if (DorisBugs.bug36070 || !globalState.getDbmsSpecificOptions().testBetween) { validOptions.remove(BooleanExpression.BETWEEN); } @@ -306,8 +310,11 @@ public DorisExpression isNull(DorisExpression predicate) { } public DorisExpression generateConstant(DorisDataType type, boolean isNullable) { - if (isNullable && Randomly.getBooleanWithSmallProbability()) { - createConstant(DorisDataType.NULL); + if (!isNullable) { + return createConstantWithoutNull(type); + } + if (Randomly.getBooleanWithSmallProbability()) { + return createConstant(DorisDataType.NULL); } return createConstant(type); } @@ -320,12 +327,29 @@ public DorisExpression generateConstant(DorisDataType type) { return createConstant(type); } + public DorisExpression createConstantWithoutNull(DorisDataType type) { + DorisExpression constant = createConstant(type); + int loopCount = 0; + while (constant instanceof DorisConstant.DorisNullConstant && loopCount < 1000) { + constant = createConstant(type); + loopCount++; + } + if (constant instanceof DorisConstant.DorisNullConstant) { + throw new IgnoreMeException(); + } + return constant; + } + public DorisExpression createConstant(DorisDataType type) { Randomly r = globalState.getRandomly(); long timestamp; switch (type) { case INT: if (globalState.getDbmsSpecificOptions().testIntConstants) { + long number = r.getInteger(); + if (DorisBugs.bug36351 && number == -1049190528) { + number = 0; + } return DorisConstant.createIntConstant(r.getInteger()); } return DorisConstant.createNullConstant(); @@ -368,6 +392,9 @@ public DorisExpression createConstant(DorisDataType type) { if (globalState.getDbmsSpecificOptions().testDateTimeConstants) { // [1970-01-01 08:00:00, 3000-01-01 00:00:00] timestamp = globalState.getRandomly().getLong(0, 32503651200L); + if (DorisBugs.bug36342) { + return DorisConstant.createDatetimeConstant(timestamp); + } return Randomly.fromOptions(DorisConstant.createDatetimeConstant(timestamp), DorisConstant.createDatetimeConstant()); } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java index d8076d3e5..1f3fe66f1 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -74,9 +74,21 @@ public void check() throws SQLException { state.getState().getLocalState().log( "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); - if (firstResult == null && secondResult != null - || firstResult != null && (!firstResult.contentEquals(secondResult) - && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { + if (firstResult == null && secondResult == null) { + return; + } + if (firstResult == null) { + throw new AssertionError(); + } + firstResult = firstResult.replace("\0", ""); + if (firstResult.contentEquals("0") && secondResult == null) { + return; + } + if (secondResult == null) { + throw new AssertionError(); + } + secondResult = secondResult.replace("\0", ""); + if (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { throw new AssertionError(); } @@ -94,6 +106,11 @@ private String createMetamorphicUnionQuery(DorisSelect select, DorisSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DorisSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DorisSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(groupByExpression); + middleSelect.setGroupByExpressions(groupByExpression); + rightSelect.setGroupByExpressions(groupByExpression); + } metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; metamorphicQuery += DorisToStringVisitor.asString(leftSelect) + " UNION ALL " + DorisToStringVisitor.asString(middleSelect) + " UNION ALL " @@ -185,9 +202,6 @@ private DorisSelect getSelect(List> aggregates, List allColumnValues = targetTables.getColumns().stream() + .map(c -> new DorisColumnValue(c, null)).collect(Collectors.toList()); HashSet columnOfLeafNode = new HashSet<>(); gen.setColumnOfLeafNode(columnOfLeafNode); initializeTernaryPredicateVariants(); select = new DorisSelect(); - columnOfLeafNode.addAll(targetTables.getColumns().stream().map(c -> new DorisColumnValue(c, null)) - .collect(Collectors.toList())); - groupByExpression = new ArrayList<>(columnOfLeafNode); + columnOfLeafNode.addAll(allColumnValues); + groupByExpression = new ArrayList<>(allColumnValues); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); List> tableList = tables.stream() diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java index 00a0cf28a..8e173d7ae 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java @@ -27,7 +27,7 @@ public DorisQueryPartitioningGroupByTester(DorisGlobalState state) { @Override public void check() throws SQLException { super.check(); - select.setGroupByExpressions(groupByExpression); + select.setGroupByExpressions(select.getFetchColumns()); select.setWhereClause(null); String originalQueryString = DorisToStringVisitor.asString(select); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java index 676065616..b3398f229 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -31,6 +31,7 @@ public void check() throws SQLException { if (Randomly.getBoolean()) { select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); } + select.setFetchColumns(groupByExpression); boolean orderBy = Randomly.getBoolean(); if (orderBy) { List> constants = new ArrayList<>(); diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 8cea624c7..baab98225 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -4,6 +4,7 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { diff --git a/test/sqlancer/dbms/TestDoris.java b/test/sqlancer/dbms/TestDoris.java deleted file mode 100644 index fd3c1e95a..000000000 --- a/test/sqlancer/dbms/TestDoris.java +++ /dev/null @@ -1,54 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; -import sqlancer.Randomly; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestDoris { - private final String host = "127.0.0.1"; - private final String port = "9030"; - private final String username = "sqlancer"; - private final String password = "sqlancer"; - - @Test - public void testdorisNoREC() { - String dorisAvailable = System.getenv("DORIS_AVAILABLE"); - boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); - assumeTrue(dorisIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--username", username, "--password", password, "--host", host, "--port", port, "doris", - "--oracle", "NOREC")); - } - - @Test - public void testdorisPQS() { - String dorisAvailable = System.getenv("DORIS_AVAILABLE"); - boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); - assumeTrue(dorisIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--username", username, "--password", password, "--host", host, "--port", port, "doris", - "--oracle", "PQS")); - } - - @Test - public void testdorisTLPQueryPartitioning() { - String dorisAvailable = System.getenv("DORIS_AVAILABLE"); - boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); - assumeTrue(dorisIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--username", username, "--password", password, "--host", host, "--port", port, "doris", - "--oracle", "QUERY_PARTITIONING")); - } -} diff --git a/test/sqlancer/dbms/TestDorisNoREC.java b/test/sqlancer/dbms/TestDorisNoREC.java new file mode 100644 index 000000000..76a9d50a8 --- /dev/null +++ b/test/sqlancer/dbms/TestDorisNoREC.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDorisNoREC { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DORIS_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "NOREC")); + } +} diff --git a/test/sqlancer/dbms/TestDorisPQS.java b/test/sqlancer/dbms/TestDorisPQS.java new file mode 100644 index 000000000..5760003d3 --- /dev/null +++ b/test/sqlancer/dbms/TestDorisPQS.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDorisPQS { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DORIS_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "PQS")); + } +} diff --git a/test/sqlancer/dbms/TestDorisTLP.java b/test/sqlancer/dbms/TestDorisTLP.java new file mode 100644 index 000000000..f8bea9e47 --- /dev/null +++ b/test/sqlancer/dbms/TestDorisTLP.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDorisTLP { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DORIS_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "QUERY_PARTITIONING")); + } +} From ad5d891bf683c24c760ae9638cd337751d6d4ce4 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Sun, 16 Jun 2024 14:28:24 +0800 Subject: [PATCH 0875/1284] [Databend] Pinned version to v1.2.452 (#946) * [Databend] Update error message for violation of not-null constraint * [Databend] Add expected error for table generation * [Databend] Add bug 15568 * [Databend] Add bug 15569 * [Databend] Add bug 15570 * [Databend] Add error messages returned from new constraints * [Databend] Add bug 15572 * [Databend] Pin CI to v1.2.452 --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/DatabendBugs.java | 4 ++++ src/sqlancer/databend/DatabendErrors.java | 8 +++++++- src/sqlancer/databend/DatabendProvider.java | 7 +++++++ .../databend/gen/DatabendNewExpressionGenerator.java | 10 ++++++++++ src/sqlancer/databend/gen/DatabendTableGenerator.java | 2 ++ 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9fb049f88..8e5269fb0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -176,7 +176,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend + image: datafuselabs/databend:v1.2.452 env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index eec2c39d0..dd11512d8 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -15,6 +15,10 @@ public final class DatabendBugs { public static boolean bug9236 = true; // https://github.com/datafuselabs/databend/issues/9236 public static boolean bug9264 = true; // https://github.com/datafuselabs/databend/issues/9264 public static boolean bug9806 = true; // https://github.com/datafuselabs/databend/issues/9806 + public static boolean bug15568 = true; // https://github.com/datafuselabs/databend/issues/15568 + public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 + public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 + public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 2f588480f..fdd8a3a69 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -21,6 +21,8 @@ public static List getExpressionErrors() { errors.add("no overload satisfies `not(Float64 NULL)`"); // TODO databend不允许出现not(float)，而a/b为float errors.add("no overload satisfies `not(Float64)`"); errors.add("number overflowed while evaluating function"); // 表达式数值溢出 + errors.add("Unable to get field named"); + errors.add("no overload satisfies `and_filters"); if (DatabendBugs.bug9162) { errors.add("downcast column error"); } @@ -42,6 +44,9 @@ public static List getExpressionErrors() { if (DatabendBugs.bug9806) { errors.add("segment pruning failure"); } + if (DatabendBugs.bug15568) { + errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE @@ -63,7 +68,7 @@ public static List getInsertErrors() { errors.add("Division by zero"); errors.add("/ by zero"); - errors.add("Can't cast column from null into non-nullable type"); + errors.add("violates not-null constraint"); errors.add("number overflowed while evaluating function `"); // 不能在int16类型column上插入int64的数据 return errors; @@ -80,6 +85,7 @@ public static List getGroupByErrors() { errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); errors.add("GROUP BY position"); + errors.add("GROUP BY items can't contain aggregate functions or window functions"); return errors; } diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 0ab0d4c9b..df7802027 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -133,6 +133,13 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE s.execute("USE " + databaseName); globalState.getState().logStatement("USE " + databaseName); } + if (DatabendBugs.bug15569) { + con.close(); + String urlWithRetry = String.format( + "jdbc:mysql://%s:%d/%s?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true&autoReconnect=true", + host, port, databaseName); + con = DriverManager.getConnection(urlWithRetry, username, password); + } return new SQLConnection(con); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index feb7b3483..2faf4376b 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -11,6 +11,7 @@ import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; @@ -160,6 +161,15 @@ DatabendExpression generateBooleanExpression(int depth) { allowAggregateFunctions = false; } List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (DatabendBugs.bug15570) { + validOptions.remove(BooleanExpression.LIKE); + validOptions.remove(BooleanExpression.IN_OPERATION); + validOptions.remove(BooleanExpression.BETWEEN); + validOptions.remove(BooleanExpression.BINARY_COMPARISON); + } + if (DatabendBugs.bug15572) { + validOptions.remove(BooleanExpression.NOT); + } BooleanExpression option = Randomly.fromList(validOptions); switch (option) { case POSTFIX_OPERATOR: diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index a34279fd2..ab4900c07 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -19,6 +20,7 @@ public class DatabendTableGenerator { public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); + DatabendErrors.addExpressionErrors(errors); StringBuilder sb = new StringBuilder(); String tableName = globalState.getSchema().getFreeTableName(); sb.append("CREATE TABLE "); From 3989d762ede46b8251a104fa66a294aa27897271 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 29 Jun 2024 16:27:43 +0800 Subject: [PATCH 0876/1284] [Databend] Refactor tests --- .github/workflows/main.yml | 4 +- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestDatabend.java | 108 ---------------------- test/sqlancer/dbms/TestDatabendNoREC.java | 23 +++++ test/sqlancer/dbms/TestDatabendPQS.java | 23 +++++ test/sqlancer/dbms/TestDatabendTLP.java | 21 +++++ 6 files changed, 71 insertions(+), 109 deletions(-) delete mode 100644 test/sqlancer/dbms/TestDatabend.java create mode 100644 test/sqlancer/dbms/TestDatabendNoREC.java create mode 100644 test/sqlancer/dbms/TestDatabendPQS.java create mode 100644 test/sqlancer/dbms/TestDatabendTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e5269fb0..6b920cd61 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -197,7 +197,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendTLP test + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendNoREC test + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendPQS test duckdb: name: DBMS Tests (DuckDB) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index baab98225..67c406fba 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -4,6 +4,7 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java deleted file mode 100644 index 707d7b8a9..000000000 --- a/test/sqlancer/dbms/TestDatabend.java +++ /dev/null @@ -1,108 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; -import sqlancer.Randomly; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestDatabend { - - @Test - public void testDatabendNoREC() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); - } - - @Test - public void testDatabendPQS() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); - } - - @Test - public void testDatabendTLPQueryPartitioning() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); - } - - // @Test - // public void testDatabendTLPWhere() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "WHERE")); - // } - // - // @Test - // public void testDatabendTLPGroupBy() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "GROUP_BY")); - // } - // - // @Test - // public void testDatabendTLPHaving() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "HAVING")); - // } - // - // @Test - // public void testDatabendTLPDistinct() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "DISTINCT")); - // } - // - // @Test - // public void testDatabendTLPAggregate() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "AGGREGATE")); - // } - -} diff --git a/test/sqlancer/dbms/TestDatabendNoREC.java b/test/sqlancer/dbms/TestDatabendNoREC.java new file mode 100644 index 000000000..679f8c161 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendNoREC.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.Randomly; + +public class TestDatabendNoREC { + + @Test + public void testDatabendNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); + } + +} diff --git a/test/sqlancer/dbms/TestDatabendPQS.java b/test/sqlancer/dbms/TestDatabendPQS.java new file mode 100644 index 000000000..fba733d44 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendPQS.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.Randomly; + +public class TestDatabendPQS { + + @Test + public void testDatabendPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); + } + +} diff --git a/test/sqlancer/dbms/TestDatabendTLP.java b/test/sqlancer/dbms/TestDatabendTLP.java new file mode 100644 index 000000000..27ba53416 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendTLP.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDatabendTLP { + + @Test + public void testDatabendTLPQueryPartitioning() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); + } +} From 37de4a2cf8b90ce7c22f3db37d85590d6dca873f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 29 Jun 2024 16:27:54 +0800 Subject: [PATCH 0877/1284] [Databend] Update to v1.2.542 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b920cd61..10cd3ace5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -176,7 +176,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.452 + image: datafuselabs/databend:v1.2.542-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer From 92fb68865daa18d1ca6e99ef7da1da6614a0abe3 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Mon, 8 Jul 2024 14:42:43 +0800 Subject: [PATCH 0878/1284] Remove duplicated logs in NoREC oracle (#953) --- src/sqlancer/common/oracle/NoRECOracle.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index acc7cffac..530488296 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -123,10 +123,6 @@ public Reproducer getLastReproducer() { private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(queryString); - } - int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { @@ -151,10 +147,6 @@ private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState< private int extractCounts(String queryString, ExpectedErrors errors, SQLGlobalState state) { SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(queryString); - } - int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { From b41b54ea82a9a0a24c6cc651c005de160389da34 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Sun, 7 Jul 2024 22:48:37 +0800 Subject: [PATCH 0879/1284] Initial DataFusion implementation --- .github/workflows/main.yml | 28 ++ README.md | 17 +- pom.xml | 23 + src/check_names.py | 1 + src/sqlancer/datafusion/DataFusionErrors.java | 44 ++ .../datafusion/DataFusionOptions.java | 43 ++ .../datafusion/DataFusionProvider.java | 135 +++++ src/sqlancer/datafusion/DataFusionSchema.java | 196 ++++++++ .../datafusion/DataFusionToStringVisitor.java | 99 ++++ src/sqlancer/datafusion/DataFusionUtil.java | 193 ++++++++ .../datafusion/ast/DataFusionConstant.java | 99 ++++ .../datafusion/ast/DataFusionExpression.java | 5 + .../datafusion/ast/DataFusionJoin.java | 92 ++++ .../datafusion/ast/DataFusionSelect.java | 67 +++ .../datafusion/gen/DataFusionBaseExpr.java | 258 ++++++++++ .../gen/DataFusionBaseExprFactory.java | 391 +++++++++++++++ .../gen/DataFusionExpressionGenerator.java | 227 +++++++++ .../gen/DataFusionInsertGenerator.java | 54 ++ .../gen/DataFusionTableGenerator.java | 33 ++ .../server/datafusion_server/Cargo.toml | 49 ++ .../server/datafusion_server/src/main.rs | 463 ++++++++++++++++++ .../test/DataFusionNoRECOracle.java | 95 ++++ .../test/DataFusionQueryPartitioningBase.java | 38 ++ ...ataFusionQueryPartitioningWhereTester.java | 67 +++ test/sqlancer/dbms/TestDataFusion.java | 21 + 25 files changed, 2730 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/datafusion/DataFusionErrors.java create mode 100644 src/sqlancer/datafusion/DataFusionOptions.java create mode 100644 src/sqlancer/datafusion/DataFusionProvider.java create mode 100644 src/sqlancer/datafusion/DataFusionSchema.java create mode 100644 src/sqlancer/datafusion/DataFusionToStringVisitor.java create mode 100644 src/sqlancer/datafusion/DataFusionUtil.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionConstant.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionExpression.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionJoin.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionSelect.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionBaseExpr.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionTableGenerator.java create mode 100644 src/sqlancer/datafusion/server/datafusion_server/Cargo.toml create mode 100644 src/sqlancer/datafusion/server/datafusion_server/src/main.rs create mode 100644 src/sqlancer/datafusion/test/DataFusionNoRECOracle.java create mode 100644 src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java create mode 100644 src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java create mode 100644 test/sqlancer/dbms/TestDataFusion.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e5269fb0..0f2a12c62 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -199,6 +199,34 @@ jobs: run: | DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test + datafusion: + name: DBMS Tests (DataFusion) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Run DataFusion Server + run: | + cd src/sqlancer/datafusion/server/datafusion_server + cargo run --features "datafusion_stable" & sleep 300 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + DATAFUSION_AVAILABLE=true mvn test -Pdatafusion-tests + duckdb: name: DBMS Tests (DuckDB) runs-on: ubuntu-latest diff --git a/README.md b/README.md index 83e207485..bb80d0794 100644 --- a/README.md +++ b/README.md @@ -56,11 +56,11 @@ Please find the `.bib` entries [here](docs/PAPERS.md). Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. | DBMS | Status | Expression Generation | Description | -|------------------------------|-------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ---------------------------- | ----------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | | MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | | PostgreSQL | Working | Typed | | -| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | +| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | | MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | | CockroachDB | Working | Typed | | | TiDB | Working | Untyped | | @@ -69,12 +69,13 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | | OceanBase | Working | Untyped | | | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | -| Databend | Working | Typed | | -| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | -| CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | -| Materialize |Working | Typed | -| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. -| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. +| Databend | Working | Typed | | +| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | +| CnosDB | Working | Typed | The implementation of CnosDB currently uses Restful API. | +| Materialize | Working | Typed | | +| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | +| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | +| DataFusion | Preliminary | Typed | Only basic SQL features are supported. | diff --git a/pom.xml b/pom.xml index 2b34d6c27..9bee83485 100644 --- a/pom.xml +++ b/pom.xml @@ -373,6 +373,11 @@ jsqlparser 4.6 + + org.apache.arrow + flight-sql-jdbc-driver + 16.1.0 + @@ -471,5 +476,23 @@ + + datafusion-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.3.0 + + + **/TestDataFusion.java + + --add-opens java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + diff --git a/src/check_names.py b/src/check_names.py index 63c1f093f..41694e060 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -38,6 +38,7 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["CnosDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cnosdb")) name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) + name_to_files["DataFusion"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "datafusion")) name_to_files["DuckDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "duckdb")) name_to_files["H2"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "h2")) name_to_files["HSQLDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "hsqldb")) diff --git a/src/sqlancer/datafusion/DataFusionErrors.java b/src/sqlancer/datafusion/DataFusionErrors.java new file mode 100644 index 000000000..d4b208f5b --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionErrors.java @@ -0,0 +1,44 @@ +package sqlancer.datafusion; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; + +import sqlancer.common.query.ExpectedErrors; + +public final class DataFusionErrors { + private DataFusionErrors() { + dfAssert(false, "Utility class cannot be instantiated"); + } + + /* + * During Oracle Checks, if ANY query returns one of the following error Then the current oracle check will be + * skipped. e.g.: NoREC Q1 -> throw an expected error NoREC Q2 -> succeed Since it's a known error, `SQLancer` will + * skip this check and don't report bug. + * + * Note now it's implemented this way for simplicity This way might cause false negative, because Q1 and Q2 should + * both succeed or both fail TODO(datafusion): ensure both succeed or both fail + */ + public static void registerExpectedExecutionErrors(ExpectedErrors errors) { + /* + * Expected + */ + errors.add("Error building plan"); // Randomly generated SQL is not valid and caused palning error + errors.add("Error during planning"); + errors.add("Execution error"); + errors.add("Overflow happened"); + errors.add("overflow"); + errors.add("Unsupported data type"); + errors.add("Divide by zero"); + /* + * Known bugs + */ + errors.add("to type Int64"); // https://github.com/apache/datafusion/issues/11252 + errors.add("bitwise"); // https://github.com/apache/datafusion/issues/11260 + errors.add("NestedLoopJoinExec"); // https://github.com/apache/datafusion/issues/11269 + /* + * False positives + */ + errors.add("Physical plan does not support logical expression AggregateFunction"); // False positive: when aggr + // is generated in where + // clause + } +} diff --git a/src/sqlancer/datafusion/DataFusionOptions.java b/src/sqlancer/datafusion/DataFusionOptions.java new file mode 100644 index 000000000..fcb0221a6 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionOptions.java @@ -0,0 +1,43 @@ +package sqlancer.datafusion; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.datafusion.DataFusionOptions.DataFusionOracleFactory; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.test.DataFusionNoRECOracle; +import sqlancer.datafusion.test.DataFusionQueryPartitioningWhereTester; + +@Parameters(commandDescription = "DataFusion") +public class DataFusionOptions implements DBMSSpecificOptions { + @Parameter(names = "--debug-info", description = "Show debug messages related to DataFusion", arity = 0) + public boolean showDebugInfo; + + @Override + public List getTestOracleFactory() { + return Arrays.asList(DataFusionOracleFactory.NOREC, DataFusionOracleFactory.QUERY_PARTITIONING_WHERE); + } + + public enum DataFusionOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DataFusionGlobalState globalState) throws SQLException { + return new DataFusionNoRECOracle(globalState); + } + }, + QUERY_PARTITIONING_WHERE { + @Override + public TestOracle create(DataFusionGlobalState globalState) throws SQLException { + return new DataFusionQueryPartitioningWhereTester(globalState); + } + } + } + +} diff --git a/src/sqlancer/datafusion/DataFusionProvider.java b/src/sqlancer/datafusion/DataFusionProvider.java new file mode 100644 index 000000000..161c324dd --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionProvider.java @@ -0,0 +1,135 @@ +package sqlancer.datafusion; + +import static java.lang.System.exit; +import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.DML; +import static sqlancer.datafusion.DataFusionUtil.displayTables; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionUtil.DataFusionInstanceID; +import sqlancer.datafusion.DataFusionUtil.DataFusionLogger; +import sqlancer.datafusion.gen.DataFusionInsertGenerator; +import sqlancer.datafusion.gen.DataFusionTableGenerator; + +@AutoService(DatabaseProvider.class) +public class DataFusionProvider extends SQLProviderAdapter { + + public DataFusionProvider() { + super(DataFusionGlobalState.class, DataFusionOptions.class); + } + + @Override + public void generateDatabase(DataFusionGlobalState globalState) throws Exception { + int tableCount = Randomly.fromOptions(1, 2, 3, 4, 5, 6, 7); + for (int i = 0; i < tableCount; i++) { + SQLQueryAdapter queryCreateRandomTable = new DataFusionTableGenerator().getQuery(globalState); + queryCreateRandomTable.execute(globalState); + globalState.updateSchema(); + globalState.dfLogger.appendToLog(DML, queryCreateRandomTable.toString() + "\n"); + } + + // Now only `INSERT` DML is supported + // If more DMLs are added later, should use`StatementExecutor` instead + // (see DuckDB's implementation for reference) + + globalState.updateSchema(); + List allTables = globalState.getSchema().getDatabaseTables(); + List allTablesName = allTables.stream().map(t -> t.getName()).collect(Collectors.toList()); + if (allTablesName.isEmpty()) { + System.out.println("Generate database failed"); + exit(1); + } + + // Randomly insert some data into existing tables + for (DataFusionTable table : allTables) { + int nInsertQuery = globalState.getRandomly().getInteger(0, 8); // [0, 10) + + for (int i = 0; i < nInsertQuery; i++) { + SQLQueryAdapter insertQuery = null; + try { + insertQuery = DataFusionInsertGenerator.getQuery(globalState, table); + } catch (IgnoreMeException e) { + // Only for special case: table has 0 column + continue; + } + + insertQuery.execute(globalState); + globalState.dfLogger.appendToLog(DML, insertQuery.toString() + "\n"); + } + } + + // TODO(datafusion) add `DataFUsionLogType.STATE` for this whole db state log + if (globalState.getDbmsSpecificOptions().showDebugInfo) { + System.out.println(displayTables(globalState, allTablesName)); + } + } + + @Override + public SQLConnection createDatabase(DataFusionGlobalState globalState) throws SQLException { + if (globalState.getDbmsSpecificOptions().showDebugInfo) { + System.out.println("A new database get created!\n"); + } + Properties props = new Properties(); + props.setProperty("UseEncryption", "false"); + // must set 'user' and 'password' to trigger server 'do_handshake()' + props.setProperty("user", "foo"); + props.setProperty("password", "bar"); + props.setProperty("create", globalState.getDatabaseName()); // Hack: use this property to let DataFusion server + // clear the current context + String url = "jdbc:arrow-flight-sql://127.0.0.1:50051"; + Connection connection = DriverManager.getConnection(url, props); + + return new SQLConnection(connection); + } + + @Override + public String getDBMSName() { + return "datafusion"; + } + + // If run SQLancer with multiple thread + // Each thread's instance will have its own `DataFusionGlobalState` + // It will store global states including: + // JDBC connection to DataFusion server + // Logger for this thread + public static class DataFusionGlobalState extends SQLGlobalState { + public DataFusionLogger dfLogger; + DataFusionInstanceID id; + + public DataFusionGlobalState() { + // HACK: test will only run in spawned thread, not main thread + // this way redundant logger files won't be created + if (Thread.currentThread().getName().equals("main")) { + return; + } + + id = new DataFusionInstanceID(Thread.currentThread().getName()); + try { + dfLogger = new DataFusionLogger(this, id); + } catch (Exception e) { + throw new IgnoreMeException(); + } + } + + @Override + protected DataFusionSchema readSchema() throws SQLException { + return DataFusionSchema.fromConnection(getConnection(), getDatabaseName()); + } + } +} diff --git a/src/sqlancer/datafusion/DataFusionSchema.java b/src/sqlancer/datafusion/DataFusionSchema.java new file mode 100644 index 000000000..4a5ed803a --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionSchema.java @@ -0,0 +1,196 @@ +package sqlancer.datafusion; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.ast.DataFusionConstant; +import sqlancer.datafusion.ast.DataFusionExpression; + +public class DataFusionSchema extends AbstractSchema { + + public DataFusionSchema(List databaseTables) { + super(databaseTables); + } + + // update existing tables in DB by query again + // (like `show tables;`) + public static DataFusionSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + DataFusionTable t = new DataFusionTable(tableName, databaseColumns, isView); + for (DataFusionColumn c : databaseColumns) { + c.setTable(t); + } + + databaseTables.add(t); + } + + return new DataFusionSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("select table_name " + "from information_schema.tables " + + "where table_schema='public'" + "order by table_name; ")) { + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + String.format("select * from information_schema.columns where table_name = '%s';", tableName))) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + boolean isNullable = rs.getString("is_nullable").contentEquals("YES"); + + DataFusionColumn c = new DataFusionColumn(columnName, + DataFusionDataType.parseFromDataFusionCatalog(dataType), isNullable); + columns.add(c); + } + } + } + + return columns; + } + + /* + * When adding a new type: 1. Update all methods inside this enum 2. Update all `DataFusionBaseExpr`'s signature, if + * it can support new type (in `DataFusionBaseExprFactory.java` + * + * Types are 'SQL DataType' in DataFusion's documentation + * https://datafusion.apache.org/user-guide/sql/data_types.html + */ + public enum DataFusionDataType { + + BIGINT, DOUBLE, BOOLEAN, NULL; + + public static DataFusionDataType getRandomWithoutNull() { + DataFusionDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DataFusionDataType.NULL); + return dt; + } + + // How to parse type in DataFusion's catalog to `DataFusionDataType` + // As displayed in: + // create table t1(v1 int, v2 bigint); + // select table_name, column_name, data_type from information_schema.columns; + public static DataFusionDataType parseFromDataFusionCatalog(String typeString) { + switch (typeString) { + case "Int64": + return DataFusionDataType.BIGINT; + case "Float64": + return DataFusionDataType.DOUBLE; + case "Boolean": + return DataFusionDataType.BOOLEAN; + default: + dfAssert(false, "Unreachable. All branches should be eovered"); + } + + dfAssert(false, "Unreachable. All branches should be eovered"); + return null; + } + + // TODO(datafusion) lots of hack here, should build our own Randomly later + public Node getRandomConstant(DataFusionGlobalState state) { + if (Randomly.getBooleanWithSmallProbability()) { + return DataFusionConstant.createNullConstant(); + } + switch (this) { + case BIGINT: + return DataFusionConstant.createIntConstant(state.getRandomly().getInteger()); + case BOOLEAN: + return new DataFusionConstant.DataFusionBooleanConstant(Randomly.getBoolean()); + case DOUBLE: + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + Double randomDouble = state.getRandomly().getDouble(); // [0.0, 1.0); + Double scaledDouble = (randomDouble - 0.5) * 2 * Double.MAX_VALUE; + return new DataFusionConstant.DataFusionDoubleConstant(scaledDouble); + } + String doubleStr = Randomly.fromOptions("'NaN'::Double", "'+Inf'::Double", "'-Inf'::Double", "-0.0", + "+0.0"); + return new DataFusionConstant.DataFusionDoubleConstant(doubleStr); + } + + return new DataFusionConstant.DataFusionDoubleConstant(state.getRandomly().getDouble()); + case NULL: + return DataFusionConstant.createNullConstant(); + default: + dfAssert(false, "Unreachable. All branches should be eovered"); + } + + dfAssert(false, "Unreachable. All branches should be eovered"); + return DataFusionConstant.createNullConstant(); + } + } + + public static class DataFusionColumn extends AbstractTableColumn { + + private final boolean isNullable; + + public DataFusionColumn(String name, DataFusionDataType columnType, boolean isNullable) { + super(name, null, columnType); + this.isNullable = isNullable; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class DataFusionTable + extends AbstractRelationalTable { + + public DataFusionTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + public static List getAllColumns(List tables) { + return tables.stream().map(AbstractTable::getColumns).flatMap(List::stream).collect(Collectors.toList()); + } + + public static List getRandomColumns(List tables) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new DataFusionColumn("*", DataFusionDataType.NULL, true)); + } + + List allColumns = getAllColumns(tables); + + return Randomly.nonEmptySubset(allColumns); + } + } + +} diff --git a/src/sqlancer/datafusion/DataFusionToStringVisitor.java b/src/sqlancer/datafusion/DataFusionToStringVisitor.java new file mode 100644 index 000000000..1f3030552 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionToStringVisitor.java @@ -0,0 +1,99 @@ +package sqlancer.datafusion; + +import java.util.List; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.datafusion.ast.DataFusionConstant; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionJoin; +import sqlancer.datafusion.ast.DataFusionSelect; + +public class DataFusionToStringVisitor extends NewToStringVisitor { + + public static String asString(Node expr) { + DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + public static String asString(List> exprs) { + DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); + visitor.visit(exprs); + return visitor.get(); + } + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof DataFusionConstant) { + visit((DataFusionConstant) expr); + } else if (expr instanceof DataFusionSelect) { + visit((DataFusionSelect) expr); + } else if (expr instanceof DataFusionJoin) { + visit((DataFusionJoin) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(DataFusionJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(DataFusionConstant constant) { + sb.append(constant.toString()); + } + + private void visit(DataFusionSelect select) { + sb.append("SELECT "); + if (select.fetchColumnsString.isPresent()) { + sb.append(select.fetchColumnsString.get()); + } else { + visit(select.getFetchColumns()); + } + + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + +} diff --git a/src/sqlancer/datafusion/DataFusionUtil.java b/src/sqlancer/datafusion/DataFusionUtil.java new file mode 100644 index 000000000..e37ad8d64 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionUtil.java @@ -0,0 +1,193 @@ +package sqlancer.datafusion; + +import static java.lang.System.exit; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; + +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; + +public final class DataFusionUtil { + private DataFusionUtil() { + dfAssert(false, "Utility class cannot be instantiated"); + } + + // Display tables in `fromTableNames` + public static String displayTables(DataFusionGlobalState state, List fromTableNames) { + StringBuilder resultStringBuilder = new StringBuilder(); + for (String tableName : fromTableNames) { + String query = String.format("select * from %s", tableName); + try (Statement stat = state.getConnection().createStatement(); + ResultSet wholeTable = stat.executeQuery(query)) { + + ResultSetMetaData metaData = wholeTable.getMetaData(); + int columnCount = metaData.getColumnCount(); + + resultStringBuilder.append("Table: ").append(tableName).append("\n"); + for (int i = 1; i <= columnCount; i++) { + resultStringBuilder.append(metaData.getColumnName(i)).append(" (") + .append(metaData.getColumnTypeName(i)).append(")"); + if (i < columnCount) { + resultStringBuilder.append(", "); + } + } + resultStringBuilder.append("\n"); + + while (wholeTable.next()) { + for (int i = 1; i <= columnCount; i++) { + resultStringBuilder.append(wholeTable.getString(i)); + if (i < columnCount) { + resultStringBuilder.append(", "); + } + } + resultStringBuilder.append("\n"); + } + resultStringBuilder.append("----------------------------------------\n\n"); + + } catch (SQLException err) { + resultStringBuilder.append("Table: ").append(tableName).append("\n"); + resultStringBuilder.append("----------------------------------------\n\n"); + // resultStringBuilder.append("Error retrieving data from table ").append(tableName).append(": + // ").append(err.getMessage()).append("\n"); + } + } + + return resultStringBuilder.toString(); + } + + public static void dfAssert(boolean condition, String message) { + if (!condition) { + String methodName = Thread.currentThread().getStackTrace()[2].getMethodName(); + System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message); + exit(1); + } + } + + /* + * Fetch all DMLs from logs/database*-cur.log + */ + public static String getReplay(String dbname) { + String path = "./logs/datafusion/" + dbname + "-cur.log"; + String absolutePath = Paths.get(path).toAbsolutePath().toString(); + + StringBuilder reproducer = new StringBuilder(); + + try (BufferedReader reader = new BufferedReader(new FileReader(absolutePath))) { + String line; + while ((line = reader.readLine()) != null) { + // Check if the line contains the /*DML*/ marker + if (line.contains("/*DML*/")) { + reproducer.append(line).append("\n"); + } + } + } catch (IOException e) { + System.err.println("Error reading from file: " + e.getMessage()); + } + + return reproducer.toString(); + } + + // UID for different fuzzer runs + public static class DataFusionInstanceID { + private final String id; + + public DataFusionInstanceID(String dfID) { + id = dfID; + } + + @Override + public String toString() { + return id; // Return the id field when toString is called + } + } + + /* + * Extra logs stored in 'logs/datafusion_custom_log/' In case re-run overwrite previous logs + */ + public static class DataFusionLogger { + private final DataFusionInstanceID dfID; + private final DataFusionGlobalState state; + /* + * Log file handles + */ + private final File errorLogFile; + + public DataFusionLogger(DataFusionGlobalState globalState, DataFusionInstanceID id) throws Exception { + this.state = globalState; + this.dfID = id; + + // Setup datafusion_custom_log folder + File baseDir = new File("logs/datafusion_custom_log/"); + if (!baseDir.exists() && !baseDir.mkdirs()) { + throw new IOException("Failed to create 'datafusion_custom_log' directory/"); + } + + // Setup error.log + errorLogFile = new File(baseDir, "error_report.log"); + errorLogFile.createNewFile(); + } + + // Caller is responsible for adding '\n' at the end of logContent + public void appendToLog(DataFusionLogType logType, String logContent) { + FileWriter logFileWriter = null; + + // Determine which log file to use based on the LogType + String logLineHeader = ""; + switch (logType) { + case ERROR: + try { + logFileWriter = new FileWriter(errorLogFile, true); + } catch (IOException e) { + System.out.println("Failed to create FileWriter for errorLogFIle"); + e.printStackTrace(); + exit(1); + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String formattedDateTime = LocalDateTime.now().format(formatter); + logLineHeader = "Run@" + formattedDateTime + " (" + dfID + ")\n"; + break; + case DML: + logFileWriter = state.getLogger().getCurrentFileWriter(); + logLineHeader = "/*DML*/"; + break; + case SELECT: + logFileWriter = state.getLogger().getCurrentFileWriter(); + break; + default: + dfAssert(false, "All branch should be covered"); + } + + // Append content to the appropriate log file + if (logFileWriter != null) { + try { + logFileWriter.write(logLineHeader); + logFileWriter.write(logContent); + logFileWriter.flush(); + } catch (IOException e) { + System.out.println("Failed to write to " + logType + " log: " + e.getMessage()); + e.printStackTrace(); + exit(1); + } + } else { + System.out.println("appending to log failed"); + Thread.currentThread().getStackTrace(); + exit(1); + } + } + + public enum DataFusionLogType { + ERROR, DML, SELECT + } + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionConstant.java b/src/sqlancer/datafusion/ast/DataFusionConstant.java new file mode 100644 index 000000000..d123adb8a --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionConstant.java @@ -0,0 +1,99 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.newast.Node; + +public class DataFusionConstant implements Node { + + private DataFusionConstant() { + } + + public static Node createIntConstant(long val) { + return new DataFusionIntConstant(val); + } + + public static Node createNullConstant() { + return new DataFusionNullConstant(); + } + + public static class DataFusionNullConstant extends DataFusionConstant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class DataFusionIntConstant extends DataFusionConstant { + + private final long value; + + public DataFusionIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class DataFusionDoubleConstant extends DataFusionConstant { + + private final String valueStr; + + public DataFusionDoubleConstant(double value) { + if (value == Double.POSITIVE_INFINITY) { + valueStr = "'+Inf'::Double"; + } else if (value == Double.NEGATIVE_INFINITY) { + valueStr = "'-Inf'::Double"; + } else if (Double.isNaN(value)) { + valueStr = "'NaN'::Double"; + } else if (Double.compare(value, -0.0) == 0) { + valueStr = "-0.0"; + } else { + valueStr = String.valueOf(value); + } + } + + // Make it more convenient to construct special value like -0, NaN, etc. + public DataFusionDoubleConstant(String valueStr) { + this.valueStr = valueStr; + } + + @Override + public String toString() { + return valueStr; + } + + } + + public static class DataFusionBooleanConstant extends DataFusionConstant { + + private final boolean value; + + public DataFusionBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + if (value) { + return "true"; + } else { + return "false"; + } + } + + } + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionExpression.java b/src/sqlancer/datafusion/ast/DataFusionExpression.java new file mode 100644 index 000000000..6da95cd93 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionExpression.java @@ -0,0 +1,5 @@ +package sqlancer.datafusion.ast; + +public interface DataFusionExpression { + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionJoin.java b/src/sqlancer/datafusion/ast/DataFusionJoin.java new file mode 100644 index 000000000..61cd9f556 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionJoin.java @@ -0,0 +1,92 @@ +package sqlancer.datafusion.ast; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +/* + NOT IMPLEMENTED YET + */ +public class DataFusionJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + + public DataFusionJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public static List> getJoins(List tables, + DataFusionGlobalState globalState) { + // [t1_join_t2, t1_join_t3, ...] + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)) + .collect(Collectors.toList()); + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + // TODO(datafusion) this `joinGen` can generate super chaotic exprsions, maybe we should make it more like a + // normal join expression + DataFusionExpressionGenerator joinGen = new DataFusionExpressionGenerator(globalState).setColumns(columns); + switch (DataFusionJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(DataFusionJoin.createInnerJoin(leftTable, rightTable, + joinGen.generateExpression(DataFusionSchema.DataFusionDataType.BOOLEAN))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static DataFusionJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DataFusionJoin(left, right, JoinType.INNER, predicate); + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + public enum JoinType { + INNER; + // NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionSelect.java b/src/sqlancer/datafusion/ast/DataFusionSelect.java new file mode 100644 index 000000000..a80758c4a --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionSelect.java @@ -0,0 +1,67 @@ +package sqlancer.datafusion.ast; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +public class DataFusionSelect extends SelectBase> implements Node { + public Optional fetchColumnsString = Optional.empty(); // When available, override `fetchColumns` in base + // class's `Node` representation (for display) + public DataFusionExpressionGenerator exprGen; + + // Construct a `DataFusionSelect` with random SELECT, FROM, WHERE + public static DataFusionSelect getRandomSelect(DataFusionGlobalState state) { + DataFusionSelect randomSelect = new DataFusionSelect(); + + // Randomly pick up to 4 tables to select from + DataFusionSchema schema = state.getSchema(); // schema of all tables + List allTables = schema.getDatabaseTables(); + List randomTables = Randomly.nonEmptySubset(allTables); + int maxSize = Randomly.fromOptions(1, 2, 3, 4); + if (randomTables.size() > maxSize) { + randomTables = randomTables.subList(0, maxSize); + } + + // Randomly choose some columns from `randomTables` + // And generate a random expression which might contain those columns + List randomColumns = DataFusionTable.getRandomColumns(randomTables); + randomSelect.exprGen = new DataFusionExpressionGenerator(state).setColumns(randomColumns); + Node whereExpr = randomSelect.exprGen + .generateExpression(DataFusionSchema.DataFusionDataType.BOOLEAN); + + // Constructing result + List> randomTableNodes = randomTables.stream() + .map(t -> new TableReferenceNode(t)) + .collect(Collectors.toList()); + List> randomColumnNodes = randomColumns.stream() + .map((c) -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + + randomSelect.setFetchColumns(randomColumnNodes); + randomSelect.setFromList(randomTableNodes); + randomSelect.setWhereClause(whereExpr); + + return randomSelect; + } + + /* + * If set fetch columns with string It will override `fetchColumns` in base class when + * `DataFusionToStringVisitor.asString()` is called + * + * This method can be helpful to mutate select in oracle checks: SELECT [expr] ... -> SELECT SUM[expr] + */ + public void setFetchColumnsString(String selectExpr) { + this.fetchColumnsString = Optional.of(selectExpr); + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionBaseExpr.java b/src/sqlancer/datafusion/gen/DataFusionBaseExpr.java new file mode 100644 index 000000000..0be57486e --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionBaseExpr.java @@ -0,0 +1,258 @@ +package sqlancer.datafusion.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; + +/* + Notes for adding new `DataFusionBaseExpr` support: + + Expression ASTs are constructed with Node<> class, it can traverse expression and convert AST to String representation + `DataFusionBaseExpr` implements `Operator<>` class, which is a field inside `Node<>` class, it includes operator properties like number of arguments, signature, or is this operator prefix/suffix, etc. + + To add new base expr (scalar functions, operators like '<<', 'AND' are all base expr): + 1. Add an enum variant to `DataFusionBaseExprType` + 2. Update `DataFusionBaseExprFactory.java` + (If a function support different argument number, make a new entry for each one. e.g. round scalar function support round(3.14) / round(3.14, 1), so it should be enum FUNC_ROUND1, FUNC_ROUND2) + */ +public class DataFusionBaseExpr implements Operator { + public String name; + public int nArgs; // number of input arguments + public DataFusionBaseExprCategory exprType; + public List possibleReturnTypes; + public List argTypes; + public boolean isVariadic; // Function supports arbitrary number of arguments, if set to `true`, it will + // override `nArgs` + + // Primary constructor + DataFusionBaseExpr(String name, int nArgs, DataFusionBaseExprCategory exprCategory, + List possibleReturnTypes, List argTypes, boolean isVariadic) { + this.name = name; + this.nArgs = nArgs; + this.exprType = exprCategory; + this.possibleReturnTypes = possibleReturnTypes; + this.argTypes = argTypes; + this.isVariadic = isVariadic; + } + + // Overloaded constructor assuming 'isVariadic' is false + DataFusionBaseExpr(String name, int nArgs, DataFusionBaseExprCategory exprCategory, + List possibleReturnTypes, List argTypes) { + this(name, nArgs, exprCategory, possibleReturnTypes, argTypes, false); + } + + public static DataFusionBaseExpr createCommonNumericFuncSingleArg(String name) { + return new DataFusionBaseExpr(name, 1, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + } + + public static DataFusionBaseExpr createCommonNumericAggrFuncSingleArg(String name) { + return new DataFusionBaseExpr(name, 1, DataFusionBaseExprCategory.AGGREGATE, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + } + + public static DataFusionBaseExpr createCommonNumericFuncTwoArgs(String name) { + return new DataFusionBaseExpr(name, 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + } + + @Override + public String getTextRepresentation() { + return name; + } + + @Override + public String toString() { + return name; + } + + /* + * Class/Enum for `DataFusionBaseExpr` fields + */ + // Used to construct `src.common.ast.*Node` + public enum DataFusionBaseExprCategory { + UNARY_PREFIX, UNARY_POSTFIX, BINARY, FUNC, AGGREGATE + } + + /* + * Operators reference: https://datafusion.apache.org/user-guide/sql/operators.html Scalar functions: + * https://datafusion.apache.org/user-guide/sql/scalar_functions.html + */ + public enum DataFusionBaseExprType { + // Null Operators + IS_NULL, // IS NULL + IS_NOT_NULL, // IS NOT NULL + + // Numeric Operators + ADD, // 1 + 1 + SUB, // 1 - 1 + MULTIPLICATION, // 2 * 3 + DIVISION, // 8 / 4 + MODULO, // 5 % 3 + + // Comparison Operators + EQUAL, // 1 = 1 + EQUAL2, // 1 == 1 + NOT_EQUAL, // 1 != 2 + LESS_THAN, // 3 < 4 + LESS_THAN_OR_EQUAL_TO, // 3 <= 3 + GREATER_THAN, // 6 > 5 + GREATER_THAN_OR_EQUAL_TO, // 5 >= 5 + + // Distinctness operators + IS_DISTINCT_FROM, // 0 IS DISTINCT FROM NULL + IS_NOT_DISTINCT_FROM, // NULL IS NOT DISTINCT FROM NULL + + /* + * // Regular expression match operators REGEX_MATCH, // 'datafusion' ~ '^datafusion(-cli)*' + * REGEX_CASE_INSENSITIVE_MATCH, // 'datafusion' ~* '^DATAFUSION(-cli)*' NOT_REGEX_MATCH, // 'datafusion' !~ + * '^DATAFUSION(-cli)*' NOT_REGEX_CASE_INSENSITIVE_MATCH, // 'datafusion' !~* '^DATAFUSION(-cli)+' + * + * // Like pattern match operators LIKE_MATCH, // 'datafusion' ~~ 'dat_f%n' CASE_INSENSITIVE_LIKE_MATCH, // + * 'datafusion' ~~* 'Dat_F%n' NOT_LIKE_MATCH, // 'datafusion' !~~ 'Dat_F%n' NOT_CASE_INSENSITIVE_LIKE_MATCH // + * 'datafusion' !~~* 'Dat%F_n' + */ + + // Logical Operators + AND, // true and true + OR, // true or false + + // Bitwise Operators + BITWISE_AND, // 5 & 3 + BITWISE_OR, // 5 | 3 + BITWISE_XOR, // 5 ^ 3 + BITWISE_SHIFT_RIGHT, // 5 >> 3 + BITWISE_SHIFT_LEFT, // 5 << 3 + + /* + * // Other operators STRING_CONCATENATION, // 'Hello, ' || 'DataFusion!' ARRAY_CONTAINS, // + * make_array(1,2,3) @> make_array(1,3) ARRAY_IS_CONTAINED_BY // make_array(1,3) <@ make_array(1,2,3) + */ + + // Unary Prefix Operators + NOT, // NOT true + PLUS, // +7 + MINUS, // -3 + + /* + * Scalar Functions + */ + + // Math Functions + FUNC_ABS, // abs(-10) + FUNC_ACOS, // acos(1) + FUNC_ACOSH, // acosh(10) + FUNC_ASIN, // asin(1) + FUNC_ASINH, // asinh(1) + FUNC_ATAN, // atan(1) + FUNC_ATANH, // atanh(0.5) + FUNC_ATAN2, // atan2(10, 10) + FUNC_CBRT, // cbrt(27) + FUNC_CEIL, // ceil(9.2) + FUNC_COS, // cos(π/3) + FUNC_COSH, // cosh(0) + FUNC_DEGREES, // degrees(π) + FUNC_EXP, // exp(1) + FUNC_FACTORIAL, // factorial(5) + FUNC_FLOOR, // floor(3.7) + FUNC_GCD, // gcd(8, 12) + FUNC_ISNAN, // isnan(NaN) + FUNC_ISZERO, // iszero(0.0) + FUNC_LCM, // lcm(5, 15) + FUNC_LN, // ln(1) + FUNC_LOG, // log(100) + FUNC_LOG_WITH_BASE, // log(10, 100) + FUNC_LOG10, // log10(100) + FUNC_LOG2, // log2(32) + FUNC_NANVL, // nanvl(NaN, 3) + FUNC_PI, // pi() + FUNC_POW, // pow(2, 3) + FUNC_POWER, // power(2, 3) + FUNC_RADIANS, // radians(180) + // FUNC_RANDOM, // random() disabled because it's non-deterministic + FUNC_ROUND, // round(3.14159) + FUNC_ROUND_WITH_DECIMAL, // round(3.14159, 2) + FUNC_SIGNUM, // signum(-10) + FUNC_SIN, // sin(π/2) + FUNC_SINH, // sinh(1) + FUNC_SQRT, // sqrt(16) + FUNC_TAN, // tan(π/4) + FUNC_TANH, // tanh(1) + FUNC_TRUNC, // trunc(3.14159) + FUNC_TRUNC_WITH_DECIMAL, // trunc(3.14159, 2) + + // Conditional Functions + FUNC_COALESCE, // coalesce(NULL, 'default value') + FUNC_NULLIF, // nullif('value', 'value') + FUNC_NVL, // nvl(NULL, 'default value') + FUNC_NVL2, // nvl2('not null', 'return if not null', 'return if null') + FUNC_IFNULL, // ifnull(NULL, 'default value') + + // String Functions + + // Time and Date Functions + + // Array Functions + + // Struct Functions + + // Hashing Functions + + // Other Functions + + // Aggregate Functions + AGGR_MIN, AGGR_MAX, AGGR_SUM, AGGR_AVG, AGGR_COUNT, + } + + /* + * Because expressions are constructed in a top-down way, we have to infer argument type given return type. For each + * arg, if its corresponding element is `SameAsReturnType`, it should be the same as the type of expression's + * evaluated value. Else, it should be specific `DataFusionDataType` + * + * e.g. let's say we're generating a round(num, digit) of double type, its `argTypes` is: Arrays.asList( new + * ArgumentType.SameAsReturnType(), // First arg type as return type new ArgumentType.Fixed(new + * ArrayList<>(Array.asList(DataFusionDataType.INT)) // Second arg always Integer ) it means: its first argument + * should be the same as returned type (double), and the second arg should always be Int. + * + * Random expression generator's policy: SameAsReturnType -> generate an expr with the same type as its return type + * SameAsReturnType -> generate an expr with the same type as its 1st arg type Fixed(type1, type2, ... typeN) -> + * randomly choose a possible type (It will also generate completely random type/null ~10%) + * + * Note this defination is not comprehensive for native `DataFusion` types. It's just for simplicity and should + * cover most common cases + */ + public abstract static class ArgumentType { + private ArgumentType() { + } + + public static class SameAsReturnType extends ArgumentType { + } + + public static class SameAsFirstArgType extends ArgumentType { + } + + public static class Fixed extends ArgumentType { + public List fixedType; // It's a list to support different possible arg types. + + public Fixed(List fixedType) { + this.fixedType = fixedType; + } + + public List getType() { + return fixedType; + } + } + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java b/src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java new file mode 100644 index 000000000..d3fe39972 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java @@ -0,0 +1,391 @@ +package sqlancer.datafusion.gen; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; +import static sqlancer.datafusion.gen.DataFusionBaseExpr.createCommonNumericAggrFuncSingleArg; +import static sqlancer.datafusion.gen.DataFusionBaseExpr.createCommonNumericFuncSingleArg; +import static sqlancer.datafusion.gen.DataFusionBaseExpr.createCommonNumericFuncTwoArgs; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; +import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprCategory; +import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; + +public final class DataFusionBaseExprFactory { + private DataFusionBaseExprFactory() { + dfAssert(false, "Utility class cannot be instantiated"); + } + + public static DataFusionBaseExpr createExpr(DataFusionBaseExprType type) { + switch (type) { + case IS_NULL: + return new DataFusionBaseExpr("IS NULL", 1, DataFusionBaseExprCategory.UNARY_POSTFIX, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.DOUBLE, DataFusionDataType.BIGINT, DataFusionDataType.NULL))))); + case IS_NOT_NULL: + return new DataFusionBaseExpr("IS NOT NULL", 1, DataFusionBaseExprCategory.UNARY_POSTFIX, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.DOUBLE, DataFusionDataType.BIGINT, DataFusionDataType.NULL))))); + case BITWISE_AND: + return new DataFusionBaseExpr("&", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case BITWISE_OR: + return new DataFusionBaseExpr("|", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case BITWISE_XOR: + return new DataFusionBaseExpr("^", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case BITWISE_SHIFT_RIGHT: + return new DataFusionBaseExpr(">>", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case BITWISE_SHIFT_LEFT: + return new DataFusionBaseExpr("<<", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case NOT: + return new DataFusionBaseExpr("NOT", 1, DataFusionBaseExprCategory.UNARY_PREFIX, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))))); + case PLUS: // unary prefix '+' + return new DataFusionBaseExpr("+", 1, DataFusionBaseExprCategory.UNARY_PREFIX, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case MINUS: // unary prefix '-' + return new DataFusionBaseExpr("-", 1, DataFusionBaseExprCategory.UNARY_PREFIX, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case MULTIPLICATION: + return new DataFusionBaseExpr("*", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case DIVISION: + return new DataFusionBaseExpr("/", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case MODULO: + return new DataFusionBaseExpr("%", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case EQUAL: + return new DataFusionBaseExpr("=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case EQUAL2: + return new DataFusionBaseExpr("==", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case NOT_EQUAL: + return new DataFusionBaseExpr("!=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case LESS_THAN: + return new DataFusionBaseExpr("<", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case LESS_THAN_OR_EQUAL_TO: + return new DataFusionBaseExpr("<=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case GREATER_THAN: + return new DataFusionBaseExpr(">", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case GREATER_THAN_OR_EQUAL_TO: + return new DataFusionBaseExpr(">=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case IS_DISTINCT_FROM: + return new DataFusionBaseExpr("IS DISTINCT FROM", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case IS_NOT_DISTINCT_FROM: + return new DataFusionBaseExpr("IS NOT DISTINCT FROM", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case AND: + return new DataFusionBaseExpr("AND", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))), // arg1 + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))) // arg2 + )); + case OR: + return new DataFusionBaseExpr("OR", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))), // arg1 + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))) // arg2 + )); + case ADD: // binary arithmetic operator '+' + return new DataFusionBaseExpr("+", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))), // arg1 + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))) // arg2 + )); + case SUB: // binary arithmetic operator '-' + return new DataFusionBaseExpr("-", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), // arg1 + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))) // arg2 + )); + case FUNC_ABS: + return createCommonNumericFuncSingleArg("ABS"); + case FUNC_ACOS: + return createCommonNumericFuncSingleArg("ACOS"); + case FUNC_ACOSH: + return createCommonNumericFuncSingleArg("ACOSH"); + case FUNC_ASIN: + return createCommonNumericFuncSingleArg("ASIN"); + case FUNC_ASINH: + return createCommonNumericFuncSingleArg("ASINH"); + case FUNC_ATAN: + return createCommonNumericFuncSingleArg("ATAN"); + case FUNC_ATANH: + return createCommonNumericFuncSingleArg("ATANH"); + case FUNC_ATAN2: + return createCommonNumericFuncTwoArgs("ATAN2"); + case FUNC_CBRT: + return createCommonNumericFuncSingleArg("CBRT"); + case FUNC_CEIL: + return createCommonNumericFuncSingleArg("CEIL"); + case FUNC_COS: + return createCommonNumericFuncSingleArg("COS"); + case FUNC_COSH: + return createCommonNumericFuncSingleArg("COSH"); + case FUNC_DEGREES: + return createCommonNumericFuncSingleArg("DEGREES"); + case FUNC_EXP: + return createCommonNumericFuncSingleArg("EXP"); + case FUNC_FACTORIAL: + return createCommonNumericFuncSingleArg("FACTORIAL"); + case FUNC_FLOOR: + return createCommonNumericFuncSingleArg("FLOOR"); + case FUNC_GCD: + return new DataFusionBaseExpr("GCD", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_ISNAN: + return createCommonNumericFuncSingleArg("ISNAN"); + case FUNC_ISZERO: + return createCommonNumericFuncSingleArg("ISZERO"); + case FUNC_LCM: + return createCommonNumericFuncTwoArgs("LCM"); + case FUNC_LN: + return createCommonNumericFuncSingleArg("LN"); + case FUNC_LOG: + return createCommonNumericFuncSingleArg("LOG"); + case FUNC_LOG_WITH_BASE: + return createCommonNumericFuncTwoArgs("LOG"); + case FUNC_LOG10: + return createCommonNumericFuncSingleArg("LOG10"); + case FUNC_LOG2: + return createCommonNumericFuncSingleArg("LOG2"); + case FUNC_NANVL: + return createCommonNumericFuncTwoArgs("NANVL"); + case FUNC_PI: + return new DataFusionBaseExpr("PI", 0, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), Arrays.asList()); + case FUNC_POW: + return createCommonNumericFuncSingleArg("POW"); + case FUNC_POWER: + return createCommonNumericFuncSingleArg("POWER"); + case FUNC_RADIANS: + return createCommonNumericFuncSingleArg("RADIANS"); + case FUNC_ROUND: + return createCommonNumericFuncSingleArg("ROUND"); + case FUNC_ROUND_WITH_DECIMAL: + return new DataFusionBaseExpr("ROUND", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case FUNC_SIGNUM: + return createCommonNumericFuncSingleArg("SIGNUM"); + case FUNC_SIN: + return createCommonNumericFuncSingleArg("SIN"); + case FUNC_SINH: + return createCommonNumericFuncSingleArg("SINH"); + case FUNC_SQRT: + return createCommonNumericFuncSingleArg("SQRT"); + case FUNC_TAN: + return createCommonNumericFuncSingleArg("TAN"); + case FUNC_TANH: + return createCommonNumericFuncSingleArg("TANH"); + case FUNC_TRUNC: + return createCommonNumericFuncSingleArg("TRUNC"); + case FUNC_TRUNC_WITH_DECIMAL: + return new DataFusionBaseExpr("TRUNC", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case FUNC_COALESCE: + return new DataFusionBaseExpr("COALESCE", -1, // overide by variadic + DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), Arrays.asList(), true); + case FUNC_NULLIF: + return new DataFusionBaseExpr("NULLIF", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_NVL: + return new DataFusionBaseExpr("NVL", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_NVL2: + return new DataFusionBaseExpr("NVL2", 3, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_IFNULL: + return new DataFusionBaseExpr("IFNULL", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + + case AGGR_MIN: + return createCommonNumericAggrFuncSingleArg("MIN"); + case AGGR_MAX: + return createCommonNumericAggrFuncSingleArg("MAX"); + case AGGR_AVG: + return createCommonNumericAggrFuncSingleArg("AVG"); + case AGGR_SUM: + return createCommonNumericAggrFuncSingleArg("SUM"); + case AGGR_COUNT: + return new DataFusionBaseExpr("COUNT", -1, DataFusionBaseExprCategory.AGGREGATE, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE)))), + true); + default: + dfAssert(false, "Unreachable. Unimplemented branch for type " + type); + } + + dfAssert(false, "Unreachable. Unimplemented branch for type " + type); + return null; + } + + // if input is Optional.empty(), return all possible `DataFusionBaseExpr`s + // else, return all `DataFusionBaseExpr` which might be evaluated to arg's type + public static List getExprsWithReturnType(Optional dataTypeOptional) { + List allExpressions = Arrays.stream(DataFusionBaseExprType.values()) + .map(DataFusionBaseExprFactory::createExpr).collect(Collectors.toList()); + + if (!dataTypeOptional.isPresent()) { + return allExpressions; // If Optional is empty, return all expressions + } + + DataFusionDataType filterType = dataTypeOptional.get(); + List exprsWithReturnType = allExpressions.stream() + .filter(expr -> expr.possibleReturnTypes.contains(filterType)).collect(Collectors.toList()); + + if (Randomly.getBoolean()) { + // Too many similar function, so test them less often + return exprsWithReturnType; + } + + return exprsWithReturnType.stream().filter(expr -> expr.exprType != DataFusionBaseExprCategory.FUNC) + .collect(Collectors.toList()); + } + + public static DataFusionBaseExpr getRandomAggregateExpr() { + List allAggrExpressions = Arrays.stream(DataFusionBaseExprType.values()) + .map(DataFusionBaseExprFactory::createExpr) + .filter(expr -> expr.exprType == DataFusionBaseExprCategory.AGGREGATE).collect(Collectors.toList()); + + return Randomly.fromList(allAggrExpressions); + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java new file mode 100644 index 000000000..3d7340615 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java @@ -0,0 +1,227 @@ +package sqlancer.datafusion.gen; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; +import static sqlancer.datafusion.gen.DataFusionBaseExprFactory.createExpr; +import static sqlancer.datafusion.gen.DataFusionBaseExprFactory.getExprsWithReturnType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; +import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; + +public final class DataFusionExpressionGenerator + extends TypedExpressionGenerator, DataFusionColumn, DataFusionDataType> { + + private final DataFusionGlobalState globalState; + + public DataFusionExpressionGenerator(DataFusionGlobalState globalState) { + this.globalState = globalState; + } + + @Override + protected DataFusionDataType getRandomType() { + DataFusionDataType dt; + do { + dt = Randomly.fromOptions(DataFusionDataType.values()); + } while (dt == DataFusionDataType.NULL); + + return dt; + } + + @Override + protected boolean canGenerateColumnOfType(DataFusionDataType type) { + return true; + } + + @Override + protected Node generateExpression(DataFusionDataType type, int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + DataFusionDataType expectedType = type; + if (Randomly.getBooleanWithRatherLowProbability()) { // ~10% + expectedType = DataFusionDataType.getRandomWithoutNull(); + } + return generateLeafNode(expectedType); + } + + // nested aggregate is not allowed, so occasionally apply it + Boolean includeAggr = Randomly.getBooleanWithSmallProbability(); + List possibleBaseExprs = getExprsWithReturnType(Optional.of(type)).stream() + // Conditinally apply filter if `includeAggr` set to false + .filter(expr -> includeAggr || expr.exprType != DataFusionBaseExpr.DataFusionBaseExprCategory.AGGREGATE) + .collect(Collectors.toList()); + + if (possibleBaseExprs.isEmpty()) { + dfAssert(type == DataFusionDataType.NULL, "should able to generate expression with type " + type); + return generateLeafNode(type); + } + + DataFusionBaseExpr randomExpr = Randomly.fromList(possibleBaseExprs); + switch (randomExpr.exprType) { + case UNARY_PREFIX: + DataFusionDataType argType = null; + dfAssert(randomExpr.argTypes.size() == 1 && randomExpr.nArgs == 1, + "Unary expression should only have 1 argument" + randomExpr.argTypes); + if (randomExpr.argTypes.get(0) instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); + argType = Randomly.fromList(possibleArgTypes.fixedType); + } else { + argType = type; + } + + return new NewUnaryPrefixOperatorNode(generateExpression(argType, depth + 1), + randomExpr); + case UNARY_POSTFIX: + dfAssert(randomExpr.argTypes.size() == 1 && randomExpr.nArgs == 1, + "Unary expression should only have 1 argument" + randomExpr.argTypes); + if (randomExpr.argTypes.get(0) instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); + argType = Randomly.fromList(possibleArgTypes.fixedType); + } else { + argType = type; + } + + return new NewUnaryPostfixOperatorNode(generateExpression(argType, depth + 1), + randomExpr); + case BINARY: + dfAssert(randomExpr.argTypes.size() == 2 && randomExpr.nArgs == 2, + "Binrary expression should only have 2 argument" + randomExpr.argTypes); + List argTypeList = new ArrayList<>(); // types of current expression's input arguments + for (ArgumentType argumentType : randomExpr.argTypes) { + if (argumentType instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); + dfAssert(!possibleArgTypes.fixedType.isEmpty(), "possible types can't be an empty list"); + DataFusionDataType determinedType = Randomly.fromList(possibleArgTypes.fixedType); + argTypeList.add(determinedType); + } else if (argumentType instanceof ArgumentType.SameAsFirstArgType) { + dfAssert(!argTypeList.isEmpty(), "First argument can't have argument type `SameAsFirstArgType`"); + DataFusionDataType firstArgType = argTypeList.get(0); + argTypeList.add(firstArgType); + } else { + // Same as expression return type + argTypeList.add(type); + } + } + + return new NewBinaryOperatorNode(generateExpression(argTypeList.get(0), depth + 1), + generateExpression(argTypeList.get(1), depth + 1), randomExpr); + case AGGREGATE: + // Fall through + case FUNC: + return generateFunctionExpression(type, depth, randomExpr); + default: + dfAssert(false, "unreachable"); + } + + dfAssert(false, "unreachable"); + return null; + } + + public Node generateFunctionExpression(DataFusionDataType type, int depth, + DataFusionBaseExpr exprType) { + if (exprType.isVariadic || Randomly.getBooleanWithSmallProbability()) { + // TODO(datafusion) maybe add possible types. e.g. some function have signature variadic(INT/DOUBLE), then + // only randomly pick from INT and DOUBLE + int nArgs = Randomly.smallNumber(); // 0, 2, 4, ... smaller one is more likely + return new NewFunctionNode(generateExpressions(nArgs), exprType); + } + + List funcArgTypeList = new ArrayList<>(); // types of current expression's input arguments + int i = 0; + for (ArgumentType argumentType : exprType.argTypes) { + if (argumentType instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) exprType.argTypes.get(i); + dfAssert(!possibleArgTypes.fixedType.isEmpty(), "possible types can't be an empty list"); + DataFusionDataType determinedType = Randomly.fromList(possibleArgTypes.fixedType); + funcArgTypeList.add(determinedType); + } else if (argumentType instanceof ArgumentType.SameAsFirstArgType) { + dfAssert(!funcArgTypeList.isEmpty(), "First argument can't have argument type `SameAsFirstArgType`"); + DataFusionDataType firstArgType = funcArgTypeList.get(0); + funcArgTypeList.add(firstArgType); + } else { + // Same as expression return type + funcArgTypeList.add(type); + } + i++; + } + + List> argExpressions = new ArrayList<>(); + + for (DataFusionDataType dataType : funcArgTypeList) { + argExpressions.add(generateExpression(dataType, depth + 1)); + } + + return new NewFunctionNode(argExpressions, exprType); + } + + List filterColumns(DataFusionDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + @Override + protected Node generateColumn(DataFusionDataType type) { + // HACK: if no col of such type exist, generate constant value instead + List colsOfType = filterColumns(type); + if (colsOfType.isEmpty()) { + return generateConstant(type); + } + + DataFusionColumn column = Randomly.fromList(colsOfType); + return new ColumnReferenceNode(column); + } + + @Override + public Node generateConstant(DataFusionDataType type) { + return type.getRandomConstant(globalState); + } + + @Override + public Node generatePredicate() { + return generateExpression(DataFusionDataType.BOOLEAN, 0); + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, createExpr(DataFusionBaseExprType.NOT)); + } + + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, createExpr(DataFusionBaseExprType.IS_NULL)); + } + + public static class DataFusionCastOperation extends NewUnaryPostfixOperatorNode { + + public DataFusionCastOperation(Node expr, DataFusionDataType type) { + super(expr, new Operator() { + + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + } + + } + +} diff --git a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java new file mode 100644 index 000000000..1ee00dd50 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java @@ -0,0 +1,54 @@ +package sqlancer.datafusion.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionToStringVisitor; + +public class DataFusionInsertGenerator extends AbstractInsertGenerator { + + private final DataFusionGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public DataFusionInsertGenerator(DataFusionGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DataFusionGlobalState globalState, DataFusionTable targetTable) { + return new DataFusionInsertGenerator(globalState).generate(targetTable); + } + + private SQLQueryAdapter generate(DataFusionTable targetTable) { + // `sb` is a global `StringBuilder` for current insert query + sb.append("INSERT INTO "); + + if (targetTable.getColumns().isEmpty()) { + throw new IgnoreMeException(); + } + List columns = targetTable.getRandomNonEmptyColumnSubset(); + + sb.append(targetTable.getName()); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); // will finally call `insertValue()` to generate random value + + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertValue(DataFusionColumn col) { + String val = DataFusionToStringVisitor + .asString(new DataFusionExpressionGenerator(globalState).generateConstant(col.getType())); + sb.append(val); + } + +} diff --git a/src/sqlancer/datafusion/gen/DataFusionTableGenerator.java b/src/sqlancer/datafusion/gen/DataFusionTableGenerator.java new file mode 100644 index 000000000..adececaa7 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionTableGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.datafusion.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; + +public class DataFusionTableGenerator { + + // Randomly generate a query like 'create table t1 (v1 bigint, v2 boolean)' + public SQLQueryAdapter getQuery(DataFusionGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + + int colCount = Randomly.smallNumber() + 1 + (Randomly.getBoolean() ? 1 : 0); + for (int i = 0; i < colCount; i++) { + sb.append("v").append(i).append(" ").append(DataFusionDataType.getRandomWithoutNull().toString()); + + if (i != colCount - 1) { + sb.append(", "); + } + } + + sb.append(");"); + + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml new file mode 100644 index 000000000..ebff15b35 --- /dev/null +++ b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "datafusion-server" +edition = "2021" +description = "Standalone DataFusion server" +license = "Apache-2.0" + +[dependencies] +ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } +arrow = { version = "52.1.0", features = ["prettyprint"] } +arrow-array = { version = "52.1.0", default-features = false, features = ["chrono-tz"] } +arrow-buffer = { version = "52.1.0", default-features = false } +arrow-flight = { version = "52.1.0", features = ["flight-sql-experimental"] } +arrow-ipc = { version = "52.1.0", default-features = false, features = ["lz4"] } +arrow-ord = { version = "52.1.0", default-features = false } +arrow-schema = { version = "52.1.0", default-features = false } +arrow-string = { version = "52.1.0", default-features = false } +async-trait = "0.1.73" +bytes = "1.4" +chrono = { version = "0.4.34", default-features = false } +dashmap = "5.5.0" +datafusion_stable = { package = "datafusion", git = "https://github.com/apache/datafusion.git", rev = "e693ed7", optional = true } +datafusion_dev = { package = "datafusion", git = "https://github.com/apache/datafusion.git", branch = "main", optional = true } +env_logger = "0.11" +futures = "0.3" +half = { version = "2.2.1", default-features = false } +hashbrown = { version = "0.14.5", features = ["raw"] } +log = "0.4" +num_cpus = "1.13.0" +object_store = { version = "0.10.1", default-features = false } +parking_lot = "0.12" +parquet = { version = "52.0.0", default-features = false, features = ["arrow", "async", "object_store"] } +rand = "0.8" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1" +tokio = { version = "1.36", features = ["macros", "rt", "sync"] } +tonic = "0.11" +uuid = "1.0" +prost = { version = "0.12", default-features = false } +prost-derive = { version = "0.12", default-features = false } +mimalloc = { version = "0.1", default-features = false } + +[[bin]] +name = "datafusion-server" +path = "src/main.rs" + +[features] +default = ["datafusion_stable_feature"] +datafusion_stable_feature = ["datafusion_stable"] +datafusion_dev_feature = ["datafusion_dev"] diff --git a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs new file mode 100644 index 000000000..c19728bb6 --- /dev/null +++ b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs @@ -0,0 +1,463 @@ +#[cfg(feature = "datafusion_dev")] +extern crate datafusion_dev as datafusion; +#[cfg(feature = "datafusion_stable")] +extern crate datafusion_stable as datafusion; + +use arrow::array::{ArrayRef, StringArray}; +use arrow::ipc::writer::IpcWriteOptions; +use arrow::record_batch::RecordBatch; +use arrow_flight::encode::FlightDataEncoderBuilder; +use arrow_flight::flight_descriptor::DescriptorType; +use arrow_flight::flight_service_server::{FlightService, FlightServiceServer}; +use arrow_flight::sql::server::{FlightSqlService, PeekableFlightDataStream}; +use arrow_flight::sql::{ + ActionClosePreparedStatementRequest, ActionCreatePreparedStatementRequest, + ActionCreatePreparedStatementResult, Any, CommandGetTables, CommandPreparedStatementQuery, + CommandPreparedStatementUpdate, ProstMessageExt, SqlInfo, +}; +use arrow_flight::{ + Action, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, HandshakeResponse, + IpcMessage, SchemaAsIpc, Ticket, +}; +use arrow_schema::{DataType, Field, Schema}; +use dashmap::DashMap; +use datafusion_stable::logical_expr::LogicalPlan; +use datafusion_stable::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; +use futures::{Stream, StreamExt, TryStreamExt}; +use log::info; +use mimalloc::MiMalloc; +use prost::Message; +use std::pin::Pin; +use std::sync::Arc; +use tokio::sync::Mutex; +use tonic::metadata::MetadataValue; +use tonic::transport::Server; +use tonic::{Request, Response, Status, Streaming}; +use uuid::Uuid; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +macro_rules! status { + ($desc:expr, $err:expr) => { + Status::internal(format!("{}: {} at {}:{}", $desc, $err, file!(), line!())) + }; +} + +/// Adapted from https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/flight/flight_sql_server.rs +/// Can be used as a remote DataFusion server and connected by `JDBC` from client +/// Supported SQL statements: +/// CREATE +/// INSERT +/// SELECT +/// +/// Only single client is supported +/// For now use `ctx` instead of `contexts` inside `FlightSqlServiceImpl` +/// +/// === Below origianl comment === +/// +/// This example shows how to wrap DataFusion with `FlightSqlService` to support connecting +/// to a standalone DataFusion-based server with a JDBC client, using the open source "JDBC Driver +/// for Arrow Flight SQL". +/// +/// To install the JDBC driver in DBeaver for example, see these instructions: +/// https://docs.dremio.com/software/client-applications/dbeaver/ +/// When configuring the driver, specify property "UseEncryption" = false +/// +/// JDBC connection string: "jdbc:arrow-flight-sql://127.0.0.1:50051/" +/// +/// Based heavily on Ballista's implementation: https://github.com/apache/datafusion-ballista/blob/main/ballista/scheduler/src/flight_sql.rs +/// and the example in arrow-rs: https://github.com/apache/arrow-rs/blob/master/arrow-flight/examples/flight_sql_server.rs +/// +#[tokio::main] +async fn main() -> Result<(), Box> { + env_logger::init(); + let addr = "0.0.0.0:50051".parse()?; + let session_ctx = SessionContext::new_with_config( + SessionConfig::new().with_information_schema(true), // enable catalog + ); + let service = FlightSqlServiceImpl { + contexts: Default::default(), + statements: Default::default(), + results: Default::default(), + ctx: Arc::new(Mutex::new(session_ctx)), + }; + info!("Listening on {addr:?}"); + let svc = FlightServiceServer::new(service); + + Server::builder().add_service(svc).serve(addr).await?; + + Ok(()) +} + +pub struct FlightSqlServiceImpl { + contexts: Arc>>, + statements: Arc>, + results: Arc>>, + ctx: Arc>, +} + +impl FlightSqlServiceImpl { + async fn create_ctx(&self) -> Result { + let uuid = Uuid::new_v4().hyphenated().to_string(); + let session_config = SessionConfig::from_env() + .map_err(|e| Status::internal(format!("Error building plan: {e}")))? + .with_information_schema(true); + let ctx = Arc::new(SessionContext::new_with_config(session_config)); + + self.contexts.insert(uuid.clone(), ctx); + Ok(uuid) + } + + fn get_ctx(&self, req: &Request) -> Result, Status> { + // get the token from the authorization header on Request + let auth = req + .metadata() + .get("authorization") + .ok_or_else(|| Status::internal("No authorization header!"))?; + let str = auth + .to_str() + .map_err(|e| Status::internal(format!("Error parsing header: {e}")))?; + let authorization = str.to_string(); + let bearer = "Bearer "; + if !authorization.starts_with(bearer) { + Err(Status::internal("Invalid auth header!"))?; + } + let auth = authorization[bearer.len()..].to_string(); + + if let Some(context) = self.contexts.get(&auth) { + Ok(context.clone()) + } else { + Err(Status::internal(format!( + "Context handle not found: {auth}" + )))? + } + } + + fn get_plan(&self, handle: &str) -> Result { + if let Some(plan) = self.statements.get(handle) { + Ok(plan.clone()) + } else { + Err(Status::internal(format!("Plan handle not found: {handle}")))? + } + } + + fn get_result(&self, handle: &str) -> Result, Status> { + if let Some(result) = self.results.get(handle) { + Ok(result.clone()) + } else { + Err(Status::internal(format!( + "Request handle not found: {handle}" + )))? + } + } + + async fn tables(&self, ctx: Arc) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("catalog_name", DataType::Utf8, true), + Field::new("db_schema_name", DataType::Utf8, true), + Field::new("table_name", DataType::Utf8, false), + Field::new("table_type", DataType::Utf8, false), + ])); + + let mut catalogs = vec![]; + let mut schemas = vec![]; + let mut names = vec![]; + let mut types = vec![]; + for catalog in ctx.catalog_names() { + let catalog_provider = ctx.catalog(&catalog).unwrap(); + for schema in catalog_provider.schema_names() { + let schema_provider = catalog_provider.schema(&schema).unwrap(); + for table in schema_provider.table_names() { + let table_provider = schema_provider.table(&table).await.unwrap().unwrap(); + catalogs.push(catalog.clone()); + schemas.push(schema.clone()); + names.push(table.clone()); + types.push(table_provider.table_type().to_string()) + } + } + } + + RecordBatch::try_new( + schema, + [catalogs, schemas, names, types] + .into_iter() + .map(|i| Arc::new(StringArray::from(i)) as ArrayRef) + .collect::>(), + ) + .unwrap() + } + + fn remove_plan(&self, handle: &str) -> Result<(), Status> { + self.statements.remove(&handle.to_string()); + Ok(()) + } + + fn remove_result(&self, handle: &str) -> Result<(), Status> { + self.results.remove(&handle.to_string()); + Ok(()) + } +} + +#[tonic::async_trait] +impl FlightSqlService for FlightSqlServiceImpl { + type FlightService = FlightSqlServiceImpl; + + // This function will be triggered if client JDBC property's `user` and `password` field set + async fn do_handshake( + &self, + _request: Request>, + ) -> Result< + Response> + Send>>>, + Status, + > { + info!("do_handshake"); + if let Some(msg) = _request.metadata().get("create") { + // A new round start at SQLancer, clear the ctx + info!("Resetting ctx {:?}", msg); + let new_ctx = + SessionContext::new_with_config(SessionConfig::new().with_information_schema(true)); + + let mut ctx_guard = self.ctx.lock().await; // Use `lock()` for async Mutex + *ctx_guard = new_ctx; + } + // no authentication actually takes place here + // see Ballista implementation for example of basic auth + // in this case, we simply accept the connection and create a new SessionContext + // the SessionContext will be re-used within this same connection/session + let token = self.create_ctx().await?; + + let result = HandshakeResponse { + protocol_version: 0, + payload: token.as_bytes().to_vec().into(), + }; + let result = Ok(result); + let output = futures::stream::iter(vec![result]); + let str = format!("Bearer {token}"); + let mut resp: Response> + Send>>> = + Response::new(Box::pin(output)); + let md = MetadataValue::try_from(str) + .map_err(|_| Status::invalid_argument("authorization not parsable"))?; + resp.metadata_mut().insert("authorization", md); + Ok(resp) + } + + async fn do_get_fallback( + &self, + _request: Request, + message: Any, + ) -> Result::DoGetStream>, Status> { + if !message.is::() { + Err(Status::unimplemented(format!( + "do_get: The defined request is invalid: {}", + message.type_url + )))? + } + + let fr: FetchResults = message + .unpack() + .map_err(|e| Status::internal(format!("{e:?}")))? + .ok_or_else(|| Status::internal("Expected FetchResults but got None!"))?; + + let handle = fr.handle; + + info!("getting results for {handle}"); + let result = self.get_result(&handle)?; + // if we get an empty result, create an empty schema + let (schema, batches) = match result.first() { + None => (Arc::new(Schema::empty()), vec![]), + Some(batch) => (batch.schema(), result.clone()), + }; + + let batch_stream = futures::stream::iter(batches).map(Ok); + + let stream = FlightDataEncoderBuilder::new() + .with_schema(schema) + .build(batch_stream) + .map_err(Status::from); + + Ok(Response::new(Box::pin(stream))) + } + + async fn get_flight_info_prepared_statement( + &self, + cmd: CommandPreparedStatementQuery, + _request: Request, + ) -> Result, Status> { + info!("get_flight_info_prepared_statement {:?}", cmd); + let handle = std::str::from_utf8(&cmd.prepared_statement_handle) + .map_err(|e| status!("Unable to parse uuid", e))?; + + //let ctx = self.get_ctx(&request)?; + let plan = self.get_plan(handle)?; + + let ctx_guard = self.ctx.lock().await; + let state = (*ctx_guard).state(); + let df = DataFrame::new(state, plan); + let result = df + .collect() + .await + .map_err(|e| status!("Error executing query", e))?; + + // if we get an empty result, create an empty schema + let schema = match result.first() { + None => Schema::empty(), + Some(batch) => (*batch.schema()).clone(), + }; + + self.results.insert(handle.to_string(), result); + + // if we had multiple endpoints to connect to, we could use this Location + // but in the case of standalone DataFusion, we don't + // let loc = Location { + // uri: "grpc+tcp://127.0.0.1:50051".to_string(), + // }; + let fetch = FetchResults { + handle: handle.to_string(), + }; + let buf = fetch.as_any().encode_to_vec().into(); + let ticket = Ticket { ticket: buf }; + + let info = FlightInfo::new() + // Encode the Arrow schema + .try_with_schema(&schema) + .expect("encoding failed") + .with_endpoint(FlightEndpoint::new().with_ticket(ticket)) + .with_descriptor(FlightDescriptor { + r#type: DescriptorType::Cmd.into(), + cmd: Default::default(), + path: vec![], + }); + let resp = Response::new(info); + Ok(resp) + } + + async fn get_flight_info_tables( + &self, + _query: CommandGetTables, + request: Request, + ) -> Result, Status> { + info!("get_flight_info_tables"); + let ctx = self.get_ctx(&request)?; + let data = self.tables(ctx).await; + let schema = data.schema(); + + let uuid = Uuid::new_v4().hyphenated().to_string(); + self.results.insert(uuid.clone(), vec![data]); + + let fetch = FetchResults { handle: uuid }; + let buf = fetch.as_any().encode_to_vec().into(); + let ticket = Ticket { ticket: buf }; + + let info = FlightInfo::new() + // Encode the Arrow schema + .try_with_schema(&schema) + .expect("encoding failed") + .with_endpoint(FlightEndpoint::new().with_ticket(ticket)) + .with_descriptor(FlightDescriptor { + r#type: DescriptorType::Cmd.into(), + cmd: Default::default(), + path: vec![], + }); + let resp = Response::new(info); + Ok(resp) + } + + async fn do_put_prepared_statement_update( + &self, + handle: CommandPreparedStatementUpdate, + _request: Request, + ) -> Result { + info!("do_put_prepared_statement_update"); + // statements like "CREATE TABLE.." or "SET datafusion.nnn.." call this function + // and we are required to return some row count here + let handle = std::str::from_utf8(&handle.prepared_statement_handle) + .map_err(|e| status!("Unable to parse uuid", e))?; + + //let ctx = self.get_ctx(&request)?; + let plan = self.get_plan(handle)?; + //println!("do_put_prepared_statement_update plan is {:?}", plan); + + let ctx_guard = self.ctx.lock().await; + let state = (*ctx_guard).state(); + let df = DataFrame::new(state, plan); + df.collect() + .await + .map_err(|e| status!("Error executing query", e))?; + + Ok(1) + } + + async fn do_action_create_prepared_statement( + &self, + query: ActionCreatePreparedStatementRequest, + _request: Request, + ) -> Result { + let user_query = query.query.as_str(); + info!("do_action_create_prepared_statement: {user_query}"); + + //let ctx = self.get_ctx(&request)?; + + let ctx_guard = self.ctx.lock().await; + let plan = (*ctx_guard) + .sql(user_query) + .await + .and_then(|df| df.into_optimized_plan()) + .map_err(|e| Status::internal(format!("Error building plan: {e}")))?; + + // store a copy of the plan, it will be used for execution + let plan_uuid = Uuid::new_v4().hyphenated().to_string(); + self.statements.insert(plan_uuid.clone(), plan.clone()); + + let plan_schema = plan.schema(); + + let arrow_schema = (&**plan_schema).into(); + let message = SchemaAsIpc::new(&arrow_schema, &IpcWriteOptions::default()) + .try_into() + .map_err(|e| status!("Unable to serialize schema", e))?; + let IpcMessage(schema_bytes) = message; + + let res = ActionCreatePreparedStatementResult { + prepared_statement_handle: plan_uuid.into(), + dataset_schema: schema_bytes, + parameter_schema: Default::default(), + }; + Ok(res) + } + + async fn do_action_close_prepared_statement( + &self, + handle: ActionClosePreparedStatementRequest, + _request: Request, + ) -> Result<(), Status> { + info!("do_action_close_prepared_statement"); + let handle = std::str::from_utf8(&handle.prepared_statement_handle); + if let Ok(handle) = handle { + info!("do_action_close_prepared_statement: removing plan and results for {handle}"); + let _ = self.remove_plan(handle); + let _ = self.remove_result(handle); + } + Ok(()) + } + + async fn register_sql_info(&self, _id: i32, _result: &SqlInfo) {} +} + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FetchResults { + #[prost(string, tag = "1")] + pub handle: ::prost::alloc::string::String, +} + +impl ProstMessageExt for FetchResults { + fn type_url() -> &'static str { + "type.googleapis.com/datafusion.example.com.sql.FetchResults" + } + + fn as_any(&self) -> Any { + Any { + type_url: FetchResults::type_url().to_string(), + value: ::prost::Message::encode_to_vec(self).into(), + } + } +} diff --git a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java new file mode 100644 index 000000000..55f18fecd --- /dev/null +++ b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java @@ -0,0 +1,95 @@ +package sqlancer.datafusion.test; + +import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.ERROR; +import static sqlancer.datafusion.ast.DataFusionSelect.getRandomSelect; + +import java.sql.SQLException; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.datafusion.DataFusionErrors; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionToStringVisitor; +import sqlancer.datafusion.DataFusionUtil; +import sqlancer.datafusion.ast.DataFusionSelect; + +public class DataFusionNoRECOracle extends NoRECBase + implements TestOracle { + + private final DataFusionGlobalState state; + + public DataFusionNoRECOracle(DataFusionGlobalState globalState) { + super(globalState); + this.state = globalState; + DataFusionErrors.registerExpectedExecutionErrors(errors); + } + + /* + * Non-Optimizing Reference Engine Construction q1: SELECT [expr1] FROM [expr2] WHERE [expr3] q2: SELECT [expr3] + * FROM [expr2] + * + * Oracle Check: q1's result size equals to `true` count in q2's result set + */ + @Override + public void check() throws SQLException { + /* + * Setup Q1 and Q2 + */ + // generate a random: + // SELECT [expr1] FROM [expr2] WHERE [expr3] + DataFusionSelect randomSelect = getRandomSelect(state); + // Q1: SELECT count(*) FROM [expr2] WHERE [expr3] + DataFusionSelect q1 = new DataFusionSelect(); + q1.setFetchColumnsString("COUNT(*)"); + q1.setFromList(randomSelect.getFromList()); + q1.setWhereClause(randomSelect.getWhereClause()); + // Q2: SELECT count(case when [expr3] then 1 else null end) FROM [expr2] + DataFusionSelect q2 = new DataFusionSelect(); + String selectExpr = String.format("COUNT(CASE WHEN %S THEN 1 ELSE NULL END)", + DataFusionToStringVisitor.asString(randomSelect.getWhereClause())); + q2.setFetchColumnsString(selectExpr); + q2.setFromList(randomSelect.getFromList()); + q2.setWhereClause(null); + + /* + * Execute Q1 and Q2 + */ + String q1String = DataFusionToStringVisitor.asString(q1); + String q2String = DataFusionToStringVisitor.asString(q2); + List q1ResultSet = null; + List q2ResultSet = null; + try { + q1ResultSet = ComparatorHelper.getResultSetFirstColumnAsString(q1String, errors, state); + q2ResultSet = ComparatorHelper.getResultSetFirstColumnAsString(q2String, errors, state); + } catch (AssertionError e) { + // Append detailed error message + String replay = DataFusionUtil.getReplay(state.getDatabaseName()); + String newMessage = e.getMessage() + "\n" + e.getCause() + "\n" + replay + "\n"; + state.dfLogger.appendToLog(ERROR, newMessage); + + throw new AssertionError(newMessage); + } + + /* + * NoREC check + */ + int count1 = q1ResultSet != null ? Integer.parseInt(q1ResultSet.get(0)) : -1; + int count2 = q2ResultSet != null ? Integer.parseInt(q2ResultSet.get(0)) : -1; + if (count1 != count2) { + StringBuilder errorMessage = new StringBuilder().append("NoREC oracle violated:\n") + .append(" Q1(result size ").append(count1).append("):").append(q1String).append(";\n") + .append(" Q2(result size ").append(count2).append("):").append(q2String).append(";\n") + .append("=======================================\n").append("Reproducer: \n"); + + String replay = DataFusionUtil.getReplay(state.getDatabaseName()); + + String errorLog = errorMessage.toString() + replay + "\n"; + String indentedErrorLog = errorLog.replaceAll("(?m)^", " "); + state.dfLogger.appendToLog(ERROR, errorLog); + + throw new AssertionError("\n\n" + indentedErrorLog); + } + } +} diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java new file mode 100644 index 000000000..b304a4131 --- /dev/null +++ b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java @@ -0,0 +1,38 @@ +package sqlancer.datafusion.test; + +import java.sql.SQLException; + +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionSelect; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +public class DataFusionQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, DataFusionGlobalState> + implements TestOracle { + DataFusionGlobalState state; + DataFusionExpressionGenerator gen; + DataFusionSelect select; + + public DataFusionQueryPartitioningBase(DataFusionGlobalState state) { + super(state); + this.state = state; + } + + @Override + public void check() throws SQLException { + select = DataFusionSelect.getRandomSelect(state); + gen = select.exprGen; + initializeTernaryPredicateVariants(); + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + +} diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java new file mode 100644 index 000000000..3235c3d69 --- /dev/null +++ b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java @@ -0,0 +1,67 @@ +package sqlancer.datafusion.test; + +import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.ERROR; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.datafusion.DataFusionErrors; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionToStringVisitor; +import sqlancer.datafusion.DataFusionUtil; +import sqlancer.datafusion.ast.DataFusionSelect; + +public class DataFusionQueryPartitioningWhereTester extends DataFusionQueryPartitioningBase { + public DataFusionQueryPartitioningWhereTester(DataFusionGlobalState state) { + super(state); + DataFusionErrors.registerExpectedExecutionErrors(errors); + } + + /* + * Query Partitioning - Where q: SELECT [expr1] FROM [expr2] qp1: SELECT [expr1] FROM [expr2] WHERE [expr3] qp2: + * SELECT [expr1] FROM [expr2] WHERE NOT [expr3] qp3: SELECT [expr1] FROM [expr2] WHERE [expr3] IS NULL + * + * Oracle check: q's result equals to union(qp1, qp2, qp3) + */ + @Override + public void check() throws SQLException { + // generate a random 'SELECT [expr1] FROM [expr2] WHERE [expr3] + super.check(); + DataFusionSelect randomSelect = select; + randomSelect.setWhereClause(null); + + // Construct q + String qString = DataFusionToStringVisitor.asString(randomSelect); + // Construct qp1, qp2, qp3 + randomSelect.setWhereClause(predicate); + String qp1String = DataFusionToStringVisitor.asString(randomSelect); + randomSelect.setWhereClause(negatedPredicate); + String qp2String = DataFusionToStringVisitor.asString(randomSelect); + randomSelect.setWhereClause(isNullPredicate); + String qp3String = DataFusionToStringVisitor.asString(randomSelect); + + try { + /* + * Run all queires + */ + List qResultSet = ComparatorHelper.getResultSetFirstColumnAsString(qString, errors, state); + List combinedString = new ArrayList<>(); + List qpResultSet = ComparatorHelper.getCombinedResultSet(qp1String, qp2String, qp3String, + combinedString, true, state, errors); + /* + * Query Partitioning-Where check + */ + ComparatorHelper.assumeResultSetsAreEqual(qResultSet, qpResultSet, qString, combinedString, state, + ComparatorHelper::canonicalizeResultValue); + } catch (AssertionError e) { + // Append more error message + String replay = DataFusionUtil.getReplay(state.getDatabaseName()); + String newMessage = e.getMessage() + "\n" + e.getCause() + "\n" + replay + "\n"; + state.dfLogger.appendToLog(ERROR, newMessage); + + throw new AssertionError(newMessage); + } + } +} diff --git a/test/sqlancer/dbms/TestDataFusion.java b/test/sqlancer/dbms/TestDataFusion.java new file mode 100644 index 000000000..ae568e40e --- /dev/null +++ b/test/sqlancer/dbms/TestDataFusion.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestDataFusion { + @Test + public void testDataFusion() { + String datafusionAvailable = System.getenv("DATAFUSION_AVAILABLE"); + boolean datafusionIsAvailable = datafusionAvailable != null && datafusionAvailable.equalsIgnoreCase("true"); + assumeTrue(datafusionIsAvailable); + + assertEquals(0, Main.executeMain("--random-seed", "0", "--num-threads", "1", // TODO(datafusion) update when + // multithread is supported + "--timeout-seconds", TestConfig.SECONDS, "--num-queries", TestConfig.NUM_QUERIES, "datafusion")); + } +} From 852c1e4fecc853dc6246b45a3cc3045da9d8e920 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:18:44 +0200 Subject: [PATCH 0880/1284] [ClickHouse] Added pattern for changed error message (#955) * [ClickHouse] Added pattern for changed error message https://fiddle.clickhouse.com/a3a95024-4da7-4275-baff-86f116014744 ``` Received exception from server (version 24.2.3): Code: 403. DB::Exception: Received from localhost:9000. DB::Exception: Cannot get JOIN keys from JOIN ON section: '476505718 = `_--right_2.c0`', found keys: [Left keys: [] Right keys [] Condition columns: '', 'equals(476505718, _--right_2.c0)']. (INVALID_JOIN_ON_EXPRESSION) (query: SELECT SUM(check <> 0) FROM ((SELECT right_0.c0 AS `check` FROM t0 AS left FULL OUTER JOIN t0 AS right_0 ON ((left.c0)=(right_0.c0)) LEFT OUTER JOIN t0 AS right_1 ON ((left.c0)=(right_1.c0)) LEFT ANTI JOIN t0 AS right_2 ON ((476505718)=(right_2.c0)))) as res;) ``` * formatter --- src/sqlancer/clickhouse/ClickHouseErrors.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index c4ccd7e44..0354a6172 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -45,8 +45,8 @@ public static List getExpectedExpressionErrors() { "Positional argument numeric constant expression is not representable as", "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)", - "Unexpected inf or nan to integer conversion", "Unsigned type must not contain", - "Unexpected inf or nan to integer conversion", + "Cannot get JOIN keys from JOIN ON section", "Unexpected inf or nan to integer conversion", + "Unsigned type must not contain", "Unexpected inf or nan to integer conversion", // The way we generate JOINs we can have ambiguous left table column without // alias From ec00b2d19115dc9ee080835b65b53d6aa7a93848 Mon Sep 17 00:00:00 2001 From: Robins Date: Tue, 16 Jul 2024 11:01:01 +0930 Subject: [PATCH 0881/1284] Avoid storage parameters during CREATE TABLE for partitioned tables (#956) CREATE TABLE WITH() allows storage parameters, but partitioned tables emit an error (given below) if attempted. This is because partitioned non-leaf tables are virutal tables [1] and don't accept storage parameters. Sample Error: "ERROR: cannot specify storage parameters for a partitioned table" This patch skips storage parameter generation for partitioned tables. Ref: 1. https://www.postgresql.org/docs/current/ddl-partitioning.html --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 7d65b8ebd..9ae649ae5 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -113,7 +113,9 @@ private void createStandard() throws AssertionError { generateInherits(); generatePartitionBy(); generateUsing(); - PostgresCommon.generateWith(sb, globalState, errors); + if (!isPartitionedTable) { + PostgresCommon.generateWith(sb, globalState, errors); + } if (Randomly.getBoolean() && isTemporaryTable) { sb.append(" ON COMMIT "); sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS", "DROP")); From 6d2b92a5b72a5904d5f7827df6ff9488668f55e6 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Tue, 16 Jul 2024 17:44:17 +0800 Subject: [PATCH 0882/1284] Review feedback --- .github/workflows/main.yml | 2 +- .../datafusion/DataFusionProvider.java | 7 +++--- src/sqlancer/datafusion/DataFusionUtil.java | 25 ++++++++----------- .../server/datafusion_server/Cargo.toml | 13 ++++------ .../server/datafusion_server/src/main.rs | 9 ++----- 5 files changed, 22 insertions(+), 34 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f2a12c62..bd65c9890 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -214,7 +214,7 @@ jobs: - name: Run DataFusion Server run: | cd src/sqlancer/datafusion/server/datafusion_server - cargo run --features "datafusion_stable" & sleep 300 + cargo run & sleep 300 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/src/sqlancer/datafusion/DataFusionProvider.java b/src/sqlancer/datafusion/DataFusionProvider.java index 161c324dd..37328e4ba 100644 --- a/src/sqlancer/datafusion/DataFusionProvider.java +++ b/src/sqlancer/datafusion/DataFusionProvider.java @@ -1,7 +1,7 @@ package sqlancer.datafusion; -import static java.lang.System.exit; import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.DML; +import static sqlancer.datafusion.DataFusionUtil.dfAssert; import static sqlancer.datafusion.DataFusionUtil.displayTables; import java.sql.Connection; @@ -52,13 +52,12 @@ public void generateDatabase(DataFusionGlobalState globalState) throws Exception List allTables = globalState.getSchema().getDatabaseTables(); List allTablesName = allTables.stream().map(t -> t.getName()).collect(Collectors.toList()); if (allTablesName.isEmpty()) { - System.out.println("Generate database failed"); - exit(1); + dfAssert(false, "Generate Database failed."); } // Randomly insert some data into existing tables for (DataFusionTable table : allTables) { - int nInsertQuery = globalState.getRandomly().getInteger(0, 8); // [0, 10) + int nInsertQuery = globalState.getRandomly().getInteger(0, globalState.getOptions().getMaxNumberInserts()); for (int i = 0; i < nInsertQuery; i++) { SQLQueryAdapter insertQuery = null; diff --git a/src/sqlancer/datafusion/DataFusionUtil.java b/src/sqlancer/datafusion/DataFusionUtil.java index e37ad8d64..8761bec9b 100644 --- a/src/sqlancer/datafusion/DataFusionUtil.java +++ b/src/sqlancer/datafusion/DataFusionUtil.java @@ -1,7 +1,5 @@ package sqlancer.datafusion; -import static java.lang.System.exit; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -66,11 +64,15 @@ public static String displayTables(DataFusionGlobalState state, List fro return resultStringBuilder.toString(); } + // During development, you might want to manually let this function call exit(1) to fail fast public static void dfAssert(boolean condition, String message) { if (!condition) { - String methodName = Thread.currentThread().getStackTrace()[2].getMethodName(); - System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message); - exit(1); + // // Development mode assertion failure + // String methodName = Thread.currentThread().getStackTrace()[2]// .getMethodName(); + // System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message); + // exit(1); + + throw new AssertionError(message); } } @@ -149,9 +151,7 @@ public void appendToLog(DataFusionLogType logType, String logContent) { try { logFileWriter = new FileWriter(errorLogFile, true); } catch (IOException e) { - System.out.println("Failed to create FileWriter for errorLogFIle"); - e.printStackTrace(); - exit(1); + dfAssert(false, "Failed to create FileWriter for errorLogFIle"); } DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); String formattedDateTime = LocalDateTime.now().format(formatter); @@ -175,14 +175,11 @@ public void appendToLog(DataFusionLogType logType, String logContent) { logFileWriter.write(logContent); logFileWriter.flush(); } catch (IOException e) { - System.out.println("Failed to write to " + logType + " log: " + e.getMessage()); - e.printStackTrace(); - exit(1); + String err = "Failed to write to " + logType + " log: " + e.getMessage(); + dfAssert(false, err); } } else { - System.out.println("appending to log failed"); - Thread.currentThread().getStackTrace(); - exit(1); + dfAssert(false, "appending to log failed"); } } diff --git a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml index ebff15b35..cd8b85e1d 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml +++ b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml @@ -18,8 +18,10 @@ async-trait = "0.1.73" bytes = "1.4" chrono = { version = "0.4.34", default-features = false } dashmap = "5.5.0" -datafusion_stable = { package = "datafusion", git = "https://github.com/apache/datafusion.git", rev = "e693ed7", optional = true } -datafusion_dev = { package = "datafusion", git = "https://github.com/apache/datafusion.git", branch = "main", optional = true } +# This version is for SQLancer CI run +datafusion = { version = "40.0.0" } +# Use following line if you want to test against the latest main branch of DataFusion +# datafusion = { git = "https://github.com/apache/datafusion.git", branch = "main" } env_logger = "0.11" futures = "0.3" half = { version = "2.2.1", default-features = false } @@ -41,9 +43,4 @@ mimalloc = { version = "0.1", default-features = false } [[bin]] name = "datafusion-server" -path = "src/main.rs" - -[features] -default = ["datafusion_stable_feature"] -datafusion_stable_feature = ["datafusion_stable"] -datafusion_dev_feature = ["datafusion_dev"] +path = "src/main.rs" \ No newline at end of file diff --git a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs index c19728bb6..13ec73e96 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs +++ b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs @@ -1,8 +1,3 @@ -#[cfg(feature = "datafusion_dev")] -extern crate datafusion_dev as datafusion; -#[cfg(feature = "datafusion_stable")] -extern crate datafusion_stable as datafusion; - use arrow::array::{ArrayRef, StringArray}; use arrow::ipc::writer::IpcWriteOptions; use arrow::record_batch::RecordBatch; @@ -21,8 +16,8 @@ use arrow_flight::{ }; use arrow_schema::{DataType, Field, Schema}; use dashmap::DashMap; -use datafusion_stable::logical_expr::LogicalPlan; -use datafusion_stable::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; +use datafusion::logical_expr::LogicalPlan; +use datafusion::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; use futures::{Stream, StreamExt, TryStreamExt}; use log::info; use mimalloc::MiMalloc; From 0316c1c9c2fa2ad2a82977cfe2587faaba78cec8 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 17 Jul 2024 12:36:27 +0930 Subject: [PATCH 0883/1284] Ensure INHERITS() only uses tables (not views) In Postgres, CREATE TABLE INHERITS () throws an error when VIEWs are provided, sample given below. ERROR: inherited relation "pg_buffercache" is not a table or foreign table Ref: 1. https://www.postgresql.org/docs/current/ddl-inherit.html --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 9ae649ae5..29ccfcf2c 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -207,7 +207,7 @@ private void generateUsing() { } private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { + if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); From cc189b4d32961c09916a8870a0c0df3c5ce57c59 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 17 Jul 2024 12:40:42 +0930 Subject: [PATCH 0884/1284] Revert "Ensure INHERITS() only uses tables (not views)" This reverts commit 0316c1c9c2fa2ad2a82977cfe2587faaba78cec8. --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 29ccfcf2c..9ae649ae5 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -207,7 +207,7 @@ private void generateUsing() { } private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { + if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); From 96adb5446e05e52a8aff88b4aee7a5e42ff2c6a2 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 17 Jul 2024 12:42:23 +0930 Subject: [PATCH 0885/1284] Ensure INHERITS() only uses tables (not views) In Postgres, CREATE TABLE INHERITS () throws an error when VIEWs are provided, sample given below. ERROR: inherited relation "pg_buffercache" is not a table or foreign table Ref: 1. https://www.postgresql.org/docs/current/ddl-inherit.html --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 9ae649ae5..29ccfcf2c 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -207,7 +207,7 @@ private void generateUsing() { } private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { + if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); From 51e7ae87daba53699d1e9318f827b612d871f510 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:22:03 +0200 Subject: [PATCH 0886/1284] Another fix --- src/sqlancer/clickhouse/ClickHouseErrors.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 0354a6172..e6a4613d9 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -46,7 +46,8 @@ public static List getExpectedExpressionErrors() { "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)", "Cannot get JOIN keys from JOIN ON section", "Unexpected inf or nan to integer conversion", - "Unsigned type must not contain", "Unexpected inf or nan to integer conversion", + "Cannot determine join keys in", "Unsigned type must not contain", + "Unexpected inf or nan to integer conversion", // The way we generate JOINs we can have ambiguous left table column without // alias From 34e46423efc7bbc24b9389d9b9ea99ffd2ae99fd Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Fri, 19 Jul 2024 17:22:17 +0200 Subject: [PATCH 0887/1284] Materialize: Use different database name (#959) Since materialize already exists by default and is owned by mz_system: org.postgresql.util.PSQLException: ERROR: must be owner of DATABASE materialize at sqlancer.qpg.materialize.TestMaterializeQueryPlan.testMaterializeQueryPlan(TestMaterializeQueryPlan.java:33) --- test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java index 08e402037..f282af42e 100644 --- a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java +++ b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java @@ -21,7 +21,7 @@ void testMaterializeQueryPlan() throws Exception { boolean materializeIsAvailable = materialize != null && materialize.equalsIgnoreCase("true"); assumeTrue(materializeIsAvailable); - String databaseName = "materialize"; + String databaseName = "queryplan"; MaterializeProvider provider = new MaterializeProvider(); MaterializeGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); MaterializeOptions materializeOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); @@ -42,7 +42,7 @@ void testMaterializeQueryPlan() throws Exception { String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); assertEquals( - "Return;Union;Get l0;Project (#2, #3, #0);Union;Negate;Project (#2);Get materialize.public.t2;Get materialize.public.t2;With;Get materialize.public.t1;Get materialize.public.t2;;Source materialize.public.t1;", + "Return // { arity: 3 };Union // { arity: 3 };Get l0 // { arity: 3 };Project (#2{c}, #3, #0) // { arity: 3 };Union // { arity: 1 };Negate // { arity: 1 };Project (#2) // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };With;ReadStorage queryplan.public.t1 // { arity: 2 };ReadStorage queryplan.public.t2 // { arity: 1 };;Source queryplan.public.t1;Source queryplan.public.t2;;Target cluster: quickstart;", queryPlan); } From 9c74a9527735cc62531f5e0ea0247222ee2f2027 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Tue, 20 Aug 2024 08:43:37 +0800 Subject: [PATCH 0888/1284] [Postgres] Use generic TLP where oracle (#943) * Implement TLPWhere generator for Postgres * Use generic TLP where oracle for Postgres --- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 31 ++++++++++- .../gen/PostgresExpressionGenerator.java | 26 +++++++-- .../gen/PostgresRandomQueryGenerator.java | 2 +- .../PostgresPivotedQuerySynthesisOracle.java | 2 +- .../tlp/PostgresTLPAggregateOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPBase.java | 2 +- .../oracle/tlp/PostgresTLPHavingOracle.java | 2 +- .../oracle/tlp/PostgresTLPWhereOracle.java | 53 +++++++++---------- 8 files changed, 80 insertions(+), 40 deletions(-) diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java index 1ccecd0c5..dbfc26e4b 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java @@ -1,14 +1,20 @@ package sqlancer.citus.oracle.tlp; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; +import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.oracle.tlp.PostgresTLPBase; -public class CitusTLPWhereOracle extends PostgresTLPWhereOracle { +public class CitusTLPWhereOracle extends PostgresTLPBase { private final CitusTLPBase citusTLPBase; @@ -32,4 +38,25 @@ public void check() throws SQLException { whereCheck(); state.setDefaultAllowedFunctionTypes(); } + + void whereCheck() throws SQLException { + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + String originalQueryString = PostgresVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setOrderByClauses(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = PostgresVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PostgresVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PostgresVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 5b5851671..8c0c2352e 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -12,6 +12,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; @@ -64,7 +65,8 @@ import sqlancer.postgres.ast.PostgresSimilarTo; public class PostgresExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private final int maxDepth; @@ -108,7 +110,8 @@ public PostgresExpression generateExpression(int depth) { return generateExpression(depth, PostgresDataType.getRandomType()); } - public List generateOrderBy() { + @Override + public List generateOrderBys() { List orderBys = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber(); i++) { orderBys.add(new PostgresOrderByTerm(PostgresColumnValue.create(Randomly.fromList(columns), null), @@ -615,7 +618,7 @@ public static PostgresSubquery createSubquery(PostgresGlobalState globalState, S select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); @@ -691,6 +694,21 @@ public List getTableRefs() { .collect(Collectors.toList()); } + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new PostgresColumnValue(PostgresColumn.createDummy("*"), null)); + } + allowAggregateFunctions = true; + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(columns); + for (PostgresColumn c : targetColumns) { + fetchColumns.add(new PostgresColumnValue(c, null)); + } + allowAggregateFunctions = false; + return fetchColumns; + } + @Override public String generateOptimizedQueryString(PostgresSelect select, PostgresExpression whereCondition, boolean shouldUseAggregate) { @@ -703,7 +721,7 @@ public String generateOptimizedQueryString(PostgresSelect select, PostgresExpres } select.setWhereClause(whereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(generateOrderBy()); + select.setOrderByClauses(generateOrderBys()); } select.setSelectType(SelectType.ALL); return select.asString(); diff --git a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java index 7fc5049cd..64d6278de 100644 --- a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java @@ -45,7 +45,7 @@ public static PostgresSelect createRandomQuery(int nrColumns, PostgresGlobalStat } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index c64f764e3..9db5365a3 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -63,7 +63,7 @@ public SQLQueryAdapter getRectifiedQuery() throws SQLException { selectStatement.setOffsetClause(offsetClause); } List orderBy = new PostgresExpressionGenerator(globalState).setColumns(columns) - .generateOrderBy(); + .generateOrderBys(); selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(PostgresVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index a1ae5597e..dba8c0a6d 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -61,7 +61,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = PostgresVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 2e34b1f87..77c4fb022 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -103,7 +103,7 @@ public static PostgresSubquery createSubquery(PostgresGlobalState globalState, S select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java index 6af683903..2f520bebb 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java @@ -36,7 +36,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setHavingClause(predicate); String firstQueryString = PostgresVisitor.asString(select); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java index d8aeb4938..3c183457d 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java @@ -1,45 +1,40 @@ package sqlancer.postgres.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class PostgresTLPWhereOracle extends PostgresTLPBase { +public class PostgresTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public PostgresTLPWhereOracle(PostgresGlobalState state) { - super(state); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - whereCheck(); + oracle.check(); } - protected void whereCheck() throws SQLException { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - String originalQueryString = PostgresVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = PostgresVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = PostgresVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = PostgresVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From 417b85a1e1dbfff7ddf30a8b183a038425259b88 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 11:08:07 +0800 Subject: [PATCH 0889/1284] [MariaDB] Implement expression interfaces --- .../mariadb/ast/MariaDBExpression.java | 5 ++++- src/sqlancer/mariadb/ast/MariaDBJoin.java | 5 ++++- .../mariadb/ast/MariaDBSelectStatement.java | 21 ++++++++++++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/mariadb/ast/MariaDBExpression.java b/src/sqlancer/mariadb/ast/MariaDBExpression.java index 5caec1b40..d57c14888 100644 --- a/src/sqlancer/mariadb/ast/MariaDBExpression.java +++ b/src/sqlancer/mariadb/ast/MariaDBExpression.java @@ -1,5 +1,8 @@ package sqlancer.mariadb.ast; -public interface MariaDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; + +public interface MariaDBExpression extends Expression { } diff --git a/src/sqlancer/mariadb/ast/MariaDBJoin.java b/src/sqlancer/mariadb/ast/MariaDBJoin.java index b8110b7f2..968b8b29c 100644 --- a/src/sqlancer/mariadb/ast/MariaDBJoin.java +++ b/src/sqlancer/mariadb/ast/MariaDBJoin.java @@ -5,12 +5,13 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; -public class MariaDBJoin implements MariaDBExpression { +public class MariaDBJoin implements MariaDBExpression, Join { public enum JoinType { NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; @@ -36,6 +37,7 @@ public MariaDBTable getTable() { return table; } + @Override public MariaDBExpression getOnClause() { return onClause; } @@ -44,6 +46,7 @@ public JoinType getType() { return type; } + @Override public void setOnClause(MariaDBExpression onClause) { this.onClause = onClause; } diff --git a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java index 813b92337..f09e7051f 100644 --- a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java +++ b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java @@ -4,8 +4,12 @@ import java.util.List; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -public class MariaDBSelectStatement extends SelectBase implements MariaDBExpression { +public class MariaDBSelectStatement extends SelectBase + implements MariaDBExpression, Select { public enum MariaDBSelectType { ALL, DISTINCT, DISTINCTROW; @@ -13,6 +17,7 @@ public enum MariaDBSelectType { private List groupBys = new ArrayList<>(); private List columns = new ArrayList<>(); + private List joinClauses = new ArrayList<>(); private MariaDBSelectType selectType = MariaDBSelectType.ALL; private MariaDBExpression whereCondition; @@ -52,4 +57,18 @@ public MariaDBExpression getWhereCondition() { return whereCondition; } + @Override + public List getJoinClauses() { + return joinClauses; + } + + @Override + public void setJoinClauses(List joinClauses) { + this.joinClauses = joinClauses; + } + + @Override + public String asString() { + return MariaDBVisitor.asString(this); + } } From 9ffe85beae597a111b95199fb48ccdb9e2ab4ffb Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 13:17:51 +0800 Subject: [PATCH 0890/1284] [MariaDB] Implement NoREC expression generator interface --- src/sqlancer/mariadb/MariaDBSchema.java | 4 + .../gen/MariaDBExpressionGenerator.java | 84 ++++++++++++++++--- .../mariadb/oracle/MariaDBDQPOracle.java | 3 +- 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index d1bcfbd97..808653b25 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -55,6 +55,10 @@ public boolean isPrimaryKey() { return isPrimaryKey; } + public static MariaDBColumn createDummy(String name) { + return new MariaDBColumn(name, MariaDBDataType.INT, false, 1); + } + } public static class MariaDBTables { diff --git a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java index 062bf509c..e769e399f 100644 --- a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java @@ -5,11 +5,14 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.StateToReproduce; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.mariadb.MariaDBProvider; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; +import sqlancer.mariadb.ast.MariaDBAggregate; +import sqlancer.mariadb.ast.MariaDBAggregate.MariaDBAggregateFunction; import sqlancer.mariadb.ast.MariaDBBinaryOperator; import sqlancer.mariadb.ast.MariaDBBinaryOperator.MariaDBBinaryComparisonOperator; import sqlancer.mariadb.ast.MariaDBColumnName; @@ -18,14 +21,21 @@ import sqlancer.mariadb.ast.MariaDBFunction; import sqlancer.mariadb.ast.MariaDBFunctionName; import sqlancer.mariadb.ast.MariaDBInOperation; +import sqlancer.mariadb.ast.MariaDBJoin; import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation; import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation.MariaDBPostfixUnaryOperator; +import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBSelectStatement.MariaDBSelectType; +import sqlancer.mariadb.ast.MariaDBTableReference; +import sqlancer.mariadb.ast.MariaDBText; import sqlancer.mariadb.ast.MariaDBUnaryPrefixOperation; import sqlancer.mariadb.ast.MariaDBUnaryPrefixOperation.MariaDBUnaryPrefixOperator; -public class MariaDBExpressionGenerator { +public class MariaDBExpressionGenerator + implements NoRECGenerator { private final Randomly r; + private List targetTables = new ArrayList<>(); private List columns = new ArrayList<>(); public MariaDBExpressionGenerator(Randomly r) { @@ -66,14 +76,6 @@ public MariaDBExpressionGenerator setColumns(List columns) { return this; } - public MariaDBExpressionGenerator setCon(SQLConnection con) { - return this; - } - - public MariaDBExpressionGenerator setState(StateToReproduce state) { - return this; - } - private enum ExpressionType { LITERAL, COLUMN, BINARY_COMPARISON, UNARY_POSTFIX_OPERATOR, UNARY_PREFIX_OPERATOR, FUNCTION, IN } @@ -146,4 +148,64 @@ public MariaDBExpression getRandomExpression() { return getRandomExpression(0); } + @Override + public MariaDBExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + this.targetTables = targetTables.getTables(); + this.columns = targetTables.getColumns(); + return this; + } + + @Override + public List getTableRefs() { + List tableRefs = new ArrayList<>(); + for (MariaDBTable t : targetTables) { + MariaDBTableReference tableRef = new MariaDBTableReference(t); + tableRefs.add(tableRef); + } + return tableRefs; + } + + @Override + public MariaDBExpression generateBooleanExpression() { + return getRandomExpression(); + } + + @Override + public MariaDBSelectStatement generateSelect() { + return new MariaDBSelectStatement(); + } + + @Override + public List getRandomJoinClauses() { + return new ArrayList<>(); + } + + @Override + public String generateOptimizedQueryString(MariaDBSelectStatement select, MariaDBExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + MariaDBAggregate aggr = new MariaDBAggregate( + new MariaDBColumnName(new MariaDBColumn("*", MariaDBDataType.INT, false, 0)), + MariaDBAggregateFunction.COUNT); + select.setFetchColumns(Arrays.asList(aggr)); + } else { + MariaDBColumnName aggr = new MariaDBColumnName(MariaDBColumn.createDummy("*")); + select.setFetchColumns(Arrays.asList(aggr)); + } + + select.setWhereClause(whereCondition); + select.setSelectType(MariaDBSelectType.ALL); + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(MariaDBSelectStatement select, MariaDBExpression whereCondition) { + MariaDBPostfixUnaryOperation isTrue = new MariaDBPostfixUnaryOperation(MariaDBPostfixUnaryOperator.IS_TRUE, + whereCondition); + MariaDBText asText = new MariaDBText(isTrue, " as count", false); + select.setFetchColumns(Arrays.asList(asText)); + select.setSelectType(MariaDBSelectType.ALL); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as asdf"; + } } diff --git a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java index ee85f2df7..931b0ad1d 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java @@ -38,8 +38,7 @@ public MariaDBDQPOracle(MariaDBGlobalState globalState) { @Override public void check() throws Exception { MariaDBTables tables = s.getRandomTableNonEmptyTables(); - gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(tables.getColumns()) - .setCon(state.getConnection()).setState(state.getState()); + gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(tables.getColumns()); List fetchColumns = new ArrayList<>(); fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new MariaDBColumnName(c)) From 6e10142ca30d7134edb38685fcd1ed0537599382 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 13:35:10 +0800 Subject: [PATCH 0891/1284] [MariaDB] Use generic NoREC oracle --- .../mariadb/oracle/MariaDBNoRECOracle.java | 120 +++--------------- 1 file changed, 20 insertions(+), 100 deletions(-) diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index ab6438275..22c1eea23 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -1,126 +1,46 @@ package sqlancer.mariadb.oracle; import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import sqlancer.IgnoreMeException; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; -import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -import sqlancer.mariadb.ast.MariaDBAggregate; -import sqlancer.mariadb.ast.MariaDBAggregate.MariaDBAggregateFunction; -import sqlancer.mariadb.ast.MariaDBColumnName; import sqlancer.mariadb.ast.MariaDBExpression; -import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation; -import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation.MariaDBPostfixUnaryOperator; +import sqlancer.mariadb.ast.MariaDBJoin; import sqlancer.mariadb.ast.MariaDBSelectStatement; -import sqlancer.mariadb.ast.MariaDBSelectStatement.MariaDBSelectType; -import sqlancer.mariadb.ast.MariaDBTableReference; -import sqlancer.mariadb.ast.MariaDBText; -import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; -public class MariaDBNoRECOracle extends NoRECBase implements TestOracle { +public class MariaDBNoRECOracle implements TestOracle { - private final MariaDBSchema s; - private static final int NOT_FOUND = -1; + NoRECOracle oracle; public MariaDBNoRECOracle(MariaDBGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - errors.add("is out of range"); - // regex - errors.add("unmatched parentheses"); - errors.add("nothing to repeat at offset"); - errors.add("missing )"); - errors.add("missing terminating ]"); - errors.add("range out of order in character class"); - errors.add("unrecognized character after "); - errors.add("Got error '(*VERB) not recognized or malformed"); - errors.add("must be followed by"); - errors.add("malformed number or name after"); - errors.add("digit expected after"); + MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(globalState.getRandomly()); + ExpectedErrors errors = ExpectedErrors.newErrors().with("is out of range").with("unmatched parentheses") + .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") + .with("range out of order in character class").with("unrecognized character after ") + .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") + .with("malformed number or name after").with("digit expected after").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - MariaDBTable randomTable = s.getRandomTable(); - List columns = randomTable.getColumns(); - MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(columns) - .setCon(con).setState(state.getState()); - MariaDBExpression randomWhereCondition = gen.getRandomExpression(); - List groupBys = Collections.emptyList(); // getRandomExpressions(columns); - int optimizedCount = getOptimizedQuery(randomTable, randomWhereCondition, groupBys); - int unoptimizedCount = getUnoptimizedQuery(randomTable, randomWhereCondition, groupBys); - if (optimizedCount == NOT_FOUND || unoptimizedCount == NOT_FOUND) { - throw new IgnoreMeException(); - } - if (optimizedCount != unoptimizedCount) { - state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); - throw new AssertionError(optimizedCount + " " + unoptimizedCount); - } + oracle.check(); } - private int getUnoptimizedQuery(MariaDBTable randomTable, MariaDBExpression randomWhereCondition, - List groupBys) throws SQLException { - MariaDBSelectStatement select = new MariaDBSelectStatement(); - select.setGroupByClause(groupBys); - MariaDBPostfixUnaryOperation isTrue = new MariaDBPostfixUnaryOperation(MariaDBPostfixUnaryOperator.IS_TRUE, - randomWhereCondition); - MariaDBText asText = new MariaDBText(isTrue, " as count", false); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(Arrays.asList(new MariaDBTableReference(randomTable))); - select.setSelectType(MariaDBSelectType.ALL); - int secondCount = 0; - - unoptimizedQueryString = "SELECT SUM(count) FROM (" + MariaDBVisitor.asString(select) + ") as asdf"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - try (SQLancerResultSet rs = q.executeAndGet(state)) { - if (rs == null) { - return NOT_FOUND; - } else { - while (rs.next()) { - secondCount = rs.getInt(1); - } - } - } - - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getOptimizedQuery(MariaDBTable randomTable, MariaDBExpression randomWhereCondition, - List groupBys) throws SQLException { - MariaDBSelectStatement select = new MariaDBSelectStatement(); - select.setGroupByClause(groupBys); - MariaDBAggregate aggr = new MariaDBAggregate( - new MariaDBColumnName(new MariaDBColumn("*", MariaDBDataType.INT, false, 0)), - MariaDBAggregateFunction.COUNT); - select.setFetchColumns(Arrays.asList(aggr)); - select.setFromList(Arrays.asList(new MariaDBTableReference(randomTable))); - select.setWhereClause(randomWhereCondition); - select.setSelectType(MariaDBSelectType.ALL); - int firstCount; - optimizedQueryString = MariaDBVisitor.asString(select); - SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - try (SQLancerResultSet rs = q.executeAndGet(state)) { - if (rs == null) { - firstCount = NOT_FOUND; - } else { - rs.next(); - firstCount = rs.getInt(1); - } - } catch (Exception e) { - throw new AssertionError(optimizedQueryString, e); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From d6695ca6406495b7d1e6ba020ff48fcdc825aa38 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 14:04:54 +0800 Subject: [PATCH 0892/1284] [MariaDB] Update tests --- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestMariaDB.java | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 67c406fba..084de35e7 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -6,6 +6,7 @@ public class TestConfig { public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; + public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { diff --git a/test/sqlancer/dbms/TestMariaDB.java b/test/sqlancer/dbms/TestMariaDB.java index f1b86a339..eb26bfb4a 100644 --- a/test/sqlancer/dbms/TestMariaDB.java +++ b/test/sqlancer/dbms/TestMariaDB.java @@ -39,11 +39,9 @@ public void testSelectAsString() { @Test public void testMariaDB() { - String mariaDBAvailable = System.getenv("MARIADB_AVAILABLE"); - boolean mariaDBIsAvailable = mariaDBAvailable != null && mariaDBAvailable.equalsIgnoreCase("true"); - assumeTrue(mariaDBIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MARIADB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-queries", "0", "mariadb" })); + "--num-queries", TestConfig.NUM_QUERIES, "mariadb" })); } } From f216c4187f12bed46a1ae98d39ee8235197ef065 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 17:29:30 +0800 Subject: [PATCH 0893/1284] [MariaDB] Add join clause generation --- src/sqlancer/mariadb/ast/MariaDBJoin.java | 6 ++---- src/sqlancer/mariadb/ast/MariaDBStringVisitor.java | 2 +- .../mariadb/gen/MariaDBExpressionGenerator.java | 2 +- src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java | 4 ++-- src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java | 12 +++++++----- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/mariadb/ast/MariaDBJoin.java b/src/sqlancer/mariadb/ast/MariaDBJoin.java index 968b8b29c..d846f4605 100644 --- a/src/sqlancer/mariadb/ast/MariaDBJoin.java +++ b/src/sqlancer/mariadb/ast/MariaDBJoin.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Join; -import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; @@ -55,7 +54,7 @@ public void setType(JoinType type) { this.type = type; } - public static List getRandomJoinClauses(List tables, MariaDBGlobalState globalState) { + public static List getRandomJoinClauses(List tables, Randomly r) { List joinStatements = new ArrayList<>(); List options = new ArrayList<>(Arrays.asList(JoinType.values())); List columns = new ArrayList<>(); @@ -71,8 +70,7 @@ public static List getRandomJoinClauses(List tables, MariaDBTable table = Randomly.fromList(tables); tables.remove(table); columns.addAll(table.getColumns()); - MariaDBExpressionGenerator joinGen = new MariaDBExpressionGenerator(globalState.getRandomly()) - .setColumns(columns); + MariaDBExpressionGenerator joinGen = new MariaDBExpressionGenerator(r).setColumns(columns); MariaDBExpression joinClause = joinGen.getRandomExpression(); JoinType selectedOption = Randomly.fromList(options); if (selectedOption == JoinType.NATURAL) { diff --git a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java index 17c84ef4a..e1dbf476f 100644 --- a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java @@ -47,7 +47,7 @@ public void visit(MariaDBSelectStatement s) { } visit(s.getFromList().get(j)); } - for (MariaDBExpression j : s.getJoinList()) { + for (MariaDBExpression j : s.getJoinClauses()) { visit(j); } if (s.getWhereCondition() != null) { diff --git a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java index e769e399f..3b7386121 100644 --- a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java @@ -177,7 +177,7 @@ public MariaDBSelectStatement generateSelect() { @Override public List getRandomJoinClauses() { - return new ArrayList<>(); + return MariaDBJoin.getRandomJoinClauses(targetTables, r); } @Override diff --git a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java index 931b0ad1d..dcd458193 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java @@ -56,8 +56,8 @@ public void check() throws Exception { } // Set the join. - List joinExpressions = MariaDBJoin.getRandomJoinClauses(tables.getTables(), state); - select.setJoinList(joinExpressions.stream().map(j -> (MariaDBExpression) j).collect(Collectors.toList())); + List joinExpressions = MariaDBJoin.getRandomJoinClauses(tables.getTables(), state.getRandomly()); + select.setJoinClauses(joinExpressions); // Set the from clause from the tables that are not used in the join. select.setFromList( diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index 22c1eea23..039abd85f 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -6,6 +6,7 @@ import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; +import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; @@ -21,11 +22,12 @@ public class MariaDBNoRECOracle implements TestOracle { public MariaDBNoRECOracle(MariaDBGlobalState globalState) { MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(globalState.getRandomly()); - ExpectedErrors errors = ExpectedErrors.newErrors().with("is out of range").with("unmatched parentheses") - .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") - .with("range out of order in character class").with("unrecognized character after ") - .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") - .with("malformed number or name after").with("digit expected after").build(); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MariaDBErrors.getCommonErrors()).with("is out of range") + .with("unmatched parentheses").with("nothing to repeat at offset").with("missing )") + .with("missing terminating ]").with("range out of order in character class") + .with("unrecognized character after ").with("Got error '(*VERB) not recognized or malformed") + .with("must be followed by").with("malformed number or name after").with("digit expected after") + .with("Could not create a join buffer").build(); this.oracle = new NoRECOracle<>(globalState, gen, errors); } From a306672cca6f9ce86e0cb541d4156f59921a6484 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:46:30 +0800 Subject: [PATCH 0894/1284] [Citus] Disconnect other sessions before dropping database (#960) * [Citus] Disconnect other sessions before dropping database * Format code * [Citus] Add comment for terminate query --- src/sqlancer/citus/CitusProvider.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index c897d503e..747f72216 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -371,6 +371,10 @@ private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List pg_backend_pid()"); s.execute("DROP DATABASE IF EXISTS " + databaseName); } try (Statement s = con.createStatement()) { From a05b0a025333368eb172a76561f9d114ec2da34b Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:25:13 +0800 Subject: [PATCH 0895/1284] [StoneDB] Remove implementation (#963) --- .github/workflows/main.yml | 29 -- src/check_names.py | 1 - src/sqlancer/Main.java | 2 - src/sqlancer/stonedb/StoneDBBugs.java | 51 -- src/sqlancer/stonedb/StoneDBErrors.java | 63 --- src/sqlancer/stonedb/StoneDBOptions.java | 99 ---- src/sqlancer/stonedb/StoneDBProvider.java | 157 ------- src/sqlancer/stonedb/StoneDBSchema.java | 441 ------------------ .../stonedb/StoneDBToStringVisitor.java | 101 ---- .../stonedb/ast/StoneDBAdvancedFunction.java | 34 -- .../stonedb/ast/StoneDBAggregate.java | 34 -- src/sqlancer/stonedb/ast/StoneDBConstant.java | 211 --------- .../stonedb/ast/StoneDBExpression.java | 4 - src/sqlancer/stonedb/ast/StoneDBJoin.java | 126 ----- src/sqlancer/stonedb/ast/StoneDBSelect.java | 17 - .../gen/StoneDBExpressionGenerator.java | 340 -------------- .../gen/StoneDBIndexCreateGenerator.java | 121 ----- .../gen/StoneDBIndexDropGenerator.java | 47 -- .../gen/StoneDBTableAlterGenerator.java | 141 ------ .../gen/StoneDBTableCreateGenerator.java | 266 ----------- .../gen/StoneDBTableDeleteGenerator.java | 70 --- .../gen/StoneDBTableInsertGenerator.java | 142 ------ .../gen/StoneDBTableUpdateGenerator.java | 71 --- .../gen/StoneDBViewCreateGenerator.java | 78 ---- .../oracle/StoneDBAggregateOracle.java | 170 ------- .../stonedb/oracle/StoneDBFuzzOracle.java | 67 --- .../stonedb/oracle/StoneDBNoRECOracle.java | 134 ------ .../oracle/StoneDBQueryPartitioningBase.java | 71 --- ...toneDBQueryPartitioningDistinctTester.java | 42 -- ...StoneDBQueryPartitioningGroupByTester.java | 50 -- .../StoneDBQueryPartitioningHavingTester.java | 47 -- .../StoneDBQueryPartitioningWhereTester.java | 42 -- test/sqlancer/dbms/TestStoneDBFuzz.java | 21 - test/sqlancer/dbms/TestStoneDBNoRec.java | 22 - test/sqlancer/dbms/TestStoneDBTLP.java | 21 - 35 files changed, 3333 deletions(-) delete mode 100644 src/sqlancer/stonedb/StoneDBBugs.java delete mode 100644 src/sqlancer/stonedb/StoneDBErrors.java delete mode 100644 src/sqlancer/stonedb/StoneDBOptions.java delete mode 100644 src/sqlancer/stonedb/StoneDBProvider.java delete mode 100644 src/sqlancer/stonedb/StoneDBSchema.java delete mode 100644 src/sqlancer/stonedb/StoneDBToStringVisitor.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBAggregate.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBConstant.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBExpression.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBJoin.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBSelect.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java delete mode 100644 test/sqlancer/dbms/TestStoneDBFuzz.java delete mode 100644 test/sqlancer/dbms/TestStoneDBNoRec.java delete mode 100644 test/sqlancer/dbms/TestStoneDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 56bca0594..68b02c443 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -451,35 +451,6 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test - stonedb: - name: DBMS Tests (StoneDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up StoneDB - run: | - docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb - ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb --format "{{.ID}}") - docker exec $ContainerID bash - sleep 60s - docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf - docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - sleep 60s - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - - name: Build - run: mvn -B package -DskipTests=true - - name: StoneDB Tests - run: | - STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBFuzz - tidb: name: DBMS Tests (TiDB) runs-on: ubuntu-latest diff --git a/src/check_names.py b/src/check_names.py index 41694e060..f2ab346c6 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -53,5 +53,4 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes name_to_files["Doris"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "doris")) - name_to_files["StoneDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "stonedb")) verify_all_dbs(name_to_files) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 1cbc02645..5ffc3e94c 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -43,7 +43,6 @@ import sqlancer.presto.PrestoProvider; import sqlancer.questdb.QuestDBProvider; import sqlancer.sqlite3.SQLite3Provider; -import sqlancer.stonedb.StoneDBProvider; import sqlancer.tidb.TiDBProvider; import sqlancer.timescaledb.TimescaleDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; @@ -746,7 +745,6 @@ private static void checkForIssue799(List> providers) providers.add(new PostgresProvider()); providers.add(new QuestDBProvider()); providers.add(new SQLite3Provider()); - providers.add(new StoneDBProvider()); providers.add(new TiDBProvider()); providers.add(new TimescaleDBProvider()); providers.add(new YCQLProvider()); diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java deleted file mode 100644 index 4864c56e5..000000000 --- a/src/sqlancer/stonedb/StoneDBBugs.java +++ /dev/null @@ -1,51 +0,0 @@ -package sqlancer.stonedb; - -public final class StoneDBBugs { - // https://github.com/stoneatom/stonedb/issues/1933 - public static boolean bug1933 = true; - // https://github.com/stoneatom/stonedb/issues/1942 - public static boolean bug1942 = true; - // https://github.com/stoneatom/stonedb/issues/1953 - public static boolean bug1953 = true; - // https://github.com/stoneatom/stonedb/issues/1945 - public static boolean bug1945 = true; - // CREATE TABLE t0(c0 INT); - // INSERT IGNORE INTO t0(c0) VALUE (DEFAULT); - // SELECT t0.c0 FROM t0 WHERE 0.4; -- expect 1 but got 0 - public static boolean bugNotReported1 = true; - // DELETE statements will result into crash, for example - // DELETE LOW_PRIORITY FROM t0; - // DELETE QUICK IGNORE FROM t0 WHERE -1370759901; - public static boolean bugNotReported2 = true; - // CREATE TABLE t0(c0 INT) ; - // INSERT INTO t0(c0) VALUES (DEFAULT); - // SELECT t0.c0 FROM t0 WHERE (('OC')>=(((t0.c0) IS NULL))); -- expected empty set but got 1 row - public static boolean bugNotReported3 = true; - // CREATE TABLE t0(c0 CHAR PRIMARY KEY); - // INSERT IGNORE INTO t0(c0) VALUE (DEFAULT); - // SELECT t0.c0 FROM t0 WHERE (t0.c0 BETWEEN (0) AND t0.c0); -- 1 - public static boolean bugNotReported4 = true; - // CREATE TABLE t0(c0 INT); - // INSERT INTO t0(c0) VALUE (DEFAULT); - // SELECT * FROM t0 WHERE (('00')>((('')AND(t0.c0)))); -- expected empty set but got 1 row - public static boolean bugNotReported5 = true; - // CREATE TABLE t0(c0 INT) ; - // INSERT INTO t0(c0) VALUE (100); - // SELECT * FROM t0 WHERE (('Xg')>(t0.c0)); -- expected empty set but got 1 row - // SELECT (('Xg')>(t0.c0)) FROM t0; - public static boolean bugNotReported6 = true; - // CREATE TABLE t0(c0 VARCHAR(1000)); - // INSERT INTO t0(c0) VALUE ('GX9vartD'); - // INSERT INTO t0(c0) VALUE ('16'); - // SELECT t0.c0 FROM t0 WHERE t0.c0; -- expected 1 row but got 2 rows - public static boolean bugNotReported7 = true; - // CREATE TABLE t0(c0 INT) ; - // SELECT * FROM t0 WHERE ((t0.c0)XOR((t0.c0))); - // ERROR 6 (HY000): The query includes syntax that is not supported by the storage engine. Either restructure the - // query with supported syntax, or enable the MySQL core::Query Path in config file to execute the query with - // reduced performance. - public static boolean bugNotReportedXOR = true; - - private StoneDBBugs() { - } -} diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java deleted file mode 100644 index d8b0fd416..000000000 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ /dev/null @@ -1,63 +0,0 @@ -package sqlancer.stonedb; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.common.query.ExpectedErrors; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; - -public final class StoneDBErrors { - private StoneDBErrors() { - } - - public static List getExpectedExpressionErrors(StoneDBGlobalState globalState) { - ArrayList errors = new ArrayList<>(); - - // java.sql.SQLException: Incorrect DATE value: '292269055-12-02' - errors.add("Incorrect DATE value: "); - // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for - // column 'c1' at row 1 - errors.add("Incorrect string value: "); - // java.sql.SQLException: Incorrect integer value: 'ST' for column 'c1' at row 1 - errors.add("Incorrect integer value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Truncated - // incorrect INTEGER value: '#Q' - errors.add("Data truncation: Truncated incorrect INTEGER value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT - // value is out of range in - // '-((`database0`.`t0`.`c1` >> (not(`database0`.`t0`.`c1`))))' - errors.add("Data truncation: BIGINT value is out of range in "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT - // UNSIGNED value is out of range in - // '(`database10`.`t0`.`c0` + (`database10`.`t0`.`c0` & (not(0.5))))' - errors.add("Data truncation: BIGINT UNSIGNED value is out of range in "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect - // time value: '0Sly6xqF0' for - // column 'c1' at row 1 - errors.add("Data truncation: Incorrect time value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: DOUBLE - // value is out of range in - // '(`database0`.`t0`.`c0` * `database0`.`t0`.`c0`)' - errors.add("Data truncation: DOUBLE value is out of range in "); - // java.sql.SQLException: Numeric result of an expression is too large and - // cannot be handled by tianmu. - errors.add("Numeric result of an expression is too large and cannot be handled by tianmu."); - // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order - // clause' - errors.add("Unknown column "); - if (globalState.getDbmsSpecificOptions().test80Version) { - // Caused by: java.sql.SQLException: Incorrect DATETIME value: '292269055-12-02 - // 16:47:04' - errors.add("Incorrect DATETIME value: "); - // Caused by: java.sql.SQLException: Incorrect TIMESTAMP value: '292269055-12-02 - // 16:47:04' - errors.add("Incorrect TIMESTAMP value: "); - } - - return errors; - } - - public static void addExpectedExpressionErrors(StoneDBGlobalState globalState, ExpectedErrors errors) { - errors.addAll(getExpectedExpressionErrors(globalState)); - } -} diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java deleted file mode 100644 index 0c9d8816a..000000000 --- a/src/sqlancer/stonedb/StoneDBOptions.java +++ /dev/null @@ -1,99 +0,0 @@ -package sqlancer.stonedb; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.stonedb.StoneDBOptions.StoneDBOracleFactory; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.oracle.StoneDBAggregateOracle; -import sqlancer.stonedb.oracle.StoneDBFuzzOracle; -import sqlancer.stonedb.oracle.StoneDBNoRECOracle; -import sqlancer.stonedb.oracle.StoneDBQueryPartitioningDistinctTester; -import sqlancer.stonedb.oracle.StoneDBQueryPartitioningGroupByTester; -import sqlancer.stonedb.oracle.StoneDBQueryPartitioningHavingTester; -import sqlancer.stonedb.oracle.StoneDBQueryPartitioningWhereTester; - -@Parameters(separators = "=", commandDescription = "StoneDB (default host: " + StoneDBOptions.DEFAULT_HOST - + ", default port: " + StoneDBOptions.DEFAULT_PORT + ")") -public class StoneDBOptions implements DBMSSpecificOptions { - public static final String DEFAULT_HOST = "localhost"; - public static final int DEFAULT_PORT = 3306; - - @Parameter(names = "--test-8-version", description = "Let SQLancer test the StoneDB 8.0, otherwise, SQLancer will test the StoneDB 5.7", arity = 1) - public boolean test80Version = true; - - @Parameter(names = "--oracle") - public List oracles = List.of(StoneDBOracleFactory.NOREC); - - public enum StoneDBOracleFactory implements OracleFactory { - FUZZER { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBFuzzOracle(globalState); - } - }, - NOREC { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBNoRECOracle(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new StoneDBQueryPartitioningWhereTester(globalState)); - oracles.add(new StoneDBQueryPartitioningHavingTester(globalState)); - oracles.add(new StoneDBAggregateOracle(globalState)); - oracles.add(new StoneDBQueryPartitioningDistinctTester(globalState)); - oracles.add(new StoneDBQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle<>(oracles, globalState); - } - }, - HAVING { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBAggregateOracle(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBQueryPartitioningDistinctTester(globalState); - } - } - } - - @Override - public List getTestOracleFactory() { - return oracles; - } -} diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java deleted file mode 100644 index e4d9c46f8..000000000 --- a/src/sqlancer/stonedb/StoneDBProvider.java +++ /dev/null @@ -1,157 +0,0 @@ -package sqlancer.stonedb; - -import static sqlancer.stonedb.StoneDBBugs.bugNotReported2; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; - -import com.google.auto.service.AutoService; - -import sqlancer.AbstractAction; -import sqlancer.DatabaseProvider; -import sqlancer.IgnoreMeException; -import sqlancer.MainOptions; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; -import sqlancer.SQLProviderAdapter; -import sqlancer.StatementExecutor; -import sqlancer.common.DBMSCommon; -import sqlancer.common.query.Query; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLQueryProvider; -import sqlancer.stonedb.gen.StoneDBIndexCreateGenerator; -import sqlancer.stonedb.gen.StoneDBIndexDropGenerator; -import sqlancer.stonedb.gen.StoneDBTableAlterGenerator; -import sqlancer.stonedb.gen.StoneDBTableCreateGenerator; -import sqlancer.stonedb.gen.StoneDBTableDeleteGenerator; -import sqlancer.stonedb.gen.StoneDBTableInsertGenerator; -import sqlancer.stonedb.gen.StoneDBTableUpdateGenerator; -import sqlancer.stonedb.gen.StoneDBViewCreateGenerator; - -@AutoService(DatabaseProvider.class) -public class StoneDBProvider extends SQLProviderAdapter { - - public StoneDBProvider() { - super(StoneDBGlobalState.class, StoneDBOptions.class); - } - - public static class StoneDBGlobalState extends SQLGlobalState { - @Override - protected StoneDBSchema readSchema() throws Exception { - return StoneDBSchema.fromConnection(getConnection(), getDatabaseName()); - } - } - - enum Action implements AbstractAction { - TABLES_SHOW((g) -> new SQLQueryAdapter("SHOW TABLES")), // - TABLE_ALTER(StoneDBTableAlterGenerator::generate), // - TABLE_CREATE((g) -> { - String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); - return StoneDBTableCreateGenerator.generate(g, tableName); - }), // - TABLE_DELETE(StoneDBTableDeleteGenerator::generate), // - TABLE_UPDATE(StoneDBTableUpdateGenerator::generate), // - INDEX_CREATE(StoneDBIndexCreateGenerator::generate), // - INDEX_DROP(StoneDBIndexDropGenerator::generate), // - TABLE_INSERT(StoneDBTableInsertGenerator::generate), // - - VIEW_CREATE((g) -> { - String viewName = g.getSchema().getFreeViewName(); - return StoneDBViewCreateGenerator.generate(g, viewName); - }); - - private final SQLQueryProvider sqlQueryProvider; - - Action(SQLQueryProvider sqlQueryProvider) { - this.sqlQueryProvider = sqlQueryProvider; - } - - @Override - public Query getQuery(StoneDBGlobalState globalState) throws Exception { - return sqlQueryProvider.getQuery(globalState); - } - } - - private static int mapActions(StoneDBGlobalState globalState, Action a) { - Randomly r = globalState.getRandomly(); - switch (a) { - case TABLES_SHOW: - return r.getInteger(0, 1); - case TABLE_ALTER: - return r.getInteger(0, 5); - case TABLE_CREATE: - return r.getInteger(0, 1); - case TABLE_DELETE: - if (bugNotReported2) { - return 0; - } - return r.getInteger(0, 10); - case TABLE_INSERT: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case TABLE_UPDATE: - return r.getInteger(0, 1); - case INDEX_CREATE: - return r.getInteger(0, 1); - case INDEX_DROP: - return r.getInteger(0, 1); - case VIEW_CREATE: - return r.getInteger(0, 1); - default: - throw new AssertionError(a); - } - } - - @Override - public void generateDatabase(StoneDBGlobalState globalState) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { - String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); - SQLQueryAdapter createTable = StoneDBTableCreateGenerator.generate(globalState, tableName); - globalState.executeStatement(createTable); - } - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - StoneDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public SQLConnection createDatabase(StoneDBGlobalState globalState) throws Exception { - String username = globalState.getOptions().getUserName(); - String password = globalState.getOptions().getPassword(); - String host = globalState.getOptions().getHost(); - int port = globalState.getOptions().getPort(); - if (host == null) { - host = StoneDBOptions.DEFAULT_HOST; - } - if (port == MainOptions.NO_SET_PORT) { - port = StoneDBOptions.DEFAULT_PORT; - } - String databaseName = globalState.getDatabaseName(); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - globalState.getState().logStatement("CREATE DATABASE " + databaseName); - globalState.getState().logStatement("USE " + databaseName); - String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", - host, port); - Connection con = DriverManager.getConnection(url, username, password); - try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); - } - try (Statement s = con.createStatement()) { - s.execute("CREATE DATABASE " + databaseName); - } - try (Statement s = con.createStatement()) { - s.execute("USE " + databaseName); - } - return new SQLConnection(con); - } - - @Override - public String getDBMSName() { - return "stonedb"; - } -} diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java deleted file mode 100644 index 346031fba..000000000 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ /dev/null @@ -1,441 +0,0 @@ -package sqlancer.stonedb; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.schema.AbstractRelationalTable; -import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.common.schema.AbstractTables; -import sqlancer.common.schema.TableIndex; -import sqlancer.stonedb.ast.StoneDBConstant; -import sqlancer.stonedb.ast.StoneDBExpression; - -public class StoneDBSchema extends AbstractSchema { - - public enum StoneDBDataType { - - NULL, TINYINT, SMALLINT, MEDIUMINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, YEAR, TIME, DATE, DATETIME, TIMESTAMP, - CHAR, VARCHAR, TINYTEXT, TEXT, MEDIUMTEXT, LONGTEXT, BINARY, VARBINARY, TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB; - - public static StoneDBDataType getRandomWithoutNull() { - List collect = Stream.of(values()).filter(c -> c != NULL).collect(Collectors.toList()); - return Randomly.fromList(collect); - } - - public static StoneDBDataType getRandom() { - return Randomly.fromOptions(values()); - } - - public static Node getRandomValue(StoneDBDataType dataType, Randomly r) { - switch (dataType) { - case TINYINT: - break; - case SMALLINT: - break; - case MEDIUMINT: - break; - case INT: - return StoneDBConstant.createIntConstant((int) Randomly.getNonCachedInteger()); - case BIGINT: - break; - case FLOAT: - break; - case DOUBLE: - return StoneDBConstant.createDoubleConstant(Randomly.getUncachedDouble()); - case DECIMAL: - break; - case YEAR: - break; - case TIME: - break; - case DATE: - return StoneDBConstant.createDateConstant(Randomly.getNonCachedInteger()); - case DATETIME: - break; - case TIMESTAMP: - return StoneDBConstant.createTimestampConstant(Randomly.getNonCachedInteger()); - case CHAR: - break; - case VARCHAR: - break; - case TINYTEXT: - break; - case TEXT: - return StoneDBConstant.createTextConstant(r.getString()); - case MEDIUMTEXT: - break; - case LONGTEXT: - break; - case BINARY: - break; - case VARBINARY: - break; - case TINYBLOB: - break; - case BLOB: - break; - case MEDIUMBLOB: - break; - case LONGBLOB: - break; - default: - throw new AssertionError(); - } - return null; - } - - public static String getTypeAndValue(StoneDBDataType dataType, Randomly r) { - StringBuilder sb = new StringBuilder(); - switch (dataType) { - case TINYINT: - return "TINYINT"; - case SMALLINT: - return "SMALLINT"; - case MEDIUMINT: - return "MEDIUMINT"; - case INT: - return "INT"; - case BIGINT: - return "BIGINT"; - case FLOAT: - sb.append("FLOAT"); - optionallyAddPrecisionAndScale(sb); - return sb.toString(); - case DOUBLE: - sb.append("DOUBLE"); - optionallyAddPrecisionAndScale(sb); - return sb.toString(); - case DECIMAL: - return "DECIMAL"; // The default value is P(10,0); - case YEAR: - return "YEAR"; - case TIME: - return "TIME"; - case DATE: - return "DATE"; - case DATETIME: - return "DATETIME"; - case TIMESTAMP: - return "TIMESTAMP"; - case CHAR: - sb.append("CHAR").append(Randomly.fromOptions("", "(" + r.getInteger(0, 255) + ")")); - return sb.toString(); - case VARCHAR: - sb.append("VARCHAR").append("(").append(r.getInteger(0, 65535)).append(")"); - return sb.toString(); - case TINYTEXT: - return "TINYTEXT"; - case TEXT: - return "TEXT"; - case MEDIUMTEXT: - return "MEDIUMTEXT"; - case LONGTEXT: - return "LONGTEXT"; - case BINARY: - return "BINARY"; - case VARBINARY: - sb.append("VARBINARY").append("(").append(r.getInteger(0, 65535)).append(")"); - return sb.toString(); - case TINYBLOB: - return "TINYBLOB"; - case BLOB: - return "BLOB"; - case MEDIUMBLOB: - return "MEDIUMBLOB"; - case LONGBLOB: - return "LONGBLOB"; - default: - throw new AssertionError(); - } - } - - private static void optionallyAddPrecisionAndScale(StringBuilder sb) { - if (Randomly.getBoolean()) { - sb.append("("); - // The maximum number of digits (M) for DECIMAL is 65 - long m = Randomly.getNotCachedInteger(1, 65); - sb.append(m); - sb.append(", "); - // The maximum number of supported decimals (D) is 30 - long nCandidate = Randomly.getNotCachedInteger(1, 30); - // For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'c0'). - long n = Math.min(nCandidate, m); - sb.append(n); - sb.append(")"); - } - } - } - - public static class StoneDBTable - extends AbstractRelationalTable { - - public StoneDBTable(String tableName, List columns, List indexes, boolean isView) { - super(tableName, columns, indexes, isView); - } - - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - - } - - public static class StoneDBTables extends AbstractTables { - public StoneDBTables(List tables) { - super(tables); - } - } - - public static final class StoneDBIndex extends TableIndex { - private StoneDBIndex(String indexName) { - super(indexName); - } - - public static StoneDBIndex create(String indexName) { - return new StoneDBIndex(indexName); - } - - @Override - public String getIndexName() { - if (super.getIndexName().contentEquals("PRIMARY")) { - return "`PRIMARY`"; - } else { - return super.getIndexName(); - } - } - } - - public static StoneDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { - List databaseTables = new ArrayList<>(); - List tableNames = getTableNames(con, databaseName); - for (String tableName : tableNames) { - List databaseColumns = getTableColumns(con, databaseName, tableName); - List indexes = getIndexes(con, databaseName, tableName); - boolean isView = tableName.startsWith("v"); - StoneDBTable t = new StoneDBTable(tableName, databaseColumns, indexes, isView); - for (StoneDBColumn c : databaseColumns) { - c.setTable(t); - } - databaseTables.add(t); - - } - return new StoneDBSchema(databaseTables); - } - - private static List getIndexes(SQLConnection con, String databaseName, String tableName) - throws SQLException { - List indexes = new ArrayList<>(); - try (ResultSet rs = con.createStatement() - .executeQuery("SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = '" - + databaseName + "' AND TABLE_NAME='" + tableName + "';")) { - while (rs.next()) { - String indexName = rs.getString("INDEX_NAME"); - indexes.add(StoneDBIndex.create(indexName)); - } - } - return indexes; - } - - private static List getTableColumns(SQLConnection con, String databaseName, String tableName) - throws SQLException { - List columns = new ArrayList<>(); - try (ResultSet rs = con.createStatement() - .executeQuery("select * from information_schema.columns where table_schema = '" + databaseName - + "' AND TABLE_NAME='" + tableName + "';")) { - while (rs.next()) { - String columnName = rs.getString("COLUMN_NAME"); - String dataType = rs.getString("DATA_TYPE"); - int precision = rs.getInt("NUMERIC_PRECISION"); - boolean isNullable = !rs.getString("IS_NULLABLE").equals("NO"); - boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); - StoneDBColumn c = new StoneDBColumn(columnName, getColumnCompositeDataType(dataType), isPrimaryKey, - isNullable, precision); - columns.add(c); - } - } - return columns; - } - - private static StoneDBCompositeDataType getColumnCompositeDataType(String typeString) { - if (Arrays.stream(StoneDBDataType.values()).noneMatch(e -> e.name().equals(typeString.toUpperCase()))) { - throw new AssertionError(typeString); - } - return new StoneDBCompositeDataType(StoneDBDataType.valueOf(typeString.toUpperCase())); - } - - public static StoneDBDataType getColumnDataType(String typeString) { - if (Arrays.stream(StoneDBDataType.values()).noneMatch(e -> e.name().equals(typeString.toUpperCase()))) { - throw new AssertionError(typeString); - } - return StoneDBDataType.valueOf(typeString.toUpperCase()); - } - - private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { - List tableNames = new ArrayList<>(); - try (ResultSet rs = con.createStatement() - .executeQuery("select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" - + databaseName + "';")) { - while (rs.next()) { - if (rs.getString("ENGINE").equals("TIANMU")) { - tableNames.add(rs.getString("TABLE_NAME")); - } - } - } - return tableNames; - } - - public static class StoneDBColumn extends AbstractTableColumn { - - private final boolean isPrimaryKey; - private final boolean isNullable; - private final int precision; - - public StoneDBColumn(String name, StoneDBCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable, - int precision) { - super(name, null, columnType); - this.isPrimaryKey = isPrimaryKey; - this.isNullable = isNullable; - this.precision = precision; - } - - public boolean isPrimaryKey() { - return isPrimaryKey; - } - - public boolean isNullable() { - return isNullable; - } - - public int getPrecision() { - return precision; - } - - } - - public StoneDBSchema(List databaseTables) { - super(databaseTables); - } - - public static class StoneDBCompositeDataType { - private final StoneDBDataType dataType; - private final int size; - - public StoneDBCompositeDataType(StoneDBDataType dataType, int size) { - this.dataType = dataType; - this.size = size; - } - - public StoneDBCompositeDataType(StoneDBDataType dataType) { - this.dataType = dataType; - int size = -1; - switch (dataType) { - case TINYINT: - size = 1; - break; - case SMALLINT: - size = 2; - break; - case MEDIUMINT: - size = 3; - break; - case INT: - size = 4; - break; - case BIGINT: - size = 8; - break; - case FLOAT: - size = 4; - break; - case DOUBLE: - size = 8; - break; - case DECIMAL: - size = -2; - break; - case YEAR: - size = -2; - break; - case TIME: - size = -2; - break; - case DATE: - size = -2; - break; - case DATETIME: - size = -2; - break; - case TIMESTAMP: - size = -2; - break; - case CHAR: - size = -2; - break; - case VARCHAR: - size = -2; - break; - case TINYTEXT: - size = -2; - break; - case TEXT: - size = -2; - break; - case MEDIUMTEXT: - size = -2; - break; - case LONGTEXT: - size = -2; - break; - case BINARY: - size = -2; - break; - case VARBINARY: - size = -2; - break; - case TINYBLOB: - size = -2; - break; - case BLOB: - size = -2; - break; - case MEDIUMBLOB: - size = -2; - break; - case LONGBLOB: - size = -2; - break; - default: - throw new AssertionError(); - } - this.size = size; - } - - public StoneDBDataType getPrimitiveDataType() { - return dataType; - } - - public int getSize() { - if (size == -1) { - throw new AssertionError(this); - } - return size; - } - - public static StoneDBCompositeDataType getRandomWithoutNull() { - StoneDBDataType type = StoneDBDataType.getRandomWithoutNull(); - return new StoneDBCompositeDataType(type); - } - } - - public StoneDBTables getRandomTableNonEmptyTables() { - return new StoneDBTables(Randomly.nonEmptySubset(getDatabaseTables())); - } -} diff --git a/src/sqlancer/stonedb/StoneDBToStringVisitor.java b/src/sqlancer/stonedb/StoneDBToStringVisitor.java deleted file mode 100644 index 971297ddd..000000000 --- a/src/sqlancer/stonedb/StoneDBToStringVisitor.java +++ /dev/null @@ -1,101 +0,0 @@ -package sqlancer.stonedb; - -import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.ast.StoneDBConstant; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; - -public class StoneDBToStringVisitor extends NewToStringVisitor { - @Override - public void visitSpecific(Node expr) { - if (expr instanceof StoneDBConstant) { - visit((StoneDBConstant) expr); - } else if (expr instanceof StoneDBSelect) { - visit((StoneDBSelect) expr); - } else if (expr instanceof StoneDBJoin) { - visit((StoneDBJoin) expr); - } else if (expr instanceof StoneDBCastOperation) { - visit((StoneDBCastOperation) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - private void visit(StoneDBConstant constant) { - sb.append(constant.toString()); - } - - private void visit(StoneDBSelect select) { - sb.append("SELECT "); - if (select.isDistinct()) { - sb.append("DISTINCT "); - } - visit(select.getFetchColumns()); - sb.append(" FROM "); - visit(select.getFromList()); - if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { - sb.append(", "); - } - if (!select.getJoinList().isEmpty()) { - visit(select.getJoinList()); - } - if (select.getWhereClause() != null) { - sb.append(" WHERE "); - visit(select.getWhereClause()); - } - if (!select.getGroupByExpressions().isEmpty()) { - sb.append(" GROUP BY "); - visit(select.getGroupByExpressions()); - } - if (select.getHavingClause() != null) { - sb.append(" HAVING "); - visit(select.getHavingClause()); - } - if (!select.getOrderByClauses().isEmpty()) { - sb.append(" ORDER BY "); - visit(select.getOrderByClauses()); - } - if (select.getLimitClause() != null) { - sb.append(" LIMIT "); - visit(select.getLimitClause()); - } - if (select.getOffsetClause() != null) { - sb.append(" OFFSET "); - visit(select.getOffsetClause()); - } - } - - private void visit(StoneDBJoin join) { - visit(join.getLeftTable()); - sb.append(" "); - sb.append(join.getJoinType()); - sb.append(" "); - if (join.getNaturalJoinType() != null) { - sb.append(join.getNaturalJoinType()); - } - sb.append(" JOIN "); - visit(join.getRightTable()); - if (join.getOnCondition() != null) { - sb.append(" ON "); - visit(join.getOnCondition()); - } - } - - private void visit(StoneDBCastOperation cast) { - sb.append("CAST("); - visit(cast.getExpr()); - sb.append(" AS "); - sb.append(cast.getType() == StoneDBDataType.INT ? "UNSIGNED" : cast.getType().toString()); - sb.append(") "); - } - - public static String asString(Node expr) { - StoneDBToStringVisitor visitor = new StoneDBToStringVisitor(); - visitor.visit(expr); - return visitor.get(); - } -} diff --git a/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java b/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java deleted file mode 100644 index 9c49f2678..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java +++ /dev/null @@ -1,34 +0,0 @@ -package sqlancer.stonedb.ast; - -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.FunctionNode; -import sqlancer.stonedb.ast.StoneDBAdvancedFunction.StoneDBAdvancedFunc; - -public class StoneDBAdvancedFunction extends FunctionNode - implements StoneDBExpression { - - // https://stonedb.io/docs/SQL-reference/functions/advanced-functions - public enum StoneDBAdvancedFunc { - IFNULL(2), IF(3), NULLIF(2), BIN(1), BINARY(1), CONV(3); - - private int nrArgs; - - StoneDBAdvancedFunc(int nrArgs) { - this.nrArgs = nrArgs; - } - - public static StoneDBAdvancedFunc getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - } - - protected StoneDBAdvancedFunction(StoneDBAdvancedFunc function, List args) { - super(function, args); - } -} diff --git a/src/sqlancer/stonedb/ast/StoneDBAggregate.java b/src/sqlancer/stonedb/ast/StoneDBAggregate.java deleted file mode 100644 index f65b54d74..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBAggregate.java +++ /dev/null @@ -1,34 +0,0 @@ -package sqlancer.stonedb.ast; - -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.FunctionNode; -import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; - -public class StoneDBAggregate extends FunctionNode - implements StoneDBExpression { - - // https://stonedb.io/docs/SQL-reference/functions/aggregate-functions/ - public enum StoneDBAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1), SUM(1); - - private int nrArgs; - - StoneDBAggregateFunction(int nrArgs) { - this.nrArgs = nrArgs; - } - - public static StoneDBAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - } - - protected StoneDBAggregate(StoneDBAggregateFunction function, List args) { - super(function, args); - } -} diff --git a/src/sqlancer/stonedb/ast/StoneDBConstant.java b/src/sqlancer/stonedb/ast/StoneDBConstant.java deleted file mode 100644 index 6760bc25c..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBConstant.java +++ /dev/null @@ -1,211 +0,0 @@ -package sqlancer.stonedb.ast; - -import static sqlancer.stonedb.StoneDBBugs.bugNotReported1; - -import java.sql.Timestamp; -import java.text.SimpleDateFormat; - -import sqlancer.common.ast.newast.Node; - -public class StoneDBConstant implements Node { - - private StoneDBConstant() { - } - - public static class StoneDBNullConstant extends StoneDBConstant { - @Override - public String toString() { - return "NULL"; - } - } - - public static Node createNullConstant() { - return new StoneDBNullConstant(); - } - - public static class StoneDBIntConstant extends StoneDBConstant { - private final Integer value; - - public StoneDBIntConstant(int value) { - this.value = value; - } - - @Override - public String toString() { - if (value.equals(Integer.MIN_VALUE)) { - return "INT_NULL"; - } - return String.valueOf(value); - } - } - - public static Node createIntConstant(int val) { - return new StoneDBIntConstant(val); - } - - public static class StoneDBBigIntConstant extends StoneDBConstant { - private final Long value; - - public StoneDBBigIntConstant(long value) { - this.value = value; - } - - @Override - public String toString() { - // For INT type: -2147483648 is reserved to indicate INT_NULL in Tianmu engine, Minimum Value Signed start - // from -2147483647 - // refer: https://stonedb.io/docs/SQL-reference/data-types - if (value.equals(Long.MIN_VALUE)) { - return "BIGINT_NULL"; - } - return String.valueOf(value); - } - } - - public static class StoneDBDoubleConstant extends StoneDBConstant { - - private final Double value; - - public StoneDBDoubleConstant(double value) { - this.value = value; - } - - public double getValue() { - return value; - } - - @Override - public String toString() { - if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; - } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; - } - return String.valueOf(value); - } - - } - - public static Node createDoubleConstant(double val) { - if (bugNotReported1 && val < 0.5 && val > 0) { - return new StoneDBDoubleConstant(0.5); - } - return new StoneDBDoubleConstant(val); - } - - public static class StoneDBTextConstant extends StoneDBConstant { - - private final String value; - - public StoneDBTextConstant(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - - @Override - public String toString() { - return "'" + value.replace("'", "''") + "'"; - } - - } - - public static Node createTextConstant(String text) { - return new StoneDBTextConstant(text); - } - - public static class StoneDBBitConstant extends StoneDBConstant { - - private final String value; - - public StoneDBBitConstant(long value) { - this.value = Long.toBinaryString(value); - } - - public String getValue() { - return value; - } - - @Override - public String toString() { - return "B'" + value + "'"; - } - - } - - public static class StoneDBDateConstant extends StoneDBConstant { - - public String textRepr; - - public StoneDBDateConstant(long val) { - Timestamp timestamp = new Timestamp(val); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - textRepr = dateFormat.format(timestamp); - } - - public String getValue() { - return textRepr; - } - - @Override - public String toString() { - return String.format("DATE '%s'", textRepr); - } - - } - - public static Node createDateConstant(long integer) { - return new StoneDBDateConstant(integer); - } - - public static class StoneDBTimestampConstant extends StoneDBConstant { - - public String textRepr; - - public StoneDBTimestampConstant(long val) { - Timestamp timestamp = new Timestamp(val); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - textRepr = dateFormat.format(timestamp); - } - - public String getValue() { - return textRepr; - } - - @Override - public String toString() { - return String.format("TIMESTAMP '%s'", textRepr); - } - - } - - public static Node createTimestampConstant(long integer) { - return new StoneDBTimestampConstant(integer); - } - - public static class StoneDBBooleanConstant extends StoneDBConstant { - - private final boolean value; - - public StoneDBBooleanConstant(boolean value) { - this.value = value; - } - - public boolean getValue() { - return value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - } - - public static Node createBooleanConstant(boolean val) { - return new StoneDBBooleanConstant(val); - } - -} diff --git a/src/sqlancer/stonedb/ast/StoneDBExpression.java b/src/sqlancer/stonedb/ast/StoneDBExpression.java deleted file mode 100644 index fb8e75124..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBExpression.java +++ /dev/null @@ -1,4 +0,0 @@ -package sqlancer.stonedb.ast; - -public interface StoneDBExpression { -} diff --git a/src/sqlancer/stonedb/ast/StoneDBJoin.java b/src/sqlancer/stonedb/ast/StoneDBJoin.java deleted file mode 100644 index 8bfff08ee..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBJoin.java +++ /dev/null @@ -1,126 +0,0 @@ -package sqlancer.stonedb.ast; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator; - -public class StoneDBJoin implements Node { - - public enum JoinType { - INNER, NATURAL, LEFT, RIGHT; - - public static JoinType getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum NaturalJoinType { - LEFT, RIGHT; - - public static NaturalJoinType getRandom() { - return Randomly.fromOptions(values()); - } - } - - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; - private final JoinType joinType; - private final Node onCondition; - private NaturalJoinType naturalJoinType; - - public StoneDBJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node onCondition) { - this.leftTable = leftTable; - this.rightTable = rightTable; - this.joinType = joinType; - this.onCondition = onCondition; - } - - public TableReferenceNode getLeftTable() { - return leftTable; - } - - public TableReferenceNode getRightTable() { - return rightTable; - } - - public JoinType getJoinType() { - return joinType; - } - - public Node getOnCondition() { - return onCondition; - } - - public NaturalJoinType getNaturalJoinType() { - return naturalJoinType; - } - - public void setNaturalJoinType(NaturalJoinType naturalJoinType) { - this.naturalJoinType = naturalJoinType; - } - - public static List> getJoins( - List> tableList, StoneDBGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); - while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - // get two tables to join - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); - // store all columns in the above two tables - List columns = new ArrayList<>(leftTable.getTable().getColumns()); - columns.addAll(rightTable.getTable().getColumns()); - // create a join generator - StoneDBExpressionGenerator joinGen = new StoneDBExpressionGenerator(globalState).setColumns(columns); - switch (StoneDBJoin.JoinType.getRandom()) { - case INNER: - joinExpressions.add(StoneDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); - break; - case NATURAL: - joinExpressions.add(StoneDBJoin.createNaturalJoin(leftTable, rightTable, NaturalJoinType.getRandom())); - break; - case LEFT: - joinExpressions - .add(StoneDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); - break; - case RIGHT: - joinExpressions - .add(StoneDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); - break; - default: - throw new AssertionError(); - } - } - return joinExpressions; - } - - public static StoneDBJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node onClause) { - return new StoneDBJoin(left, right, JoinType.RIGHT, onClause); - } - - public static StoneDBJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node onClause) { - return new StoneDBJoin(left, right, JoinType.LEFT, onClause); - } - - public static StoneDBJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node onClause) { - return new StoneDBJoin(left, right, JoinType.INNER, onClause); - } - - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, NaturalJoinType naturalJoinType) { - StoneDBJoin join = new StoneDBJoin(left, right, JoinType.NATURAL, null); - join.setNaturalJoinType(naturalJoinType); - return join; - } -} diff --git a/src/sqlancer/stonedb/ast/StoneDBSelect.java b/src/sqlancer/stonedb/ast/StoneDBSelect.java deleted file mode 100644 index 081106260..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBSelect.java +++ /dev/null @@ -1,17 +0,0 @@ -package sqlancer.stonedb.ast; - -import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; - -public class StoneDBSelect extends SelectBase> implements Node { - - private boolean isDistinct; - - public void setDistinct(boolean isDistinct) { - this.isDistinct = isDistinct; - } - - public boolean isDistinct() { - return isDistinct; - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java deleted file mode 100644 index 3e1dcd348..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ /dev/null @@ -1,340 +0,0 @@ -package sqlancer.stonedb.gen; - -import static sqlancer.stonedb.StoneDBBugs.bug1942; -import static sqlancer.stonedb.StoneDBBugs.bugNotReported3; -import static sqlancer.stonedb.StoneDBBugs.bugNotReported6; -import static sqlancer.stonedb.StoneDBBugs.bugNotReportedXOR; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewCaseOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.ast.StoneDBAdvancedFunction.StoneDBAdvancedFunc; -import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; -import sqlancer.stonedb.ast.StoneDBConstant; -import sqlancer.stonedb.ast.StoneDBExpression; - -public class StoneDBExpressionGenerator extends UntypedExpressionGenerator, StoneDBColumn> { - - private final StoneDBGlobalState globalState; - - public StoneDBExpressionGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - } - - private enum Expression { - UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, BINARY_BITWISE, BETWEEN, IN, - NOT_IN, CASE - } - - public static class StoneDBCastOperation implements Node { - Node expr; - StoneDBDataType type; - - public StoneDBCastOperation(Node expr, StoneDBDataType type) { - this.expr = expr; - this.type = type; - } - - public Node getExpr() { - return expr; - } - - public StoneDBDataType getType() { - return type; - } - } - - @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, StoneDBUnaryPrefixOperator.NOT); - } - - @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, StoneDBUnaryPostfixOperator.IS_NULL); - } - - @Override - public Node generateConstant() { - if (Randomly.getBooleanWithSmallProbability()) { - return StoneDBConstant.createNullConstant(); - } - StoneDBDataType type = StoneDBDataType.getRandomWithoutNull(); - return generateConstant(type); - } - - public Node generateConstant(StoneDBDataType dataType) { - switch (dataType) { - case INT: - return StoneDBConstant - .createIntConstant(globalState.getRandomly().getInteger(Integer.MIN_VALUE + 1, Integer.MAX_VALUE)); - case DATE: - return StoneDBConstant.createDateConstant(globalState.getRandomly().getInteger()); - case TIMESTAMP: - return StoneDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); - case VARCHAR: - return StoneDBConstant.createTextConstant(globalState.getRandomly().getString()); - case DOUBLE: - return StoneDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - default: - throw new IgnoreMeException(); - } - } - - public Node generateConstant(StoneDBDataType dataType, boolean isNullable) { - if (isNullable && Randomly.getBooleanWithSmallProbability()) { - generateConstant(StoneDBDataType.NULL); - } - return generateConstant(dataType); - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - if (allowAggregates && Randomly.getBoolean()) { - StoneDBAggregateFunction aggregateFunction = StoneDBAggregateFunction.getRandom(); - allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs(), depth + 1), - aggregateFunction); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - StoneDBAdvancedFunc advancedFunction = StoneDBAdvancedFunc.getRandom(); - return new NewFunctionNode<>(generateExpressions(advancedFunction.getNrArgs(), depth + 1), - advancedFunction); - } - List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - Expression expr = Randomly.fromList(possibleOptions); - Operator op; - switch (expr) { - case UNARY_PREFIX: - op = StoneDBUnaryPrefixOperator.getRandom(); - return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), op); - case UNARY_POSTFIX: - if (!bug1942) { - op = StoneDBUnaryPostfixOperator.getRandom(); - } else if (!bugNotReported3) { - op = StoneDBUnaryPostfixOperator.IS_NULL; - } else { - throw new IgnoreMeException(); - } - return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), op); - case BINARY_COMPARISON: - if (!bugNotReported6) { - op = StoneDBBinaryComparisonOperator.getRandom(); - } else { - throw new IgnoreMeException(); - } - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case IN: - return new NewInOperatorNode<>(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), false); - case NOT_IN: - return new NewInOperatorNode<>(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), true); - case BINARY_LOGICAL: - if (bugNotReportedXOR) { - op = Randomly.fromList(Arrays.stream(StoneDBBinaryLogicalOperator.values()) - .filter(p -> p != StoneDBBinaryLogicalOperator.XOR).collect(Collectors.toList())); - } else { - op = StoneDBBinaryLogicalOperator.getRandom(); - } - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case BINARY_ARITHMETIC: - op = StoneDBBinaryArithmeticOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case BINARY_BITWISE: - op = StoneDBBinaryBitwiseOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case BETWEEN: - return new NewBetweenOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), - generateExpression(depth + 1), Randomly.getBoolean()); - case CASE: - int nr = Randomly.smallNumber() + 1; - return new NewCaseOperatorNode<>(generateExpression(depth + 1), generateExpressions(nr, depth + 1), - generateExpressions(nr, depth + 1), generateExpression(depth + 1)); - default: - throw new AssertionError(); - } - } - - @Override - protected Node generateColumn() { - StoneDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode<>(column); - } - - protected List> generateColumns() { - int size = globalState.getRandomly().getInteger(1, columns.size()); - Set> set = new HashSet<>(); - while (set.size() < size) { - set.add(generateColumn()); - } - return new ArrayList<>(set); - } - - public enum StoneDBUnaryPrefixOperator implements Operator { - - NOT("NOT"), PLUS("+"), MINUS("-"), INVERSION("!"); - - private final String textRepr; - - StoneDBUnaryPrefixOperator(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - public static StoneDBUnaryPrefixOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum StoneDBUnaryPostfixOperator implements Operator { - - IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); - - private final String textRepr; - - StoneDBUnaryPostfixOperator(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - public static StoneDBUnaryPostfixOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - /* - * Comparison operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/comparison-operators - */ - public enum StoneDBBinaryComparisonOperator implements Operator { - EQUAL("="), GREATER(">"), LESS("<"), GREATER_EQUAL(">="), LESS_EQUAL("<="), - NOT_EQUALS(Randomly.fromList(Arrays.asList("!=", "<>"))), NULL_SAFE_EQUAL("<=>"), LIKE("LIKE"); - - private final String textRepr; - - StoneDBBinaryComparisonOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - } - - /* - * Logical operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/logical-operators - */ - public enum StoneDBBinaryLogicalOperator implements Operator { - - AND("AND"), OR("OR"), XOR("XOR"); - - private final String textRepr; - - StoneDBBinaryLogicalOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - /* - * Arithmetic operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/arithmetic-operators - */ - public enum StoneDBBinaryArithmeticOperator implements Operator { - ADDITION("+"), MINUS("-"), MULTIPLICATION("*"), DIVISION(Randomly.fromList(Arrays.asList("/", " div "))), - MODULO(Randomly.fromList(Arrays.asList("%", " mod "))); - - private final String textRepr; - - StoneDBBinaryArithmeticOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - /* - * Bitwise operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/bitwise-operators - */ - public enum StoneDBBinaryBitwiseOperator implements Operator { - AND("&"), OR("|"), XOR("^"), LEFTSHIFT("<<"), RIGHTSHIFT(">>"); - - private final String textRepr; - - StoneDBBinaryBitwiseOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - public NewFunctionNode generateAggregateAndArgs( - StoneDBAggregateFunction aggregateFunction) { - return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); - } - - public Node generateAggregate() { - StoneDBAggregateFunction aggrFunc = StoneDBAggregateFunction.getRandom(); - return generateAggregateAndArgs(aggrFunc); - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java deleted file mode 100644 index 82eab3413..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java +++ /dev/null @@ -1,121 +0,0 @@ -package sqlancer.stonedb.gen; - -import sqlancer.Randomly; -import sqlancer.Randomly.StringGenerationStrategy; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; - -public class StoneDBIndexCreateGenerator { - private final StoneDBGlobalState globalState; - private final Randomly r; - // which table to add index - StoneDBTable table; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBIndexCreateGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - r = globalState.getRandomly(); - table = globalState.getSchema().getRandomTable(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBIndexCreateGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("CREATE "); - // Tianmu engine does not support fulltext index. - sb.append(Randomly.fromOptions("UNIQUE" /* "FULLTEXT" */, "SPATIAL")); - sb.append(" INDEX "); - sb.append(globalState.getSchema().getFreeIndexName()); - if (Randomly.getBoolean()) { - appendIndexType(); - } - sb.append(" ON "); - sb.append(table.getName()); - appendKeyParts(); - appendIndexOption(); - appendAlgoOrLockOption(); - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void addExpectedErrors() { - // java.sql.SQLException: Tianmu engine does not support unique index. - errors.add("Tianmu engine does not support unique index"); - // java.sql.SQLException: Tianmu engine does not support fulltext index. - errors.add("Tianmu engine does not support fulltext index"); - // java.sql.SQLException: The used table type doesn't support SPATIAL indexes - errors.add("The used table type doesn't support SPATIAL indexes"); - // java.sql.SQLException: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY. - errors.add("ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY."); - // java.sql.SQLSyntaxErrorException: Key column 'c0' doesn't exist in table - errors.add("doesn't exist in table"); - // java.sql.SQLSyntaxErrorException: A SPATIAL index may only contain a geometrical type column - errors.add("A SPATIAL index may only contain a geometrical type column"); - } - - private void appendIndexType() { - // The index_type clause cannot be used for FULLTEXT INDEX or SPATIAL INDEX specifications. - if (sb.toString().contains("FULLTEXT") || sb.toString().contains("SPATIAL")) { - return; - } - if (Randomly.getBoolean()) { - return; - } - sb.append(" USING "); - sb.append(Randomly.fromOptions("BTREE", "HASH")); - } - - private void appendKeyParts() { - int numberOfKeyParts = Randomly.fromOptions(1, 1, 1, 1, table.getColumns().size()); - for (int i = 0; i < numberOfKeyParts; i++) { - appendKeyPart(); - } - } - - private void appendKeyPart() { - sb.append("("); - StoneDBColumn randomColumn = table.getRandomColumn(); - sb.append(randomColumn.getName()); - if (Randomly.getBoolean()) { - sb.append(" (").append(Randomly.smallNumber()).append(")"); - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions(" ASC", " DESC")); - } - sb.append(")"); - } - - private void appendIndexOption() { - if (Randomly.getBoolean()) { - return; - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions("KEY_BLOCK_SIZE ", "KEY_BLOCK_SIZE = ")); - sb.append(r.getInteger(1, Math.max(1, Randomly.smallNumber()))); - sb.append(" "); - } - if (Randomly.getBoolean()) { - StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; - sb.append(String.format("COMMENT '%s' ", strategy.getString(r))); - } - } - - private void appendAlgoOrLockOption() { - if (Randomly.getBoolean()) { - return; - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); - sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); - } else { - sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); - sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java deleted file mode 100644 index dda72c735..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java +++ /dev/null @@ -1,47 +0,0 @@ -package sqlancer.stonedb.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; - -public class StoneDBIndexDropGenerator { - // which table to drop index - StoneDBSchema.StoneDBTable table; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBIndexDropGenerator(StoneDBGlobalState globalState) { - table = globalState.getSchema().getRandomTable(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBIndexDropGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - if (!table.hasIndexes()) { - return null; - } - sb.append("DROP INDEX "); - sb.append(table.getRandomIndex().getIndexName()); - sb.append(" ON "); - sb.append(table.getName()); - appendAlgoOrLockOption(); - return new SQLQueryAdapter(sb.toString(), errors); - } - - private void appendAlgoOrLockOption() { - if (Randomly.getBoolean()) { - return; - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); - sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); - } else { - sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); - sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java deleted file mode 100644 index c37024669..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ /dev/null @@ -1,141 +0,0 @@ -package sqlancer.stonedb.gen; - -import java.util.regex.Pattern; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; - -public class StoneDBTableAlterGenerator { - private final StoneDBGlobalState globalState; - private final StringBuilder sb = new StringBuilder(); - private final StoneDBTable table; - ExpectedErrors errors = new ExpectedErrors(); - - enum Action { - ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN, CHANGE_COLUMN - } - - public StoneDBTableAlterGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - table = globalState.getSchema().getRandomTable(t -> !t.isView()); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBTableAlterGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("ALTER TABLE "); - sb.append(table.getName()); - sb.append(" "); - appendAlterOption(Randomly.fromOptions(Action.values())); - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void addExpectedErrors() { - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Data too long for column 'c0' at row 2 - errors.addRegex(Pattern.compile("Data truncation: Data too long for column 'c\\d{1,3}' at row \\d{1,3}")); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: - // '0.571272522740968' for column 'c1' at row 1 - errors.add("Incorrect datetime value: "); - // java.sql.SQLSyntaxErrorException: Invalid default value for 'c0' - errors.add("Invalid default value for "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row - // 2 - errors.add("Data truncation: Out of range value for column "); - // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes - errors.add("Specified key was too long; max key length is 3072 bytes"); - // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead - errors.add("You can't delete all columns with ALTER TABLE; use DROP TABLE instead"); - // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 't1' - errors.addRegex(Pattern.compile("Unknown column 'c\\d{1,3}' in 't\\d{1,3}'")); - // java.sql.SQLSyntaxErrorException: BLOB, TEXT, GEOMETRY or JSON column 'c0' can't have a default value - errors.addRegex(Pattern.compile("BLOB, TEXT, GEOMETRY or JSON column 'c\\d{1,3}' can't have a default value")); - // java.sql.SQLSyntaxErrorException: Column length too big for column 'c91' (max = 16383); use BLOB or TEXT - // instead - errors.addRegex(Pattern - .compile("Column length too big for column 'c\\d{1,3}' \\(max = 16383\\); use BLOB or TEXT instead")); - } - - private void appendAlterOption(Action action) { - StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) - .setColumns(table.getColumns()); - if (globalState.getDbmsSpecificOptions().test80Version && Randomly.getBooleanWithSmallProbability()) { - sb.append("RENAME COLUMN "); - sb.append(table.getRandomColumn().getName()); - sb.append(" TO "); - sb.append(table.getFreeColumnName()); - } - switch (action) { - case ADD_COLUMN: - sb.append("ADD COLUMN "); - String columnName = table.getFreeColumnName(); - sb.append(" ").append(columnName).append(" "); - sb.append( - StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); - // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT - // instead - errors.addRegex(Pattern - .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); - if (Randomly.getBoolean()) { - if (Randomly.getBoolean()) { - sb.append(" FIRST"); - } else { - sb.append(" AFTER "); - sb.append(table.getRandomColumn().getName()); - } - } - break; - case DROP_COLUMN: - sb.append(Randomly.fromOptions("DROP COLUMN ", "DROP ")); - sb.append(table.getRandomColumn().getName()); - break; - case ALTER_COLUMN: - sb.append(Randomly.fromOptions("ALTER COLUMN ", "ALTER ")); - StoneDBColumn randomColumn = table.getRandomColumn(); - sb.append(randomColumn.getName()); - if (Randomly.getBoolean()) { - sb.append(" SET DEFAULT ").append(generator - .generateConstant(randomColumn.getType().getPrimitiveDataType(), Randomly.getBoolean())); - } else { - sb.append(" DROP DEFAULT"); - } - break; - case CHANGE_COLUMN: - sb.append(Randomly.fromOptions("CHANGE COLUMN ", "CHANGE ")); - String oldColumnName = table.getRandomColumn().getName(); - String newColumnName = table.getFreeColumnName(); - sb.append(oldColumnName).append(" ").append(newColumnName).append(" "); - errors.add("Incorrect integer value: "); - // java.sql.SQLException: Data truncated for column 'c1' at row 1 - errors.add("Data truncated for column "); - sb.append( - StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); - // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT - // instead - errors.addRegex(Pattern - .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); - // java.sql.SQLSyntaxErrorException: BLOB column 'c1' can't be used in key specification with the used table - // type - errors.addRegex(Pattern - .compile("BLOB column 'c\\d{1,3}' can't be used in key specification with the used table type")); - if (Randomly.getBoolean()) { - if (Randomly.getBoolean()) { - sb.append(" FIRST"); - } else { - sb.append(" AFTER "); - sb.append(table.getRandomColumn().getName()); - } - } - break; - default: - throw new AssertionError(action); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java deleted file mode 100644 index c288c57fd..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java +++ /dev/null @@ -1,266 +0,0 @@ -package sqlancer.stonedb.gen; - -import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.PRIMARY_KEY; -import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.UNIQUE_KEY; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.regex.Pattern; - -import sqlancer.Randomly; -import sqlancer.Randomly.StringGenerationStrategy; -import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; - -public class StoneDBTableCreateGenerator { - // the name of the table to create - private final String tableName; - private final StoneDBSchema schema; - // the name of the columns in the table - private final List columns = new ArrayList<>(); - private final boolean allowPrimaryKey; - private boolean setPrimaryKey; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - private final Randomly r; - - public StoneDBTableCreateGenerator(StoneDBGlobalState globalState, String tableName) { - this.tableName = tableName; - this.schema = globalState.getSchema(); - allowPrimaryKey = Randomly.getBoolean(); - this.r = globalState.getRandomly(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String tableName) { - return new StoneDBTableCreateGenerator(globalState, tableName).getQuery(); - } - - public SQLQueryAdapter getQuery() { - sb.append(Randomly.fromOptions("CREATE TABLE "/* , "CREATE TEMPORARY TABLE " */)); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(tableName); - // ues link statement - if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { - sb.append(" LIKE "); - sb.append(schema.getRandomTable().getName()); - } else { - appendColumns(); - if (Randomly.getBoolean()) { - sb.append(" "); - appendTableOptions(); - } - } - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void addExpectedErrors() { - // java.sql.SQLSyntaxErrorException: BLOB/TEXT column 'c0' used in key specification without a key length - errors.add("used in key specification without a key length"); - // java.sql.SQLException: Tianmu engine does not support unique index. - errors.add("Tianmu engine does not support unique index"); - // java.sql.SQLException: BLOB column 'c0' can't be used in key specification with the used table type - errors.add("can't be used in key specification with the used table type"); - // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes - errors.add("Specified key was too long; max key length is 3072 bytes"); - // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT - // instead - errors.add("Column length too big for column"); - // BLOB/TEXT column 'c0' used in key specification without a key length - errors.addRegex(Pattern.compile("BLOB/TEXT column 'c.*' used in key specification without a key length")); - } - - private enum TableOptions { - AUTO_INCREMENT, AVG_ROW_LENGTH, CHECKSUM, COMPRESSION, DELAY_KEY_WRITE, /* ENCRYPTION, */ INSERT_METHOD, - KEY_BLOCK_SIZE, MAX_ROWS, MIN_ROWS, PACK_KEYS, STATS_AUTO_RECALC, STATS_PERSISTENT, STATS_SAMPLE_PAGES; - - public static List getRandomTableOptions() { - List options; - // try to ensure that usually, only a few of these options are generated - if (Randomly.getBooleanWithSmallProbability()) { - options = Randomly.subset(TableOptions.values()); - } else { - if (Randomly.getBoolean()) { - options = Collections.emptyList(); - } else { - options = Randomly.nonEmptySubset(Arrays.asList(TableOptions.values()), Randomly.smallNumber()); - } - } - return options; - } - } - - private void appendTableOptions() { - List tableOptions = TableOptions.getRandomTableOptions(); - int i = 0; - for (TableOptions o : tableOptions) { - if (i++ != 0) { - sb.append(", "); - } - switch (o) { - case AUTO_INCREMENT: - sb.append("AUTO_INCREMENT = "); - sb.append(r.getPositiveInteger()); - break; - // The valid range for avg_row_length is [0,4294967295] - case AVG_ROW_LENGTH: - sb.append("AVG_ROW_LENGTH = "); - sb.append(r.getLong(0, 4294967295L + 1)); - break; - case CHECKSUM: - sb.append("CHECKSUM = 1"); - break; - case COMPRESSION: - sb.append("COMPRESSION = '"); - sb.append(Randomly.fromOptions("ZLIB", "LZ4", "NONE")); - sb.append("'"); - break; - case DELAY_KEY_WRITE: - sb.append("DELAY_KEY_WRITE = "); - sb.append(Randomly.fromOptions(0, 1)); - break; - case INSERT_METHOD: - sb.append("INSERT_METHOD = "); - sb.append(Randomly.fromOptions("NO", "FIRST", "LAST")); - break; - // The valid range for key_block_size is [0,65535] - case KEY_BLOCK_SIZE: - sb.append("KEY_BLOCK_SIZE = "); - sb.append(r.getInteger(0, 65535 + 1)); - break; - case MAX_ROWS: - sb.append("MAX_ROWS = "); - sb.append(r.getLong(0, Long.MAX_VALUE)); - break; - case MIN_ROWS: - sb.append("MIN_ROWS = "); - sb.append(r.getLong(1, Long.MAX_VALUE)); - break; - case PACK_KEYS: - sb.append("PACK_KEYS = "); - sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); - break; - case STATS_AUTO_RECALC: - sb.append("STATS_AUTO_RECALC = "); - sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); - break; - case STATS_PERSISTENT: - sb.append("STATS_PERSISTENT = "); - sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); - break; - case STATS_SAMPLE_PAGES: - sb.append("STATS_SAMPLE_PAGES = "); - sb.append(r.getInteger(1, Short.MAX_VALUE)); - break; - default: - throw new AssertionError(o); - } - } - } - - private void appendColumns() { - sb.append("("); - for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { - if (i != 0) { - sb.append(", "); - } - appendColumn(i); - } - sb.append(")"); - } - - private void appendColumn(int columnId) { - String columnName = DBMSCommon.createColumnName(columnId); - columns.add(columnName); - sb.append(columnName); - appendColumnDefinition(); - } - - private void appendColumnDefinition() { - sb.append(" "); - StoneDBDataType randomType = StoneDBDataType.getRandomWithoutNull(); - appendType(randomType); - appendColumnOption(randomType); - } - - protected enum ColumnOptions { - NULL_OR_NOT_NULL, PRIMARY_KEY, UNIQUE_KEY, COMMENT, COLUMN_FORMAT, STORAGE - } - - private void appendColumnOption(StoneDBDataType type) { - boolean isTextType = type == StoneDBDataType.VARCHAR; - boolean isNull = false; - boolean columnHasPrimaryKey = false; - List columnOptions = Randomly.subset(ColumnOptions.values()); - // boolean tableHasNullableColumn; - // if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { - // tableHasNullableColumn = true; - // } - // only use one key, unique key or primary key, but not both - if (columnOptions.contains(PRIMARY_KEY) && columnOptions.contains(UNIQUE_KEY)) { - columnOptions.remove(Randomly.fromOptions(PRIMARY_KEY, UNIQUE_KEY)); - } - if (isTextType) { - // TODO: restriction due to the limited key length - columnOptions.remove(PRIMARY_KEY); - columnOptions.remove(UNIQUE_KEY); - } - for (ColumnOptions o : columnOptions) { - sb.append(" "); - switch (o) { - case NULL_OR_NOT_NULL: - // PRIMARY KEYs cannot be NULL - if (!columnHasPrimaryKey) { - if (Randomly.getBoolean()) { - sb.append("NULL"); - } - // tableHasNullableColumn = true; - isNull = true; - } else { - sb.append("NOT NULL"); - } - break; - case UNIQUE_KEY: - sb.append("UNIQUE"); - if (Randomly.getBoolean()) { - sb.append(" KEY"); - } - break; - case COMMENT: - StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; - sb.append(String.format("COMMENT '%s' ", strategy.getString(r))); - break; - case COLUMN_FORMAT: - sb.append("COLUMN_FORMAT "); - sb.append(Randomly.fromOptions("FIXED", "DYNAMIC", "DEFAULT")); - break; - case STORAGE: - sb.append("STORAGE "); - sb.append(Randomly.fromOptions("DISK", "MEMORY")); - break; - case PRIMARY_KEY: - // PRIMARY KEYs cannot be NULL - if (allowPrimaryKey && !setPrimaryKey && !isNull) { - sb.append("PRIMARY KEY"); - setPrimaryKey = true; - columnHasPrimaryKey = true; - } - break; - default: - throw new AssertionError(); - } - } - } - - private void appendType(StoneDBDataType randomType) { - sb.append(StoneDBDataType.getTypeAndValue(randomType, r)); - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java deleted file mode 100644 index 0d8446d4b..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ /dev/null @@ -1,70 +0,0 @@ -package sqlancer.stonedb.gen; - -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.stonedb.StoneDBBugs; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public final class StoneDBTableDeleteGenerator { - private final StoneDBGlobalState globalState; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - Randomly r; - - private StoneDBTableDeleteGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - r = globalState.getRandomly(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBTableDeleteGenerator(globalState).getQuery(); - } - - public SQLQueryAdapter getQuery() { - StoneDBTable randomTable = globalState.getSchema().getRandomTable(); - sb.append("DELETE"); - if (Randomly.getBoolean()) { - sb.append(" LOW_PRIORITY"); - } - if (Randomly.getBoolean()) { - sb.append(" QUICK"); - } - if (Randomly.getBoolean()) { - sb.append(" IGNORE"); - } - sb.append(" FROM "); - sb.append(randomTable.getName()); - if (globalState.getDbmsSpecificOptions().test80Version) { - sb.append(" AS "); - sb.append(globalState.getSchema().getFreeTableName()); - } - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) - .setColumns(randomTable.getColumns()).generateExpression())); - } - if (!StoneDBBugs.bug1933 && Randomly.getBoolean()) { - sb.append(" ORDER BY "); - sb.append(String.join(", ", Randomly.fromOptions( - randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))) - .replace('[', '(').replace(']', ')')); - } - if (Randomly.getBoolean()) { - sb.append(" LIMIT "); - sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); - } - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors); - } - - private void addExpectedErrors() { - // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 - errors.add("Incorrect string value: "); - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java deleted file mode 100644 index 0a70c8467..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ /dev/null @@ -1,142 +0,0 @@ -package sqlancer.stonedb.gen; - -import static sqlancer.stonedb.StoneDBBugs.bugNotReported4; -import static sqlancer.stonedb.StoneDBBugs.bugNotReported5; - -import java.util.List; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBTableInsertGenerator extends AbstractInsertGenerator { - private final StoneDBGlobalState globalState; - // which table to insert into - private final StoneDBTable table; - // which subset columns of the table to add values - private final List columns; - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBTableInsertGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - table = globalState.getSchema().getRandomTable(); - columns = table.getRandomNonEmptyColumnSubset(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBTableInsertGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("INSERT"); - if (Randomly.getBoolean()) { - sb.append(" "); - sb.append(Randomly.fromOptions("LOW_PRIORITY", "DELAYED", "HIGH_PRIORITY")); - } - if (!bugNotReported4 && Randomly.getBoolean()) { - sb.append(" IGNORE"); - } - sb.append(" INTO "); - sb.append(table.getName()); - appendPartition(); - appendColumnsAndValues(columns); - appendAS(); - appendOnDuplicateUpdate(); - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors); - } - - private void addExpectedErrors() { - // java.sql.SQLException: Incorrect DATE value: '292278994-08-17' - errors.add("Incorrect DATE value: '"); - // java.sql.SQLIntegrityConstraintViolationException: Duplicate entry '1970-01-14' for key 'PRIMARY' - errors.add("Duplicate entry "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row - errors.add("Data truncation: Out of range value for column "); - // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 'field list' - errors.add("Unknown column "); - // java.sql.SQLException: Insert duplicate key on row: 4, pk: 138609795916627968 - errors.add("Insert duplicate key on row: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: - errors.add("Data truncation: Incorrect datetime value: "); - // java.sql.SQLException: Field 'c0' doesn't have a default value - errors.add("doesn't have a default value"); - // java.sql.SQLException: Data truncated for column 'c0' at row 1 - errors.addRegex(Pattern.compile("Data truncated for column 'c.*' at row .*")); - } - - private void appendPartition() { - - } - - private void appendColumnsAndValues(List columns) { - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(Randomly.fromOptions(" VALUES ", " VALUE ")); - appendValues(); - } - - private void appendAS() { - if (!globalState.getDbmsSpecificOptions().test80Version) { - return; - } - sb.append(" AS "); - sb.append("r").append(table.getNrRows(globalState)); - } - - private void appendOnDuplicateUpdate() { - sb.append("on duplicate key update "); - StoneDBColumn randomColumn = table.getRandomColumn(); - sb.append(randomColumn.getName()); - sb.append("="); - insertValue(randomColumn); - } - - // append nrRows rows - private void appendValues() { - int nrRows; - if (Randomly.getBoolean()) { - nrRows = 1; - } else { - nrRows = 1 + Randomly.smallNumber(); - } - for (int row = 0; row < nrRows; row++) { - appendOneValue(row); - } - } - - // append all columns of one row - private void appendOneValue(int nrRow) { - if (nrRow != 0) { - sb.append(", "); - } - sb.append("("); - for (int c = 0; c < columns.size(); c++) { - if (c != 0) { - sb.append(", "); - } - insertValue(columns.get(c)); - } - sb.append(")"); - } - - // append one column of one row - @Override - protected void insertValue(StoneDBColumn column) { - if (!bugNotReported5 && Randomly.getBooleanWithRatherLowProbability()) { - sb.append("DEFAULT"); - } else { - sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) - .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java deleted file mode 100644 index 259136135..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java +++ /dev/null @@ -1,71 +0,0 @@ -package sqlancer.stonedb.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBTableUpdateGenerator { - private final StoneDBGlobalState globalState; - // which table to insert into - private final StoneDBTable table; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBTableUpdateGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - table = globalState.getSchema().getRandomTable(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBTableUpdateGenerator(globalState).getQuery(); - } - - public SQLQueryAdapter getQuery() { - sb.append("UPDATE"); - sb.append(Randomly.fromOptions(" ", " LOW_PRIORITY ", " LOW_PRIORITY IGNORE ", " IGNORE ")); - sb.append(table.getName()); - sb.append("SET "); - appendAssignmentList(); - if (Randomly.getBoolean()) { - appendWhereCondition(); - } - if (Randomly.getBoolean()) { - appendOrderBy(); - } - if (Randomly.getBoolean()) { - appendLimit(); - } - return new SQLQueryAdapter(sb.toString(), errors); - } - - public void appendAssignmentList() { - for (int i = 0; i < new Randomly().getInteger(1, table.getColumns().size()); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(table.getColumns().get(i)); - sb.append(" = "); - sb.append(Randomly.fromOptions( - StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType.getRandomValue( - table.getColumns().get(i).getType().getPrimitiveDataType(), globalState.getRandomly())), - "DEFAULT")); - } - } - - private void appendWhereCondition() { - } - - private void appendOrderBy() { - sb.append(" ORDER BY "); - sb.append(table.getRandomColumn().getName()); - } - - private void appendLimit() { - sb.append(" LIMIT "); - sb.append(new Randomly().getInteger(0, (int) table.getNrRows(globalState))); - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java deleted file mode 100644 index d4ada0332..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java +++ /dev/null @@ -1,78 +0,0 @@ -package sqlancer.stonedb.gen; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBSchema.StoneDBTables; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; - -public class StoneDBViewCreateGenerator { - // the name of the view to create - private final String viewName; - private final StoneDBGlobalState globalState; - private StoneDBSelect select; - private final StringBuilder sb = new StringBuilder(); - - public StoneDBViewCreateGenerator(StoneDBGlobalState globalState, String viewName) { - this.globalState = globalState; - this.viewName = viewName; - setSelect(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String viewName) { - return new StoneDBViewCreateGenerator(globalState, viewName).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("CREATE "); - sb.append("VIEW "); - sb.append(viewName); - sb.append(" AS "); - sb.append(StoneDBToStringVisitor.asString(select)); - return new SQLQueryAdapter(sb.toString(), true); - } - - private void setSelect() { - StoneDBSchema schema = globalState.getSchema(); - StoneDBTables targetTables = schema.getRandomTableNonEmptyTables(); - StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(globalState) - .setColumns(targetTables.getColumns()); - - select = new StoneDBSelect(); - select.setFetchColumns(generateFetchColumns(targetTables)); - List tables = targetTables.getTables(); - List> tableReferenceNodeList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = StoneDBJoin.getJoins(tableReferenceNodeList, globalState); - select.setJoinList(new ArrayList<>(joins)); - select.setFromList(new ArrayList<>(tableReferenceNodeList)); - select.setWhereClause(gen.generateExpression()); - select.setOrderByClauses(gen.generateOrderBys()); - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - } - - private List> generateFetchColumns(StoneDBTables tables) { - List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))); - } else { - columns = Randomly.nonEmptySubset(tables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)) - .collect(Collectors.toList()); - } - return columns; - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java deleted file mode 100644 index eb1482d7f..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ /dev/null @@ -1,170 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.stonedb.StoneDBErrors; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPostfixOperator; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPrefixOperator; - -public class StoneDBAggregateOracle extends StoneDBQueryPartitioningBase { - - public StoneDBAggregateOracle(StoneDBGlobalState state) { - super(state); - StoneDBErrors.addExpectedExpressionErrors(state, errors); - } - - @Override - public void check() throws Exception { - super.check(); - - StoneDBAggregateFunction aggregateFunction = Randomly.fromOptions(StoneDBAggregateFunction.values()); - NewFunctionNode aggregate = gen - .generateAggregateAndArgs(aggregateFunction); - - List> fetchColumns = new ArrayList<>(); - fetchColumns.add(aggregate); - select.setFetchColumns(fetchColumns); - - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - - String originalQuery = StoneDBToStringVisitor.asString(select); - String originalResult = getAggregateResult(originalQuery); - - String metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); - String metamorphicResult = getAggregateResult(metamorphicQuery); - - String line1 = "--" + originalQuery + ";"; - String line2 = "--" + originalResult + ";"; - String line3 = "--" + originalResult + ";"; - String line4 = "--" + metamorphicResult + ";"; - String output = String.join(System.lineSeparator(), line1, line2, line3, line4); - state.getState().getLocalState().log(output); - - if (originalResult == null && metamorphicResult != null - || originalResult != null && (!originalResult.contentEquals(metamorphicResult) - && !ComparatorHelper.isEqualDouble(originalResult, metamorphicResult))) { - throw new AssertionError("aggregate result mismatch!" + System.lineSeparator() + output); - } - } - - private String createMetamorphicUnionQuery(StoneDBSelect select, - NewFunctionNode aggregate, - List> from) { - String metamorphicQuery; - Node whereClause = gen.generateExpression(); - - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, - StoneDBUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, - StoneDBUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); - StoneDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); - StoneDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); - StoneDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; - metamorphicQuery += StoneDBToStringVisitor.asString(leftSelect) + " UNION ALL " - + StoneDBToStringVisitor.asString(middleSelect) + " UNION ALL " - + StoneDBToStringVisitor.asString(rightSelect); - metamorphicQuery += ") as result"; - return metamorphicQuery; - } - - private String getAggregateResult(String queryString) throws SQLException { - String resultString; - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - try (SQLancerResultSet result = q.executeAndGet(state)) { - if (result == null) { - throw new IgnoreMeException(); - } - if (!result.next()) { - resultString = null; - } else { - resultString = result.getString(1); - } - return resultString; - } catch (SQLException e) { - if (!e.getMessage().contains("Not implemented type")) { - throw new AssertionError(queryString, e); - } else { - throw new IgnoreMeException(); - } - } - } - - private List> mapped( - NewFunctionNode aggregate) { - - StoneDBCastOperation count; - switch (aggregate.getFunc()) { - case COUNT: - case MAX: - case MIN: - case SUM: - return aliasArgs(List.of(aggregate)); - case AVG: - NewFunctionNode sum = new NewFunctionNode<>( - aggregate.getArgs(), StoneDBAggregateFunction.SUM); - count = new StoneDBCastOperation(new NewFunctionNode<>(aggregate.getArgs(), StoneDBAggregateFunction.COUNT), - new StoneDBCompositeDataType(StoneDBDataType.DECIMAL).getPrimitiveDataType()); - return aliasArgs(Arrays.asList(sum, count)); - default: - throw new AssertionError(aggregate.getFunc()); - } - } - - private List> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); - int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode(expr, "agg" + i++)); - } - return args; - } - - private String getOuterAggregateFunction(NewFunctionNode aggregate) { - switch (aggregate.getFunc()) { - case AVG: - return "SUM(agg0)/SUM(agg1)"; - case COUNT: - return StoneDBAggregateFunction.SUM + "(agg0)"; - default: - return aggregate.getFunc().toString() + "(agg0)"; - } - } - - private StoneDBSelect getSelect(List> aggregates, List> fromList, - Node whereClause, List> joinList) { - StoneDBSelect select = new StoneDBSelect(); - select.setFetchColumns(aggregates); - select.setFromList(fromList); - select.setWhereClause(whereClause); - select.setJoinList(joinList); - if (Randomly.getBooleanWithSmallProbability()) { - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - } - return select; - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java deleted file mode 100644 index ebf1f9967..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java +++ /dev/null @@ -1,67 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBErrors; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBSchema.StoneDBTables; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator; - -public class StoneDBFuzzOracle implements TestOracle { - private final StoneDBGlobalState globalState; - private final StoneDBSchema schema; - private final ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBFuzzOracle(StoneDBGlobalState globalState) { - this.globalState = globalState; - this.schema = globalState.getSchema(); - StoneDBErrors.addExpectedExpressionErrors(globalState, errors); - } - - @Override - public void check() throws Exception { - StoneDBTables randomTables = schema.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(globalState).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); - List> joins = StoneDBJoin.getJoins(tableList, globalState); - - StoneDBSelect select = new StoneDBSelect(); - select.setFetchColumns( - Randomly.fromOptions(List.of(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))), - allColumns, Randomly.subset(allColumns))); - select.setFromList(new ArrayList<>(tableList)); - select.setJoinList(joins); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses( - new StoneDBExpressionGenerator(globalState).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - - String queryString = StoneDBToStringVisitor.asString(select); - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - q.executeAndGetLogged(globalState); - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java deleted file mode 100644 index d6efe30d1..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ /dev/null @@ -1,134 +0,0 @@ -package sqlancer.stonedb.oracle; - -import static sqlancer.stonedb.StoneDBBugs.bug1953; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.common.oracle.NoRECBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.stonedb.StoneDBErrors; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBSchema.StoneDBTables; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBBinaryLogicalOperator; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; - -public class StoneDBNoRECOracle extends NoRECBase implements TestOracle { - - private final StoneDBSchema schema; - - public StoneDBNoRECOracle(StoneDBGlobalState globalState) { - super(globalState); - this.schema = globalState.getSchema(); - StoneDBErrors.addExpectedExpressionErrors(globalState, errors); - } - - @Override - public void check() throws Exception { - StoneDBTables randomTables = schema.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = StoneDBJoin.getJoins(tableList, state); - // get and check count - int secondCount = getUnoptimizedQueryCount(new ArrayList<>(tableList), randomWhereCondition, joins); - int firstCount = getOptimizedQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError(optimizedQueryString + "; -- " + firstCount + System.lineSeparator() - + unoptimizedQueryString + " -- " + secondCount); - } - } - - private int getUnoptimizedQueryCount(List> tableList, - Node randomWhereCondition, List> joins) throws SQLException { - StoneDBSelect select = new StoneDBSelect(); - Node asText = new NewPostfixTextNode<>( - new StoneDBCastOperation( - new NewBinaryOperatorNode<>(new NewPostfixTextNode<>(randomWhereCondition, " IS NOT NULL "), - randomWhereCondition, StoneDBBinaryLogicalOperator.AND), - StoneDBDataType.INT), - " as count"); - select.setFetchColumns(List.of(asText)); - select.setFromList(tableList); - select.setJoinList(joins); - unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; - if (bug1953) { - unoptimizedQueryString = "SELECT * FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; - } - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError("error occurred when executing: \"" + unoptimizedQueryString + "\"", e); - } - if (rs == null) { - return -1; - } - int secondCount = 0; - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(SQLConnection con, List> tableList, - List columns, Node randomWhereCondition, - List> joins) { - StoneDBSelect select = new StoneDBSelect(); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new StoneDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = StoneDBToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java deleted file mode 100644 index 84d9f794f..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java +++ /dev/null @@ -1,71 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBSchema.StoneDBTables; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator; - -public class StoneDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, StoneDBGlobalState> - implements TestOracle { - - StoneDBSchema schema; - StoneDBTables targetTables; - StoneDBExpressionGenerator gen; - StoneDBSelect select; - - public StoneDBQueryPartitioningBase(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getRandomTableNonEmptyTables(); - gen = new StoneDBExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - - select = new StoneDBSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List> tableReferenceNodeList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = StoneDBJoin.getJoins(tableReferenceNodeList, state); - select.setJoinList(new ArrayList<>(joins)); - select.setFromList(new ArrayList<>(tableReferenceNodeList)); - select.setWhereClause(null); - } - - List> generateFetchColumns() { - List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))); - } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)) - .collect(Collectors.toList()); - } - return columns; - } - - @Override - protected ExpressionGenerator> getGen() { - return gen; - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java deleted file mode 100644 index cba8290b4..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBQueryPartitioningDistinctTester extends StoneDBQueryPartitioningBase { - public StoneDBQueryPartitioningDistinctTester(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - - select.setDistinct(true); - select.setWhereClause(null); - String originalQueryString = StoneDBToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - if (Randomly.getBoolean()) { - select.setDistinct(false); - } - - select.setWhereClause(predicate); - String firstQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = StoneDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, - secondQueryString, thirdQueryString, combinedString, true, state, errors); - - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java deleted file mode 100644 index d7674a71c..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java +++ /dev/null @@ -1,50 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBExpression; - -public class StoneDBQueryPartitioningGroupByTester extends StoneDBQueryPartitioningBase { - public StoneDBQueryPartitioningGroupByTester(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - // common to both original and combined query string - select.setGroupByExpressions(select.getFetchColumns()); - // specific to original query string - select.setWhereClause(null); - String originalQueryString = StoneDBToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - // specific to combined query string, will produce the same result as original query string in logic - select.setWhereClause(predicate); - String firstQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = StoneDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, - secondQueryString, thirdQueryString, combinedString, true, state, errors); - // compare the result - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } - - @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java deleted file mode 100644 index ce9dd6fc7..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java +++ /dev/null @@ -1,47 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBQueryPartitioningHavingTester extends StoneDBQueryPartitioningBase { - public StoneDBQueryPartitioningHavingTester(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - // common to both original and combined query string - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); - } - boolean orderBy = Randomly.getBoolean(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - // specific to original query string - select.setHavingClause(null); - String originalQueryString = StoneDBToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - // specific to combined query string, will produce the same result as original - // query string in logic - select.setHavingClause(predicate); - String firstQueryString = StoneDBToStringVisitor.asString(select); - select.setHavingClause(negatedPredicate); - String secondQueryString = StoneDBToStringVisitor.asString(select); - select.setHavingClause(isNullPredicate); - String thirdQueryString = StoneDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - // compare the result - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java deleted file mode 100644 index 4c9267f68..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBQueryPartitioningWhereTester extends StoneDBQueryPartitioningBase { - public StoneDBQueryPartitioningWhereTester(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - - select.setWhereClause(null); - String originalQueryString = StoneDBToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = StoneDBToStringVisitor.asString(select); - - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - // compare the result - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } -} diff --git a/test/sqlancer/dbms/TestStoneDBFuzz.java b/test/sqlancer/dbms/TestStoneDBFuzz.java deleted file mode 100644 index 2f5d52424..000000000 --- a/test/sqlancer/dbms/TestStoneDBFuzz.java +++ /dev/null @@ -1,21 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestStoneDBFuzz { - - String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); - boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); - - @Test - public void testStoneDB() { - assumeTrue(stoneDBIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", - "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "FUZZER")); - } - -} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestStoneDBNoRec.java b/test/sqlancer/dbms/TestStoneDBNoRec.java deleted file mode 100644 index 4b158f372..000000000 --- a/test/sqlancer/dbms/TestStoneDBNoRec.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestStoneDBNoRec { - - String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); - boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); - - @Test - public void testStoneDB() { - assumeTrue(stoneDBIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", - "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "NoREC")); - } - -} diff --git a/test/sqlancer/dbms/TestStoneDBTLP.java b/test/sqlancer/dbms/TestStoneDBTLP.java deleted file mode 100644 index d5db5b9ab..000000000 --- a/test/sqlancer/dbms/TestStoneDBTLP.java +++ /dev/null @@ -1,21 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestStoneDBTLP { - - String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); - boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); - - @Test - public void testStoneDB() { - assumeTrue(stoneDBIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", - "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "QUERY_PARTITIONING")); - } - -} From 5652c9db60349eda399275496ab57a39b3bda6b0 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 21 Aug 2024 15:47:20 +0800 Subject: [PATCH 0896/1284] Update new AST to support common test oracles --- .../common/ast/newast/NewAliasNode.java | 6 +++--- .../ast/newast/NewBetweenOperatorNode.java | 14 +++++++------- .../ast/newast/NewBinaryOperatorNode.java | 10 +++++----- .../ast/newast/NewCaseOperatorNode.java | 19 +++++++++---------- .../common/ast/newast/NewFunctionNode.java | 6 +++--- .../common/ast/newast/NewInOperatorNode.java | 10 +++++----- .../common/ast/newast/NewOrderingTerm.java | 6 +++--- .../common/ast/newast/NewPostfixTextNode.java | 6 +++--- .../common/ast/newast/NewTernaryNode.java | 14 +++++++------- .../common/ast/newast/NewToStringVisitor.java | 6 +++--- .../newast/NewUnaryPostfixOperatorNode.java | 7 +++---- .../newast/NewUnaryPrefixOperatorNode.java | 6 +++--- 12 files changed, 54 insertions(+), 56 deletions(-) diff --git a/src/sqlancer/common/ast/newast/NewAliasNode.java b/src/sqlancer/common/ast/newast/NewAliasNode.java index 9d7d84c43..9fbbb35d5 100644 --- a/src/sqlancer/common/ast/newast/NewAliasNode.java +++ b/src/sqlancer/common/ast/newast/NewAliasNode.java @@ -2,15 +2,15 @@ public class NewAliasNode implements Node { - private final Node expr; + private final E expr; private final String alias; - public NewAliasNode(Node expr, String alias) { + public NewAliasNode(E expr, String alias) { this.expr = expr; this.alias = alias; } - public Node getExpr() { + public E getExpr() { return expr; } diff --git a/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java index c5a0acf27..e47012267 100644 --- a/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java @@ -2,27 +2,27 @@ public class NewBetweenOperatorNode implements Node { - protected Node left; - protected Node middle; - protected Node right; + protected T left; + protected T middle; + protected T right; protected boolean isTrue; - public NewBetweenOperatorNode(Node left, Node middle, Node right, boolean isTrue) { + public NewBetweenOperatorNode(T left, T middle, T right, boolean isTrue) { this.left = left; this.middle = middle; this.right = right; this.isTrue = isTrue; } - public Node getLeft() { + public T getLeft() { return left; } - public Node getMiddle() { + public T getMiddle() { return middle; } - public Node getRight() { + public T getRight() { return right; } diff --git a/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java index 2b640ed0c..6f150499c 100644 --- a/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java @@ -5,10 +5,10 @@ public class NewBinaryOperatorNode implements Node { protected final Operator op; - protected final Node left; - protected final Node right; + protected final T left; + protected final T right; - public NewBinaryOperatorNode(Node left, Node right, Operator op) { + public NewBinaryOperatorNode(T left, T right, Operator op) { this.left = left; this.right = right; this.op = op; @@ -18,11 +18,11 @@ public String getOperatorRepresentation() { return op.getTextRepresentation(); } - public Node getLeft() { + public T getLeft() { return left; } - public Node getRight() { + public T getRight() { return right; } diff --git a/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java index 04f6be4c5..5ab2ed853 100644 --- a/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java @@ -4,13 +4,12 @@ public class NewCaseOperatorNode implements Node { - private final List> conditions; - private final List> expressions; - private final Node elseExpr; - private final Node switchCondition; + private final List conditions; + private final List expressions; + private final T elseExpr; + private final T switchCondition; - public NewCaseOperatorNode(Node switchCondition, List> conditions, List> expressions, - Node elseExpr) { + public NewCaseOperatorNode(T switchCondition, List conditions, List expressions, T elseExpr) { this.switchCondition = switchCondition; this.conditions = conditions; this.expressions = expressions; @@ -20,19 +19,19 @@ public NewCaseOperatorNode(Node switchCondition, List> conditions, Li } } - public Node getSwitchCondition() { + public T getSwitchCondition() { return switchCondition; } - public List> getConditions() { + public List getConditions() { return conditions; } - public List> getExpressions() { + public List getExpressions() { return expressions; } - public Node getElseExpr() { + public T getElseExpr() { return elseExpr; } diff --git a/src/sqlancer/common/ast/newast/NewFunctionNode.java b/src/sqlancer/common/ast/newast/NewFunctionNode.java index fae214713..baa5d1bc3 100644 --- a/src/sqlancer/common/ast/newast/NewFunctionNode.java +++ b/src/sqlancer/common/ast/newast/NewFunctionNode.java @@ -4,15 +4,15 @@ public class NewFunctionNode implements Node { - protected List> args; + protected List args; protected F func; - public NewFunctionNode(List> args, F func) { + public NewFunctionNode(List args, F func) { this.args = args; this.func = func; } - public List> getArgs() { + public List getArgs() { return args; } diff --git a/src/sqlancer/common/ast/newast/NewInOperatorNode.java b/src/sqlancer/common/ast/newast/NewInOperatorNode.java index 346156666..b4bcf535d 100644 --- a/src/sqlancer/common/ast/newast/NewInOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewInOperatorNode.java @@ -4,21 +4,21 @@ public class NewInOperatorNode implements Node { - private final Node left; - private final List> right; + private final T left; + private final List right; private final boolean isNegated; - public NewInOperatorNode(Node left, List> right, boolean isNegated) { + public NewInOperatorNode(T left, List right, boolean isNegated) { this.left = left; this.right = right; this.isNegated = isNegated; } - public Node getLeft() { + public T getLeft() { return left; } - public List> getRight() { + public List getRight() { return right; } diff --git a/src/sqlancer/common/ast/newast/NewOrderingTerm.java b/src/sqlancer/common/ast/newast/NewOrderingTerm.java index 5afe71821..be938fbb4 100644 --- a/src/sqlancer/common/ast/newast/NewOrderingTerm.java +++ b/src/sqlancer/common/ast/newast/NewOrderingTerm.java @@ -4,7 +4,7 @@ public class NewOrderingTerm implements Node { - private final Node expr; + private final T expr; private final Ordering ordering; public enum Ordering { @@ -15,12 +15,12 @@ public static Ordering getRandom() { } } - public NewOrderingTerm(Node expr, Ordering ordering) { + public NewOrderingTerm(T expr, Ordering ordering) { this.expr = expr; this.ordering = ordering; } - public Node getExpr() { + public T getExpr() { return expr; } diff --git a/src/sqlancer/common/ast/newast/NewPostfixTextNode.java b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java index 70e65b17b..4d03a300c 100644 --- a/src/sqlancer/common/ast/newast/NewPostfixTextNode.java +++ b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java @@ -2,15 +2,15 @@ public class NewPostfixTextNode implements Node { - private final Node expr; + private final T expr; private final String text; - public NewPostfixTextNode(Node expr, String text) { + public NewPostfixTextNode(T expr, String text) { this.expr = expr; this.text = text; } - public Node getExpr() { + public T getExpr() { return expr; } diff --git a/src/sqlancer/common/ast/newast/NewTernaryNode.java b/src/sqlancer/common/ast/newast/NewTernaryNode.java index f40c480ec..ab884e48c 100644 --- a/src/sqlancer/common/ast/newast/NewTernaryNode.java +++ b/src/sqlancer/common/ast/newast/NewTernaryNode.java @@ -2,13 +2,13 @@ public class NewTernaryNode implements Node { - protected final Node left; - protected final Node middle; - protected final Node right; + protected final T left; + protected final T middle; + protected final T right; private final String leftStr; private final String rightStr; - public NewTernaryNode(Node left, Node middle, Node right, String leftStr, String rightStr) { + public NewTernaryNode(T left, T middle, T right, String leftStr, String rightStr) { this.left = left; this.middle = middle; this.right = right; @@ -16,15 +16,15 @@ public NewTernaryNode(Node left, Node middle, Node right, String leftSt this.rightStr = rightStr; } - public Node getLeft() { + public T getLeft() { return left; } - public Node getMiddle() { + public T getMiddle() { return middle; } - public Node getRight() { + public T getRight() { return right; } diff --git a/src/sqlancer/common/ast/newast/NewToStringVisitor.java b/src/sqlancer/common/ast/newast/NewToStringVisitor.java index 3ad625440..82b6bace2 100644 --- a/src/sqlancer/common/ast/newast/NewToStringVisitor.java +++ b/src/sqlancer/common/ast/newast/NewToStringVisitor.java @@ -7,7 +7,7 @@ public abstract class NewToStringVisitor { protected final StringBuilder sb = new StringBuilder(); @SuppressWarnings("unchecked") - public void visit(Node expr) { + public void visit(E expr) { assert expr != null; if (expr instanceof ColumnReferenceNode) { sb.append(((ColumnReferenceNode) expr).getColumn().getFullQualifiedName()); @@ -40,7 +40,7 @@ public void visit(Node expr) { } } - public void visit(List> expressions) { + public void visit(List expressions) { for (int i = 0; i < expressions.size(); i++) { if (i != 0) { sb.append(", "); @@ -165,6 +165,6 @@ public String get() { return sb.toString(); } - public abstract void visitSpecific(Node expr); + public abstract void visitSpecific(E expr); } diff --git a/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java index ebd17945f..fb7c28a0f 100644 --- a/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java @@ -5,9 +5,9 @@ public class NewUnaryPostfixOperatorNode implements Node { protected final Operator op; - private final Node expr; + private final T expr; - public NewUnaryPostfixOperatorNode(Node expr, Operator op) { + public NewUnaryPostfixOperatorNode(T expr, Operator op) { this.expr = expr; this.op = op; } @@ -16,8 +16,7 @@ public String getOperatorRepresentation() { return op.getTextRepresentation(); } - public Node getExpr() { + public T getExpr() { return expr; } - } diff --git a/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java index 5274bd942..df75140f7 100644 --- a/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java @@ -5,9 +5,9 @@ public class NewUnaryPrefixOperatorNode implements Node { protected final Operator op; - private final Node expr; + private final T expr; - public NewUnaryPrefixOperatorNode(Node expr, Operator op) { + public NewUnaryPrefixOperatorNode(T expr, Operator op) { this.expr = expr; this.op = op; } @@ -16,7 +16,7 @@ public String getOperatorRepresentation() { return op.getTextRepresentation(); } - public Node getExpr() { + public T getExpr() { return expr; } From f401c57199013ba671be69ded2692322abe1549b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 21 Aug 2024 15:48:51 +0800 Subject: [PATCH 0897/1284] [DuckDB] Update expressions to match new AST --- .../duckdb/DuckDBToStringVisitor.java | 10 +- src/sqlancer/duckdb/ast/DuckDBAlias.java | 9 ++ .../duckdb/ast/DuckDBBetweenOperator.java | 10 ++ .../duckdb/ast/DuckDBBinaryOperator.java | 10 ++ .../duckdb/ast/DuckDBCaseOperator.java | 12 +++ .../duckdb/ast/DuckDBColumnReference.java | 12 +++ src/sqlancer/duckdb/ast/DuckDBConstant.java | 18 ++-- src/sqlancer/duckdb/ast/DuckDBExpression.java | 4 +- src/sqlancer/duckdb/ast/DuckDBFunction.java | 11 +++ src/sqlancer/duckdb/ast/DuckDBInOperator.java | 11 +++ src/sqlancer/duckdb/ast/DuckDBJoin.java | 51 +++++----- .../duckdb/ast/DuckDBOrderingTerm.java | 9 ++ .../duckdb/ast/DuckDBPostFixText.java | 9 ++ src/sqlancer/duckdb/ast/DuckDBSelect.java | 3 +- .../duckdb/ast/DuckDBTableReference.java | 11 +++ src/sqlancer/duckdb/ast/DuckDBTernary.java | 10 ++ .../ast/DuckDBUnaryPostfixOperator.java | 11 +++ .../duckdb/ast/DuckDBUnaryPrefixOperator.java | 11 +++ .../duckdb/gen/DuckDBExpressionGenerator.java | 92 +++++++++---------- .../duckdb/gen/DuckDBIndexGenerator.java | 3 +- .../gen/DuckDBRandomQuerySynthesizer.java | 13 ++- .../duckdb/gen/DuckDBTableGenerator.java | 5 +- .../duckdb/gen/DuckDBUpdateGenerator.java | 3 +- .../duckdb/test/DuckDBNoRECOracle.java | 32 +++---- ...uckDBQueryPartitioningAggregateTester.java | 53 +++++------ .../test/DuckDBQueryPartitioningBase.java | 26 +++--- .../DuckDBQueryPartitioningGroupByTester.java | 10 +- .../DuckDBQueryPartitioningHavingTester.java | 5 +- 28 files changed, 289 insertions(+), 175 deletions(-) create mode 100644 src/sqlancer/duckdb/ast/DuckDBAlias.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBCaseOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBColumnReference.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBFunction.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBInOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBPostFixText.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBTableReference.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBTernary.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index 37493d0f6..b1a8eb703 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -1,7 +1,7 @@ package sqlancer.duckdb; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.duckdb.ast.DuckDBConstant; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; @@ -10,7 +10,7 @@ public class DuckDBToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(DuckDBExpression expr) { if (expr instanceof DuckDBConstant) { visit((DuckDBConstant) expr); } else if (expr instanceof DuckDBSelect) { @@ -23,7 +23,7 @@ public void visitSpecific(Node expr) { } private void visit(DuckDBJoin join) { - visit(join.getLeftTable()); + visit((TableReferenceNode) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" "); @@ -31,7 +31,7 @@ private void visit(DuckDBJoin join) { sb.append(join.getOuterType()); } sb.append(" JOIN "); - visit(join.getRightTable()); + visit((TableReferenceNode) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); @@ -82,7 +82,7 @@ private void visit(DuckDBSelect select) { } } - public static String asString(Node expr) { + public static String asString(DuckDBExpression expr) { DuckDBToStringVisitor visitor = new DuckDBToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/duckdb/ast/DuckDBAlias.java b/src/sqlancer/duckdb/ast/DuckDBAlias.java new file mode 100644 index 000000000..cc1a91921 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBAlias.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class DuckDBAlias extends NewAliasNode implements DuckDBExpression { + public DuckDBAlias(DuckDBExpression expr, String string) { + super(expr, string); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java b/src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java new file mode 100644 index 000000000..eabaa6162 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java @@ -0,0 +1,10 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class DuckDBBetweenOperator extends NewBetweenOperatorNode implements DuckDBExpression { + public DuckDBBetweenOperator(DuckDBExpression left, DuckDBExpression middle, DuckDBExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java b/src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java new file mode 100644 index 000000000..24492c145 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java @@ -0,0 +1,10 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DuckDBBinaryOperator extends NewBinaryOperatorNode implements DuckDBExpression { + public DuckDBBinaryOperator(DuckDBExpression left, DuckDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBCaseOperator.java b/src/sqlancer/duckdb/ast/DuckDBCaseOperator.java new file mode 100644 index 000000000..ba1d9d96f --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBCaseOperator.java @@ -0,0 +1,12 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class DuckDBCaseOperator extends NewCaseOperatorNode implements DuckDBExpression { + public DuckDBCaseOperator(DuckDBExpression switchCondition, List conditions, + List expressions, DuckDBExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBColumnReference.java b/src/sqlancer/duckdb/ast/DuckDBColumnReference.java new file mode 100644 index 000000000..d24382777 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.duckdb.DuckDBSchema; + +public class DuckDBColumnReference extends ColumnReferenceNode + implements DuckDBExpression { + public DuckDBColumnReference(DuckDBSchema.DuckDBColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/duckdb/ast/DuckDBConstant.java b/src/sqlancer/duckdb/ast/DuckDBConstant.java index b895a18e5..f4af6918a 100644 --- a/src/sqlancer/duckdb/ast/DuckDBConstant.java +++ b/src/sqlancer/duckdb/ast/DuckDBConstant.java @@ -3,9 +3,7 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.common.ast.newast.Node; - -public class DuckDBConstant implements Node { +public class DuckDBConstant implements DuckDBExpression { private DuckDBConstant() { } @@ -161,31 +159,31 @@ public String toString() { } - public static Node createStringConstant(String text) { + public static DuckDBExpression createStringConstant(String text) { return new DuckDBTextConstant(text); } - public static Node createFloatConstant(double val) { + public static DuckDBExpression createFloatConstant(double val) { return new DuckDBDoubleConstant(val); } - public static Node createIntConstant(long val) { + public static DuckDBExpression createIntConstant(long val) { return new DuckDBIntConstant(val); } - public static Node createNullConstant() { + public static DuckDBExpression createNullConstant() { return new DuckDBNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static DuckDBExpression createBooleanConstant(boolean val) { return new DuckDBBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static DuckDBExpression createDateConstant(long integer) { return new DuckDBDateConstant(integer); } - public static Node createTimestampConstant(long integer) { + public static DuckDBExpression createTimestampConstant(long integer) { return new DuckDBTimestampConstant(integer); } diff --git a/src/sqlancer/duckdb/ast/DuckDBExpression.java b/src/sqlancer/duckdb/ast/DuckDBExpression.java index 3ff66ce33..21d4e6eb5 100644 --- a/src/sqlancer/duckdb/ast/DuckDBExpression.java +++ b/src/sqlancer/duckdb/ast/DuckDBExpression.java @@ -1,5 +1,7 @@ package sqlancer.duckdb.ast; -public interface DuckDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; +public interface DuckDBExpression extends Expression { } diff --git a/src/sqlancer/duckdb/ast/DuckDBFunction.java b/src/sqlancer/duckdb/ast/DuckDBFunction.java new file mode 100644 index 000000000..445583090 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBFunction.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DuckDBFunction extends NewFunctionNode implements DuckDBExpression { + public DuckDBFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBInOperator.java b/src/sqlancer/duckdb/ast/DuckDBInOperator.java new file mode 100644 index 000000000..619601749 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBInOperator.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class DuckDBInOperator extends NewInOperatorNode implements DuckDBExpression { + public DuckDBInOperator(DuckDBExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBJoin.java b/src/sqlancer/duckdb/ast/DuckDBJoin.java index c1c0c94a9..82031d1e7 100644 --- a/src/sqlancer/duckdb/ast/DuckDBJoin.java +++ b/src/sqlancer/duckdb/ast/DuckDBJoin.java @@ -4,19 +4,18 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.ast.newast.Join; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -public class DuckDBJoin implements Node { +public class DuckDBJoin implements DuckDBExpression, Join { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final DuckDBTableReference leftTable; + private final DuckDBTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private DuckDBExpression onCondition; private OuterType outerType; public enum JoinType { @@ -35,20 +34,19 @@ public static OuterType getRandom() { } } - public DuckDBJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public DuckDBJoin(DuckDBTableReference leftTable, DuckDBTableReference rightTable, JoinType joinType, + DuckDBExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public TableReferenceNode getLeftTable() { + public DuckDBTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public DuckDBTableReference getRightTable() { return rightTable; } @@ -56,7 +54,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public DuckDBExpression getOnCondition() { return onCondition; } @@ -68,12 +66,11 @@ public OuterType getOuterType() { return outerType; } - public static List> getJoins( - List> tableList, DuckDBGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DuckDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + DuckDBTableReference leftTable = tableList.remove(0); + DuckDBTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); DuckDBExpressionGenerator joinGen = new DuckDBExpressionGenerator(globalState).setColumns(columns); @@ -99,26 +96,30 @@ public static List> getJoins( return joinExpressions; } - public static DuckDBJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DuckDBJoin createRightOuterJoin(DuckDBTableReference left, DuckDBTableReference right, + DuckDBExpression predicate) { return new DuckDBJoin(left, right, JoinType.RIGHT, predicate); } - public static DuckDBJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DuckDBJoin createLeftOuterJoin(DuckDBTableReference left, DuckDBTableReference right, + DuckDBExpression predicate) { return new DuckDBJoin(left, right, JoinType.LEFT, predicate); } - public static DuckDBJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DuckDBJoin createInnerJoin(DuckDBTableReference left, DuckDBTableReference right, + DuckDBExpression predicate) { return new DuckDBJoin(left, right, JoinType.INNER, predicate); } - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, OuterType naturalJoinType) { + public static DuckDBExpression createNaturalJoin(DuckDBTableReference left, DuckDBTableReference right, + OuterType naturalJoinType) { DuckDBJoin join = new DuckDBJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; } + @Override + public void setOnClause(DuckDBExpression onClause) { + this.onCondition = onClause; + } } diff --git a/src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java b/src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java new file mode 100644 index 000000000..8d7177655 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class DuckDBOrderingTerm extends NewOrderingTerm implements DuckDBExpression { + public DuckDBOrderingTerm(DuckDBExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBPostFixText.java b/src/sqlancer/duckdb/ast/DuckDBPostFixText.java new file mode 100644 index 000000000..a877a4c92 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBPostFixText.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class DuckDBPostFixText extends NewPostfixTextNode implements DuckDBExpression { + public DuckDBPostFixText(DuckDBExpression expr, String string) { + super(expr, string); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBSelect.java b/src/sqlancer/duckdb/ast/DuckDBSelect.java index 9a2391aa3..10d76fbbd 100644 --- a/src/sqlancer/duckdb/ast/DuckDBSelect.java +++ b/src/sqlancer/duckdb/ast/DuckDBSelect.java @@ -1,9 +1,8 @@ package sqlancer.duckdb.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class DuckDBSelect extends SelectBase> implements Node { +public class DuckDBSelect extends SelectBase implements DuckDBExpression { private boolean isDistinct; diff --git a/src/sqlancer/duckdb/ast/DuckDBTableReference.java b/src/sqlancer/duckdb/ast/DuckDBTableReference.java new file mode 100644 index 000000000..0a8d795c8 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.duckdb.DuckDBSchema; + +public class DuckDBTableReference extends TableReferenceNode + implements DuckDBExpression { + public DuckDBTableReference(DuckDBSchema.DuckDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBTernary.java b/src/sqlancer/duckdb/ast/DuckDBTernary.java new file mode 100644 index 000000000..921f77b90 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBTernary.java @@ -0,0 +1,10 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewTernaryNode; + +public class DuckDBTernary extends NewTernaryNode implements DuckDBExpression { + public DuckDBTernary(DuckDBExpression left, DuckDBExpression middle, DuckDBExpression right, String leftString, + String rightString) { + super(left, middle, right, leftString, rightString); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java b/src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java new file mode 100644 index 000000000..856c129ec --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class DuckDBUnaryPostfixOperator extends NewUnaryPostfixOperatorNode + implements DuckDBExpression { + public DuckDBUnaryPostfixOperator(DuckDBExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java b/src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java new file mode 100644 index 000000000..95c8b4990 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class DuckDBUnaryPrefixOperator extends NewUnaryPrefixOperatorNode + implements DuckDBExpression { + public DuckDBUnaryPrefixOperator(DuckDBExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index eeba94027..49a390c07 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -7,27 +7,25 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewCaseOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; -import sqlancer.common.ast.newast.NewTernaryNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; +import sqlancer.duckdb.ast.DuckDBBetweenOperator; +import sqlancer.duckdb.ast.DuckDBBinaryOperator; +import sqlancer.duckdb.ast.DuckDBCaseOperator; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBConstant; import sqlancer.duckdb.ast.DuckDBExpression; +import sqlancer.duckdb.ast.DuckDBFunction; +import sqlancer.duckdb.ast.DuckDBInOperator; +import sqlancer.duckdb.ast.DuckDBOrderingTerm; +import sqlancer.duckdb.ast.DuckDBTernary; -public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator, DuckDBColumn> { +public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator { private final DuckDBGlobalState globalState; @@ -41,14 +39,14 @@ private enum Expression { } @Override - protected Node generateExpression(int depth) { + protected DuckDBExpression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); } if (allowAggregates && Randomly.getBoolean()) { DuckDBAggregateFunction aggregate = DuckDBAggregateFunction.getRandom(); allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + return new DuckDBFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); if (!globalState.getDbmsSpecificOptions().testCollate) { @@ -78,44 +76,41 @@ protected Node generateExpression(int depth) { Expression expr = Randomly.fromList(possibleOptions); switch (expr) { case COLLATE: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(generateExpression(depth + 1), DuckDBCollate.getRandom()); case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), + return new sqlancer.duckdb.ast.DuckDBUnaryPrefixOperator(generateExpression(depth + 1), DuckDBUnaryPrefixOperator.getRandom()); case UNARY_POSTFIX: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(generateExpression(depth + 1), DuckDBUnaryPostfixOperator.getRandom()); case BINARY_COMPARISON: Operator op = DuckDBBinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new DuckDBBinaryOperator(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_LOGICAL: op = DuckDBBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new DuckDBBinaryOperator(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), DuckDBBinaryArithmeticOperator.getRandom()); + return new DuckDBBinaryOperator(generateExpression(depth + 1), generateExpression(depth + 1), + DuckDBBinaryArithmeticOperator.getRandom()); case CAST: return new DuckDBCastOperation(generateExpression(depth + 1), DuckDBCompositeDataType.getRandomWithoutNull()); case FUNC: DBFunction func = DBFunction.getRandom(); - return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + return new DuckDBFunction<>(generateExpressions(func.getNrArgs()), func); case BETWEEN: - return new NewBetweenOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + return new DuckDBBetweenOperator(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); case IN: - return new NewInOperatorNode(generateExpression(depth + 1), + return new DuckDBInOperator(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; - return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), - generateExpression(depth + 1)); + return new DuckDBCaseOperator(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); case LIKE_ESCAPE: - return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), + return new DuckDBTernary(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1), "LIKE", "ESCAPE"); default: throw new AssertionError(); @@ -123,13 +118,13 @@ protected Node generateExpression(int depth) { } @Override - protected Node generateColumn() { + protected DuckDBExpression generateColumn() { DuckDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode(column); + return new DuckDBColumnReference(column); } @Override - public Node generateConstant() { + public DuckDBExpression generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return DuckDBConstant.createNullConstant(); } @@ -171,21 +166,22 @@ public Node generateConstant() { } @Override - public List> generateOrderBys() { - List> expr = super.generateOrderBys(); - List> newExpr = new ArrayList<>(expr.size()); - for (Node curExpr : expr) { + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (DuckDBExpression curExpr : expr) { if (Randomly.getBoolean()) { - curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); + curExpr = new DuckDBOrderingTerm(curExpr, Ordering.getRandom()); } newExpr.add(curExpr); } return newExpr; }; - public static class DuckDBCastOperation extends NewUnaryPostfixOperatorNode { + public static class DuckDBCastOperation extends NewUnaryPostfixOperatorNode + implements DuckDBExpression { - public DuckDBCastOperation(Node expr, DuckDBCompositeDataType type) { + public DuckDBCastOperation(DuckDBExpression expr, DuckDBCompositeDataType type) { super(expr, new Operator() { @Override @@ -422,25 +418,23 @@ public String getTextRepresentation() { } - public NewFunctionNode generateArgsForAggregate( - DuckDBAggregateFunction aggregateFunction) { - return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + public DuckDBFunction generateArgsForAggregate(DuckDBAggregateFunction aggregateFunction) { + return new DuckDBFunction<>(generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); } - public Node generateAggregate() { + public DuckDBExpression generateAggregate() { DuckDBAggregateFunction aggrFunc = DuckDBAggregateFunction.getRandom(); return generateArgsForAggregate(aggrFunc); } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, DuckDBUnaryPrefixOperator.NOT); + public DuckDBExpression negatePredicate(DuckDBExpression predicate) { + return new sqlancer.duckdb.ast.DuckDBUnaryPrefixOperator(predicate, DuckDBUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, DuckDBUnaryPostfixOperator.IS_NULL); + public DuckDBExpression isNull(DuckDBExpression expr) { + return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(expr, DuckDBUnaryPostfixOperator.IS_NULL); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index eec5384aa..daa22da5f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -45,7 +44,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(")"); if (Randomly.getBoolean()) { sb.append(" WHERE "); - Node expr = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()) + DuckDBExpression expr = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(); sb.append(DuckDBToStringVisitor.asString(expr)); } diff --git a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java index aef88ef13..c4c0d7c95 100644 --- a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java +++ b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java @@ -5,8 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBSchema.DuckDBTables; @@ -14,6 +12,7 @@ import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; public final class DuckDBRandomQuerySynthesizer { @@ -28,10 +27,10 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC // TODO: distinct // select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = gen.generateExpression(); + DuckDBExpression expression = gen.generateExpression(); columns.add(expression); // } else { // columns.add(gen()); @@ -39,9 +38,9 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC } select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + List joins = DuckDBJoin.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index d3cd27fde..ea6d3537f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -5,7 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -27,8 +26,8 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(tableName); sb.append("("); List columns = getNewColumns(); - UntypedExpressionGenerator, DuckDBColumn> gen = new DuckDBExpressionGenerator( - globalState).setColumns(columns); + UntypedExpressionGenerator gen = new DuckDBExpressionGenerator(globalState) + .setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 13b7f9d74..3bb22b2ea 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; @@ -40,7 +39,7 @@ private SQLQueryAdapter generate() { @Override protected void updateValue(DuckDBColumn column) { - Node expr; + DuckDBExpression expr; if (Randomly.getBooleanWithSmallProbability()) { expr = gen.generateExpression(); DuckDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 2ce6a8ad2..63932492d 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -10,10 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -27,9 +23,12 @@ import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBSchema.DuckDBTables; import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; +import sqlancer.duckdb.ast.DuckDBPostFixText; import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBCastOperation; @@ -48,11 +47,11 @@ public void check() throws SQLException { DuckDBTables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); + DuckDBExpression randomWhereCondition = gen.generateExpression(); List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + List joins = DuckDBJoin.getJoins(tableList, state); int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, randomWhereCondition, joins); @@ -65,14 +64,14 @@ public void check() throws SQLException { } } - private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + private int getSecondQuery(List tableList, DuckDBExpression randomWhereCondition, + List joins) throws SQLException { DuckDBSelect select = new DuckDBSelect(); // select.setGroupByClause(groupBys); // DuckDBExpression isTrue = DuckDBPostfixOperation.create(randomWhereCondition, // PostfixOperator.IS_TRUE); - Node asText = new NewPostfixTextNode<>(new DuckDBCastOperation( - new NewPostfixTextNode(randomWhereCondition, + DuckDBExpression asText = new DuckDBPostFixText(new DuckDBCastOperation( + new DuckDBPostFixText(randomWhereCondition, " IS NOT NULL AND " + DuckDBToStringVisitor.asString(randomWhereCondition)), new DuckDBCompositeDataType(DuckDBDataType.INT, 8)), "as count"); select.setFetchColumns(Arrays.asList(asText)); @@ -99,14 +98,13 @@ private int getSecondQuery(List> tableList, Node> tableList, - List columns, Node randomWhereCondition, List> joins) - throws SQLException { + private int getFirstQueryCount(SQLConnection con, List tableList, List columns, + DuckDBExpression randomWhereCondition, List joins) throws SQLException { DuckDBSelect select = new DuckDBSelect(); // select.setGroupByClause(groupBys); // DuckDBAggregate aggr = new DuckDBAggregate( - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List allColumns = columns.stream().map((c) -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); // DuckDBAggregateFunction.COUNT); // select.setFetchColumns(Arrays.asList(aggr)); select.setFetchColumns(allColumns); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index de8bd96a0..e82f97db6 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -8,12 +8,7 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; @@ -22,7 +17,10 @@ import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.ast.DuckDBAlias; +import sqlancer.duckdb.ast.DuckDBBinaryOperator; import sqlancer.duckdb.ast.DuckDBExpression; +import sqlancer.duckdb.ast.DuckDBFunction; import sqlancer.duckdb.ast.DuckDBSelect; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBAggregateFunction; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBBinaryArithmeticOperator; @@ -49,9 +47,8 @@ public void check() throws SQLException { DuckDBAggregateFunction aggregateFunction = Randomly.fromOptions(DuckDBAggregateFunction.MAX, DuckDBAggregateFunction.MIN, DuckDBAggregateFunction.SUM, DuckDBAggregateFunction.COUNT, DuckDBAggregateFunction.AVG/* , DuckDBAggregateFunction.STDDEV_POP */); - NewFunctionNode aggregate = gen - .generateArgsForAggregate(aggregateFunction); - List> fetchColumns = new ArrayList<>(); + DuckDBFunction aggregate = gen.generateArgsForAggregate(aggregateFunction); + List fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { fetchColumns.add(gen.generateAggregate()); @@ -78,15 +75,15 @@ public void check() throws SQLException { } - private String createMetamorphicUnionQuery(DuckDBSelect select, - NewFunctionNode aggregate, List> from) { + private String createMetamorphicUnionQuery(DuckDBSelect select, DuckDBFunction aggregate, + List from) { String metamorphicQuery; - Node whereClause = gen.generateExpression(); - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + DuckDBExpression whereClause = gen.generateExpression(); + DuckDBExpression negatedClause = new sqlancer.duckdb.ast.DuckDBUnaryPrefixOperator(whereClause, DuckDBUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + DuckDBExpression notNullClause = new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(whereClause, DuckDBUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); + List mappedAggregate = mapped(aggregate); DuckDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DuckDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DuckDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); @@ -120,7 +117,7 @@ private String getAggregateResult(String queryString) throws SQLException { } } - private List> mapped(NewFunctionNode aggregate) { + private List mapped(DuckDBFunction aggregate) { DuckDBCastOperation count; switch (aggregate.getFunc()) { case COUNT: @@ -129,21 +126,19 @@ private List> mapped(NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + DuckDBFunction sum = new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.SUM); - count = new DuckDBCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DuckDBAggregateFunction.COUNT), + count = new DuckDBCastOperation(new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.COUNT), new DuckDBCompositeDataType(DuckDBDataType.FLOAT, 8)); return aliasArgs(Arrays.asList(sum, count)); case STDDEV_POP: - NewFunctionNode sumSquared = new NewFunctionNode<>( - Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DuckDBFunction sumSquared = new DuckDBFunction<>( + Arrays.asList(new DuckDBBinaryOperator(aggregate.getArgs().get(0), aggregate.getArgs().get(0), DuckDBBinaryArithmeticOperator.MULT)), DuckDBAggregateFunction.SUM); - count = new DuckDBCastOperation( - new NewFunctionNode(aggregate.getArgs(), - DuckDBAggregateFunction.COUNT), + count = new DuckDBCastOperation(new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.COUNT), new DuckDBCompositeDataType(DuckDBDataType.FLOAT, 8)); - NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), + DuckDBFunction avg = new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.AVG); return aliasArgs(Arrays.asList(sumSquared, count, avg)); default: @@ -151,11 +146,11 @@ private List> mapped(NewFunctionNode> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode(expr, "agg" + i++)); + for (DuckDBExpression expr : originalAggregateArgs) { + args.add(new DuckDBAlias(expr, "agg" + i++)); } return args; } @@ -173,8 +168,8 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, - Node whereClause, List> joinList) { + private DuckDBSelect getSelect(List aggregates, List from, + DuckDBExpression whereClause, List joinList) { DuckDBSelect leftSelect = new DuckDBSelect(); leftSelect.setFetchColumns(aggregates); leftSelect.setFromList(from); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 2acc6a6d6..b4858df4c 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -6,9 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -18,13 +15,14 @@ import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBSchema.DuckDBTables; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -public class DuckDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, DuckDBGlobalState> +public class DuckDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase implements TestOracle { DuckDBSchema s; @@ -46,27 +44,27 @@ public void check() throws SQLException { select = new DuckDBSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + List joins = DuckDBJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new DuckDBColumn("*", null, false, false))); + columns.add(new DuckDBColumnReference(new DuckDBColumn("*", null, false, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); } return columns; } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java index 86c26370d..c40a63a18 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java @@ -7,12 +7,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; public class DuckDBQueryPartitioningGroupByTester extends DuckDBQueryPartitioningBase { @@ -45,9 +43,9 @@ public void check() throws SQLException { } @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index b75bf02a5..b0ff0a44c 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -7,7 +7,6 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -51,12 +50,12 @@ public void check() throws SQLException { } @Override - protected Node generatePredicate() { + protected DuckDBExpression generatePredicate() { return gen.generateHavingClause(); } @Override - List> generateFetchColumns() { + List generateFetchColumns() { return Arrays.asList(gen.generateHavingClause()); } From eac479e2d61c5c8bc817c25e3b434e92304c153d Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 23 Aug 2024 09:54:57 +0800 Subject: [PATCH 0898/1284] Remove Node supertype for new AST expressions --- src/sqlancer/common/ast/newast/ColumnReferenceNode.java | 2 +- src/sqlancer/common/ast/newast/Join.java | 2 -- src/sqlancer/common/ast/newast/NewAliasNode.java | 2 +- src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewCaseOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewFunctionNode.java | 2 +- src/sqlancer/common/ast/newast/NewInOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewOrderingTerm.java | 2 +- src/sqlancer/common/ast/newast/NewPostfixTextNode.java | 2 +- src/sqlancer/common/ast/newast/NewTernaryNode.java | 2 +- src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/TableReferenceNode.java | 2 +- 14 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/common/ast/newast/ColumnReferenceNode.java b/src/sqlancer/common/ast/newast/ColumnReferenceNode.java index 18ff09db6..2d8688d8f 100644 --- a/src/sqlancer/common/ast/newast/ColumnReferenceNode.java +++ b/src/sqlancer/common/ast/newast/ColumnReferenceNode.java @@ -2,7 +2,7 @@ import sqlancer.common.schema.AbstractTableColumn; -public class ColumnReferenceNode> implements Node { +public class ColumnReferenceNode> { private final C c; diff --git a/src/sqlancer/common/ast/newast/Join.java b/src/sqlancer/common/ast/newast/Join.java index 90935b338..94d26c139 100644 --- a/src/sqlancer/common/ast/newast/Join.java +++ b/src/sqlancer/common/ast/newast/Join.java @@ -6,7 +6,5 @@ public interface Join, T extends AbstractTable, C extends AbstractTableColumn> extends Expression { - Expression getOnClause(); - void setOnClause(E onClause); } diff --git a/src/sqlancer/common/ast/newast/NewAliasNode.java b/src/sqlancer/common/ast/newast/NewAliasNode.java index 9fbbb35d5..260900712 100644 --- a/src/sqlancer/common/ast/newast/NewAliasNode.java +++ b/src/sqlancer/common/ast/newast/NewAliasNode.java @@ -1,6 +1,6 @@ package sqlancer.common.ast.newast; -public class NewAliasNode implements Node { +public class NewAliasNode { private final E expr; private final String alias; diff --git a/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java index e47012267..4bb9ce17a 100644 --- a/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java @@ -1,6 +1,6 @@ package sqlancer.common.ast.newast; -public class NewBetweenOperatorNode implements Node { +public class NewBetweenOperatorNode { protected T left; protected T middle; diff --git a/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java index 6f150499c..b2fcc5ef0 100644 --- a/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java @@ -2,7 +2,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; -public class NewBinaryOperatorNode implements Node { +public class NewBinaryOperatorNode { protected final Operator op; protected final T left; diff --git a/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java index 5ab2ed853..440078566 100644 --- a/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java @@ -2,7 +2,7 @@ import java.util.List; -public class NewCaseOperatorNode implements Node { +public class NewCaseOperatorNode { private final List conditions; private final List expressions; diff --git a/src/sqlancer/common/ast/newast/NewFunctionNode.java b/src/sqlancer/common/ast/newast/NewFunctionNode.java index baa5d1bc3..cdd91bd61 100644 --- a/src/sqlancer/common/ast/newast/NewFunctionNode.java +++ b/src/sqlancer/common/ast/newast/NewFunctionNode.java @@ -2,7 +2,7 @@ import java.util.List; -public class NewFunctionNode implements Node { +public class NewFunctionNode { protected List args; protected F func; diff --git a/src/sqlancer/common/ast/newast/NewInOperatorNode.java b/src/sqlancer/common/ast/newast/NewInOperatorNode.java index b4bcf535d..94a3a0886 100644 --- a/src/sqlancer/common/ast/newast/NewInOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewInOperatorNode.java @@ -2,7 +2,7 @@ import java.util.List; -public class NewInOperatorNode implements Node { +public class NewInOperatorNode { private final T left; private final List right; diff --git a/src/sqlancer/common/ast/newast/NewOrderingTerm.java b/src/sqlancer/common/ast/newast/NewOrderingTerm.java index be938fbb4..2efed05bc 100644 --- a/src/sqlancer/common/ast/newast/NewOrderingTerm.java +++ b/src/sqlancer/common/ast/newast/NewOrderingTerm.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class NewOrderingTerm implements Node { +public class NewOrderingTerm { private final T expr; private final Ordering ordering; diff --git a/src/sqlancer/common/ast/newast/NewPostfixTextNode.java b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java index 4d03a300c..716dbf8e3 100644 --- a/src/sqlancer/common/ast/newast/NewPostfixTextNode.java +++ b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java @@ -1,6 +1,6 @@ package sqlancer.common.ast.newast; -public class NewPostfixTextNode implements Node { +public class NewPostfixTextNode { private final T expr; private final String text; diff --git a/src/sqlancer/common/ast/newast/NewTernaryNode.java b/src/sqlancer/common/ast/newast/NewTernaryNode.java index ab884e48c..2ca7b1f30 100644 --- a/src/sqlancer/common/ast/newast/NewTernaryNode.java +++ b/src/sqlancer/common/ast/newast/NewTernaryNode.java @@ -1,6 +1,6 @@ package sqlancer.common.ast.newast; -public class NewTernaryNode implements Node { +public class NewTernaryNode { protected final T left; protected final T middle; diff --git a/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java index fb7c28a0f..b3ccd4cd3 100644 --- a/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java @@ -2,7 +2,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; -public class NewUnaryPostfixOperatorNode implements Node { +public class NewUnaryPostfixOperatorNode { protected final Operator op; private final T expr; diff --git a/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java index df75140f7..8668aec35 100644 --- a/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java @@ -2,7 +2,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; -public class NewUnaryPrefixOperatorNode implements Node { +public class NewUnaryPrefixOperatorNode { protected final Operator op; private final T expr; diff --git a/src/sqlancer/common/ast/newast/TableReferenceNode.java b/src/sqlancer/common/ast/newast/TableReferenceNode.java index 981da70ad..0719a0ed6 100644 --- a/src/sqlancer/common/ast/newast/TableReferenceNode.java +++ b/src/sqlancer/common/ast/newast/TableReferenceNode.java @@ -2,7 +2,7 @@ import sqlancer.common.schema.AbstractTable; -public class TableReferenceNode> implements Node { +public class TableReferenceNode> { private final T t; From baa1f1c04e6d97bf12b33c83475e18ac3fb94544 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 23 Aug 2024 11:04:13 +0800 Subject: [PATCH 0899/1284] [Databend] Update expressions to match new AST --- .../DatabendExpectedValueVisitor.java | 103 +++++++++--------- src/sqlancer/databend/DatabendExprToNode.java | 25 ----- .../databend/DatabendToStringVisitor.java | 5 +- .../ast/DatabendAggregateOperation.java | 9 +- src/sqlancer/databend/ast/DatabendAlias.java | 9 ++ .../ast/DatabendBetweenOperation.java | 9 +- .../DatabendBinaryArithmeticOperation.java | 7 +- .../DatabendBinaryComparisonOperation.java | 7 +- .../ast/DatabendBinaryLogicalOperation.java | 7 +- .../databend/ast/DatabendBinaryOperation.java | 12 ++ .../databend/ast/DatabendCastOperation.java | 3 +- .../databend/ast/DatabendColumnReference.java | 11 ++ .../databend/ast/DatabendConstant.java | 3 +- .../ast/DatabendFunctionOperation.java | 9 +- .../databend/ast/DatabendInOperation.java | 3 +- src/sqlancer/databend/ast/DatabendJoin.java | 49 ++++----- .../databend/ast/DatabendLikeOperation.java | 7 +- .../databend/ast/DatabendOrderByTerm.java | 3 +- .../databend/ast/DatabendPostFixText.java | 9 ++ src/sqlancer/databend/ast/DatabendSelect.java | 3 +- .../databend/ast/DatabendTableReference.java | 11 ++ .../ast/DatabendUnaryPostfixOperation.java | 3 +- .../ast/DatabendUnaryPrefixOperation.java | 3 +- .../databend/gen/DatabendDeleteGenerator.java | 5 +- .../databend/gen/DatabendInsertGenerator.java | 6 +- .../gen/DatabendNewExpressionGenerator.java | 3 +- .../gen/DatabendRandomQuerySynthesizer.java | 23 ++-- .../databend/gen/DatabendTableGenerator.java | 4 +- .../databend/test/DatabendNoRECOracle.java | 37 +++---- .../DatabendPivotedQuerySynthesisOracle.java | 28 ++--- ...abendQueryPartitioningAggregateTester.java | 62 +++++------ .../tlp/DatabendQueryPartitioningBase.java | 31 +++--- ...tabendQueryPartitioningDistinctTester.java | 7 +- ...atabendQueryPartitioningGroupByTester.java | 17 ++- ...DatabendQueryPartitioningHavingTester.java | 17 ++- .../DatabendQueryPartitioningWhereTester.java | 7 +- .../doris/visitor/DorisExprToNode.java | 25 ----- 37 files changed, 268 insertions(+), 314 deletions(-) delete mode 100644 src/sqlancer/databend/DatabendExprToNode.java create mode 100644 src/sqlancer/databend/ast/DatabendAlias.java create mode 100644 src/sqlancer/databend/ast/DatabendBinaryOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendColumnReference.java create mode 100644 src/sqlancer/databend/ast/DatabendPostFixText.java create mode 100644 src/sqlancer/databend/ast/DatabendTableReference.java delete mode 100644 src/sqlancer/doris/visitor/DorisExprToNode.java diff --git a/src/sqlancer/databend/DatabendExpectedValueVisitor.java b/src/sqlancer/databend/DatabendExpectedValueVisitor.java index f6ff93819..e10ade97a 100644 --- a/src/sqlancer/databend/DatabendExpectedValueVisitor.java +++ b/src/sqlancer/databend/DatabendExpectedValueVisitor.java @@ -2,60 +2,57 @@ import java.util.List; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.ast.DatabendAlias; +import sqlancer.databend.ast.DatabendBetweenOperation; +import sqlancer.databend.ast.DatabendBinaryOperation; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendFunctionOperation; +import sqlancer.databend.ast.DatabendInOperation; import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendOrderByTerm; +import sqlancer.databend.ast.DatabendPostFixText; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; public class DatabendExpectedValueVisitor { protected final StringBuilder sb = new StringBuilder(); - private void print(Node expr) { + private void print(DatabendExpression expr) { sb.append(DatabendToStringVisitor.asString(expr)); sb.append(" -- "); - sb.append(((DatabendExpression) expr).getExpectedValue()); + sb.append((expr).getExpectedValue()); sb.append("\n"); } - @SuppressWarnings("unchecked") - public void visit(Node expr) { + public void visit(DatabendExpression expr) { assert expr != null; - if (expr instanceof ColumnReferenceNode) { - visit((ColumnReferenceNode) expr); - } else if (expr instanceof NewUnaryPostfixOperatorNode) { - visit((NewUnaryPostfixOperatorNode) expr); - } else if (expr instanceof NewUnaryPrefixOperatorNode) { - visit((NewUnaryPrefixOperatorNode) expr); - } else if (expr instanceof NewBinaryOperatorNode) { - visit((NewBinaryOperatorNode) expr); - } else if (expr instanceof TableReferenceNode) { - visit((TableReferenceNode) expr); - } else if (expr instanceof NewFunctionNode) { - visit((NewFunctionNode) expr); - } else if (expr instanceof NewBetweenOperatorNode) { - visit((NewBetweenOperatorNode) expr); - } else if (expr instanceof NewInOperatorNode) { - visit((NewInOperatorNode) expr); - } else if (expr instanceof NewOrderingTerm) { - visit((NewOrderingTerm) expr); - } else if (expr instanceof NewAliasNode) { - visit((NewAliasNode) expr); - } else if (expr instanceof NewPostfixTextNode) { - visit((NewPostfixTextNode) expr); + if (expr instanceof DatabendColumnReference) { + visit((DatabendColumnReference) expr); + } else if (expr instanceof DatabendUnaryPostfixOperation) { + visit((DatabendUnaryPostfixOperation) expr); + } else if (expr instanceof DatabendUnaryPrefixOperation) { + visit((DatabendUnaryPrefixOperation) expr); + } else if (expr instanceof DatabendBinaryOperation) { + visit((DatabendBinaryOperation) expr); + } else if (expr instanceof DatabendTableReference) { + visit((DatabendTableReference) expr); + } else if (expr instanceof DatabendFunctionOperation) { + visit((DatabendFunctionOperation) expr); + } else if (expr instanceof DatabendBetweenOperation) { + visit((DatabendBetweenOperation) expr); + } else if (expr instanceof DatabendInOperation) { + visit((DatabendInOperation) expr); + } else if (expr instanceof DatabendOrderByTerm) { + visit((DatabendOrderByTerm) expr); + } else if (expr instanceof DatabendAlias) { + visit((DatabendAlias) expr); + } else if (expr instanceof DatabendPostFixText) { + visit((DatabendPostFixText) expr); } else if (expr instanceof DatabendConstant) { visit((DatabendConstant) expr); } else if (expr instanceof DatabendSelect) { @@ -67,65 +64,65 @@ public void visit(Node expr) { } } - public void visit(ColumnReferenceNode c) { + public void visit(DatabendColumnReference c) { print(c); } - public void visit(NewUnaryPostfixOperatorNode op) { + public void visit(DatabendUnaryPostfixOperation op) { print(op); visit(op.getExpr()); } - public void visit(NewUnaryPrefixOperatorNode op) { + public void visit(DatabendUnaryPrefixOperation op) { print(op); visit(op.getExpr()); } - public void visit(NewBinaryOperatorNode op) { + public void visit(DatabendBinaryOperation op) { print(op); visit(op.getLeft()); visit(op.getRight()); } - public void visit(TableReferenceNode t) { + public void visit(DatabendTableReference t) { print(t); } - public void visit(NewFunctionNode fun) { + public void visit(DatabendFunctionOperation fun) { print(fun); visit(fun.getArgs()); } - public void visit(List> expressions) { - for (Node expression : expressions) { + public void visit(List expressions) { + for (DatabendExpression expression : expressions) { visit(expression); } } - public void visit(NewBetweenOperatorNode op) { + public void visit(DatabendBetweenOperation op) { print(op); visit(op.getLeft()); visit(op.getMiddle()); visit(op.getRight()); } - public void visit(NewInOperatorNode op) { + public void visit(DatabendInOperation op) { print(op); visit(op.getLeft()); visit(op.getRight()); } - public void visit(NewOrderingTerm op) { + public void visit(DatabendOrderByTerm op) { print(op); visit(op.getExpr()); } - public void visit(NewAliasNode op) { + public void visit(DatabendAlias op) { print(op); visit(op.getExpr()); } - public void visit(NewPostfixTextNode postFixText) { + public void visit(DatabendPostFixText postFixText) { print(postFixText); visit(postFixText.getExpr()); } @@ -146,7 +143,7 @@ public String get() { return sb.toString(); } - public static String asExpectedValues(Node expr) { + public static String asExpectedValues(DatabendExpression expr) { DatabendExpectedValueVisitor v = new DatabendExpectedValueVisitor(); v.visit(expr); return v.get(); diff --git a/src/sqlancer/databend/DatabendExprToNode.java b/src/sqlancer/databend/DatabendExprToNode.java deleted file mode 100644 index 0b2815f5c..000000000 --- a/src/sqlancer/databend/DatabendExprToNode.java +++ /dev/null @@ -1,25 +0,0 @@ -package sqlancer.databend; - -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.ast.newast.Node; -import sqlancer.databend.ast.DatabendExpression; - -public final class DatabendExprToNode { - - private DatabendExprToNode() { - - } - - @SuppressWarnings("unchecked") - public static Node cast(DatabendExpression expression) { - return (Node) expression; - } - - @SuppressWarnings("unchecked") - public static List> casts(List expressions) { - return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); - } - -} diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java index 87c94d074..05d09d7f2 100644 --- a/src/sqlancer/databend/DatabendToStringVisitor.java +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.databend; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -10,7 +9,7 @@ public class DatabendToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(DatabendExpression expr) { if (expr instanceof DatabendConstant) { visit((DatabendConstant) expr); } else if (expr instanceof DatabendSelect) { @@ -91,7 +90,7 @@ private void visit(DatabendSelect select) { } } - public static String asString(Node expr) { + public static String asString(DatabendExpression expr) { DatabendToStringVisitor visitor = new DatabendToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/databend/ast/DatabendAggregateOperation.java b/src/sqlancer/databend/ast/DatabendAggregateOperation.java index df5e96199..5070991d8 100644 --- a/src/sqlancer/databend/ast/DatabendAggregateOperation.java +++ b/src/sqlancer/databend/ast/DatabendAggregateOperation.java @@ -3,15 +3,12 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema; -public class DatabendAggregateOperation - extends NewFunctionNode - implements DatabendExpression { +public class DatabendAggregateOperation extends + DatabendFunctionOperation implements DatabendExpression { public DatabendAggregateOperation(List args, DatabendAggregateFunction func) { - super(DatabendExprToNode.casts(args), func); + super(args, func); } public enum DatabendAggregateFunction { diff --git a/src/sqlancer/databend/ast/DatabendAlias.java b/src/sqlancer/databend/ast/DatabendAlias.java new file mode 100644 index 000000000..5f6827409 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendAlias.java @@ -0,0 +1,9 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class DatabendAlias extends NewAliasNode implements DatabendExpression { + public DatabendAlias(DatabendExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/databend/ast/DatabendBetweenOperation.java b/src/sqlancer/databend/ast/DatabendBetweenOperation.java index 27f5532c4..a6a0134bf 100644 --- a/src/sqlancer/databend/ast/DatabendBetweenOperation.java +++ b/src/sqlancer/databend/ast/DatabendBetweenOperation.java @@ -1,25 +1,24 @@ package sqlancer.databend.ast; import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema; public class DatabendBetweenOperation extends NewBetweenOperatorNode implements DatabendExpression { public DatabendBetweenOperation(DatabendExpression left, DatabendExpression middle, DatabendExpression right, boolean isTrue) { - super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(middle), DatabendExprToNode.cast(right), isTrue); + super(left, middle, right, isTrue); } public DatabendExpression getLeftExpr() { - return (DatabendExpression) left; + return left; } public DatabendExpression getMiddleExpr() { - return (DatabendExpression) middle; + return middle; } public DatabendExpression getRightExpr() { - return (DatabendExpression) right; + return right; } @Override diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java index 391597941..5554cc474 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -4,7 +4,6 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode @@ -12,7 +11,7 @@ public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode @@ -10,15 +9,15 @@ public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode @@ -11,15 +10,15 @@ public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode implements DatabendExpression { + public DatabendBinaryOperation(DatabendExpression left, DatabendExpression right, + BinaryOperatorNode.Operator operator) { + super(left, right, operator); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendCastOperation.java b/src/sqlancer/databend/ast/DatabendCastOperation.java index 534836d59..07e7d7f9c 100644 --- a/src/sqlancer/databend/ast/DatabendCastOperation.java +++ b/src/sqlancer/databend/ast/DatabendCastOperation.java @@ -2,7 +2,6 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; @@ -11,7 +10,7 @@ public class DatabendCastOperation extends NewUnaryPostfixOperatorNode expr, DatabendCompositeDataType type) { + public DatabendCastOperation(DatabendExpression expr, DatabendCompositeDataType type) { super(expr, new BinaryOperatorNode.Operator() { @Override public String getTextRepresentation() { diff --git a/src/sqlancer/databend/ast/DatabendColumnReference.java b/src/sqlancer/databend/ast/DatabendColumnReference.java new file mode 100644 index 000000000..4b53ef4df --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendColumnReference extends ColumnReferenceNode + implements DatabendExpression { + public DatabendColumnReference(DatabendSchema.DatabendColumn column) { + super(column); + } +} diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 0cc3d0bd2..435bf6ece 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -3,10 +3,9 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendSchema.DatabendDataType; -public abstract class DatabendConstant implements Node, DatabendExpression { +public abstract class DatabendConstant implements DatabendExpression { private DatabendConstant() { } diff --git a/src/sqlancer/databend/ast/DatabendFunctionOperation.java b/src/sqlancer/databend/ast/DatabendFunctionOperation.java index 855e1c8f1..69cb16175 100644 --- a/src/sqlancer/databend/ast/DatabendFunctionOperation.java +++ b/src/sqlancer/databend/ast/DatabendFunctionOperation.java @@ -1,4 +1,11 @@ package sqlancer.databend.ast; -public class DatabendFunctionOperation { // TODO add databend function +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DatabendFunctionOperation extends NewFunctionNode implements DatabendExpression { + public DatabendFunctionOperation(List args, F func) { + super(args, func); + } } diff --git a/src/sqlancer/databend/ast/DatabendInOperation.java b/src/sqlancer/databend/ast/DatabendInOperation.java index 304c3e3f9..f382aae04 100644 --- a/src/sqlancer/databend/ast/DatabendInOperation.java +++ b/src/sqlancer/databend/ast/DatabendInOperation.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema; public class DatabendInOperation extends NewInOperatorNode implements DatabendExpression { @@ -12,7 +11,7 @@ public class DatabendInOperation extends NewInOperatorNode i private final List rightExpr; public DatabendInOperation(DatabendExpression left, List right, boolean isNegated) { - super(DatabendExprToNode.cast(left), DatabendExprToNode.casts(right), isNegated); + super(left, right, isNegated); this.leftExpr = left; this.rightExpr = right; } diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index 123916542..b927eab92 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -4,21 +4,19 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendJoin implements Node { +public class DatabendJoin implements DatabendExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final DatabendTableReference leftTable; + private final DatabendTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final DatabendExpression onCondition; private OuterType outerType; public enum JoinType { @@ -37,9 +35,8 @@ public static OuterType getRandom() { } } - public DatabendJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public DatabendJoin(DatabendTableReference leftTable, DatabendTableReference rightTable, JoinType joinType, + DatabendExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; @@ -58,7 +55,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public DatabendExpression getOnCondition() { return onCondition; } @@ -70,12 +67,12 @@ public OuterType getOuterType() { return outerType; } - public static List> getJoins( - List> tableList, DatabendGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, + DatabendGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + DatabendTableReference leftTable = tableList.remove(0); + DatabendTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState) @@ -84,18 +81,18 @@ public static List> getJoins( switch (JoinType.getRandom()) { case INNER: joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, - DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case NATURAL: joinExpressions.add(DatabendJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); break; case LEFT: joinExpressions.add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, - DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case RIGHT: joinExpressions.add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, - DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; default: throw new AssertionError(); @@ -104,23 +101,23 @@ public static List> getJoins( return joinExpressions; } - public static DatabendJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DatabendJoin createRightOuterJoin(DatabendTableReference left, DatabendTableReference right, + DatabendExpression predicate) { return new DatabendJoin(left, right, JoinType.RIGHT, predicate); } - public static DatabendJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DatabendJoin createLeftOuterJoin(DatabendTableReference left, DatabendTableReference right, + DatabendExpression predicate) { return new DatabendJoin(left, right, JoinType.LEFT, predicate); } - public static DatabendJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DatabendJoin createInnerJoin(DatabendTableReference left, DatabendTableReference right, + DatabendExpression predicate) { return new DatabendJoin(left, right, JoinType.INNER, predicate); } - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, OuterType naturalJoinType) { + public static DatabendExpression createNaturalJoin(DatabendTableReference left, DatabendTableReference right, + OuterType naturalJoinType) { DatabendJoin join = new DatabendJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; diff --git a/src/sqlancer/databend/ast/DatabendLikeOperation.java b/src/sqlancer/databend/ast/DatabendLikeOperation.java index 36ac73603..f40d804bb 100644 --- a/src/sqlancer/databend/ast/DatabendLikeOperation.java +++ b/src/sqlancer/databend/ast/DatabendLikeOperation.java @@ -4,13 +4,12 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; public class DatabendLikeOperation extends NewBinaryOperatorNode implements DatabendExpression { public DatabendLikeOperation(DatabendExpression left, DatabendExpression right, DatabendLikeOperator op) { - super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); + super(left, right, op); } @Override @@ -19,11 +18,11 @@ public DatabendDataType getExpectedType() { } public DatabendExpression getLeftExpr() { - return (DatabendExpression) super.getLeft(); + return super.getLeft(); } public DatabendExpression getRightExpr() { - return (DatabendExpression) super.getRight(); + return super.getRight(); } public DatabendLikeOperator getOp() { diff --git a/src/sqlancer/databend/ast/DatabendOrderByTerm.java b/src/sqlancer/databend/ast/DatabendOrderByTerm.java index 5c78bae3f..7492c7c33 100644 --- a/src/sqlancer/databend/ast/DatabendOrderByTerm.java +++ b/src/sqlancer/databend/ast/DatabendOrderByTerm.java @@ -1,10 +1,9 @@ package sqlancer.databend.ast; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.Node; public class DatabendOrderByTerm extends NewOrderingTerm implements DatabendExpression { - public DatabendOrderByTerm(Node expr, Ordering ordering) { + public DatabendOrderByTerm(DatabendExpression expr, Ordering ordering) { super(expr, ordering); } } diff --git a/src/sqlancer/databend/ast/DatabendPostFixText.java b/src/sqlancer/databend/ast/DatabendPostFixText.java new file mode 100644 index 000000000..8f854fbe1 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendPostFixText.java @@ -0,0 +1,9 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class DatabendPostFixText extends NewPostfixTextNode implements DatabendExpression { + public DatabendPostFixText(DatabendExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/databend/ast/DatabendSelect.java b/src/sqlancer/databend/ast/DatabendSelect.java index b3ccc08f5..3b0854ede 100644 --- a/src/sqlancer/databend/ast/DatabendSelect.java +++ b/src/sqlancer/databend/ast/DatabendSelect.java @@ -1,9 +1,8 @@ package sqlancer.databend.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class DatabendSelect extends SelectBase> implements Node { +public class DatabendSelect extends SelectBase implements DatabendExpression { private boolean isDistinct; diff --git a/src/sqlancer/databend/ast/DatabendTableReference.java b/src/sqlancer/databend/ast/DatabendTableReference.java new file mode 100644 index 000000000..7a98d3877 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendTableReference extends TableReferenceNode + implements DatabendExpression { + public DatabendTableReference(DatabendSchema.DatabendTable table) { + super(table); + } +} diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java index a19512edf..119f93e5c 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -3,14 +3,13 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements DatabendExpression { public DatabendUnaryPostfixOperation(DatabendExpression expr, DatabendUnaryPostfixOperator op) { - super(DatabendExprToNode.cast(expr), op); + super(expr, op); } public DatabendExpression getExpression() { diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index f115684e0..ea6b65f07 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -3,14 +3,13 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements DatabendExpression { public DatabendUnaryPrefixOperation(DatabendExpression expr, DatabendUnaryPrefixOperator op) { - super(DatabendExprToNode.cast(expr), op); + super(expr, op); } public DatabendExpression getExpression() { diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index 9fabfc125..256030409 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -4,7 +4,6 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; @@ -20,8 +19,8 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); - sb.append(DatabendToStringVisitor.asString(DatabendExprToNode.cast( - new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN)))); + sb.append(DatabendToStringVisitor.asString( + new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN))); DatabendErrors.addExpressionErrors(errors); } return new SQLQueryAdapter(sb.toString(), errors); diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index a86f0f3a9..340e494ef 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -8,7 +8,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; @@ -49,9 +48,8 @@ protected void insertValue(DatabendColumn column) { // } else { // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); // } - String value = DatabendToStringVisitor - .asString(DatabendExprToNode.cast(new DatabendNewExpressionGenerator(globalState) - .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); // 生成一个与column相同的常量类型 + String value = DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 2faf4376b..19a45eaa2 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -9,7 +9,6 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendProvider.DatabendGlobalState; @@ -93,7 +92,7 @@ private DatabendExpression createColumnOfType(DatabendDataType type) { return DatabendColumnValue.create(column, value); } - public List> generateOrderBy() { + public List generateOrderBy() { List randomColumns = Randomly.subset(columns); return randomColumns.stream().map( c -> new DatabendOrderByTerm(new DatabendColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 4f0b2fd00..7b7d0e35b 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -6,9 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -19,6 +16,7 @@ import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; public final class DatabendRandomQuerySynthesizer { @@ -30,19 +28,19 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int List targetColumns = targetTables.getColumns(); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(targetColumns); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); HashSet columnOfLeafNode = new HashSet<>(); gen.setColumnOfLeafNode(columnOfLeafNode); int freeColumns = targetColumns.size(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node column = null; + DatabendExpression column = null; if (freeColumns > 0 && Randomly.getBoolean()) { column = new DatabendColumnValue(targetColumns.get(freeColumns - 1), null); freeColumns -= 1; columnOfLeafNode.add((DatabendColumnValue) column); } else { - column = DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); + column = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); } columns.add(column); } @@ -51,17 +49,16 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setDistinct(isDistinct); select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DatabendJoin.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); + List joins = DatabendJoin.getJoins(tableList, globalState); select.setJoinList(joins); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { - select.setWhereClause( - DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - List> noExprColumns = new ArrayList<>(columnOfLeafNode); + List noExprColumns = new ArrayList<>(columnOfLeafNode); if (Randomly.getBoolean() && !noExprColumns.isEmpty() && !isDistinct) { select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); @@ -71,7 +68,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int } if (Randomly.getBoolean()) { // 可能产生新的column叶子结点 - select.setHavingClause(DatabendExprToNode.cast(gen.generateHavingClause())); + select.setHavingClause(gen.generateHavingClause()); } noExprColumns = new ArrayList<>(columnOfLeafNode); diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index ab4900c07..514d740d0 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -8,7 +8,6 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -47,8 +46,7 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 - DatabendExprToNode - .cast(gen.generateConstant(columns.get(i).getType().getPrimitiveDataType())))); + gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); sb.append(")"); } } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 50a8c4de4..54f17858c 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -9,16 +9,11 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -28,9 +23,12 @@ import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendCastOperation; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendPostFixText; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.gen.DatabendNewExpressionGenerator; public class DatabendNoRECOracle extends NoRECBase implements TestOracle { @@ -51,12 +49,11 @@ public void check() throws SQLException { debugColumns(columns, randomTables); // 调试代码，可忽略 } DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = DatabendExprToNode - .cast(gen.generateExpression(DatabendDataType.BOOLEAN)); // 生成随机where条件 + DatabendExpression randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件 List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DatabendJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); + List joins = DatabendJoin.getJoins(tableList, state); int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, @@ -70,12 +67,12 @@ public void check() throws SQLException { } } - private int getUnoptimizedQueryCount(List> tableList, - Node randomWhereCondition, List> joins) throws SQLException { + private int getUnoptimizedQueryCount(List tableList, DatabendExpression randomWhereCondition, + List joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); - Node asText = new NewPostfixTextNode<>(new DatabendCastOperation( - new NewPostfixTextNode(randomWhereCondition, + DatabendExpression asText = new DatabendPostFixText(new DatabendCastOperation( + new DatabendPostFixText(randomWhereCondition, " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); @@ -101,20 +98,18 @@ private int getUnoptimizedQueryCount(List> tableList, return secondCount; } - private int getOptimizedQueryCount(SQLConnection con, List> tableList, - List columns, Node randomWhereCondition, - List> joins) throws SQLException { + private int getOptimizedQueryCount(SQLConnection con, List tableList, + List columns, DatabendExpression randomWhereCondition, List joins) + throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)) + List allColumns = columns.stream().map((c) -> new DatabendColumnReference(c)) .collect(Collectors.toList()); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new DatabendNewExpressionGenerator(state).setColumns(columns).generateOrderBys() - .stream().map(DatabendExprToNode::cast).collect(Collectors.toList())); + select.setOrderByClauses(new DatabendNewExpressionGenerator(state).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); int firstCount = 0; diff --git a/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java index bfb21414e..4a72454b5 100644 --- a/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java @@ -6,25 +6,22 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendExpectedValueVisitor; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendSchema.DatabendRowValue; -import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.gen.DatabendNewExpressionGenerator; @@ -52,21 +49,21 @@ protected Query getRectifiedQuery() throws Exception { selectStatement.setFetchColumns(fetchColumns.stream() .map(c -> new DatabendColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) .collect(Collectors.toList())); - selectStatement.setFromList(randomTables.getTables().stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList())); + selectStatement.setFromList( + randomTables.getTables().stream().map(t -> new DatabendTableReference(t)).collect(Collectors.toList())); DatabendExpression whereClause = generateRectifiedExpression(columns, pivotRow); - selectStatement.setWhereClause(DatabendExprToNode.cast(whereClause)); - List> groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); - Node limitClause = generateLimit(); + DatabendExpression limitClause = generateLimit(); selectStatement.setLimitClause(limitClause); if (limitClause != null) { - Node offsetClause = generateOffset(); + DatabendExpression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); if (!isDistinct) { - List> orderBys = gen.generateOrderBy(); + List orderBys = gen.generateOrderBy(); selectStatement.setOrderByClauses(orderBys); } return new SQLQueryAdapter(DatabendToStringVisitor.asString(selectStatement), errors); @@ -122,11 +119,10 @@ private DatabendColumn getFetchValueAliasedColumn(DatabendColumn c) { @Override protected String getExpectedValues(DatabendExpression expr) { - return DatabendExpectedValueVisitor.asExpectedValues(DatabendExprToNode.cast(expr)); + return DatabendExpectedValueVisitor.asExpectedValues(expr); } - private List> generateGroupByClause(List columns, - DatabendRowValue rowValue) { + private List generateGroupByClause(List columns, DatabendRowValue rowValue) { if (Randomly.getBoolean()) { return columns.stream().map(c -> new DatabendColumnValue(c, rowValue.getValues().get(c))) .collect(Collectors.toList()); @@ -135,7 +131,7 @@ private List> generateGroupByClause(List generateLimit() { + private DatabendExpression generateLimit() { if (Randomly.getBoolean()) { return DatabendConstant.createIntConstant(Integer.MAX_VALUE); } else { @@ -143,7 +139,7 @@ private Node generateLimit() { } } - private Node generateOffset() { + private DatabendExpression generateOffset() { if (Randomly.getBoolean()) { return DatabendConstant.createIntConstant(0); } else { diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 043e93963..0ba0d9c86 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -8,27 +8,25 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendAggregateOperation; import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; +import sqlancer.databend.ast.DatabendAlias; import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; +import sqlancer.databend.ast.DatabendBinaryOperation; import sqlancer.databend.ast.DatabendCastOperation; import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendFunctionOperation; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase { @@ -49,9 +47,9 @@ public void check() throws SQLException { DatabendAggregateFunction aggregateFunction = Randomly.fromOptions(DatabendAggregateFunction.MAX, DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT, DatabendAggregateFunction.AVG/* , DatabendAggregateFunction.STDDEV_POP */); - NewFunctionNode aggregate = (DatabendAggregateOperation) gen + DatabendFunctionOperation aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); - List> fetchColumns = new ArrayList<>(); + List fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { fetchColumns.add((DatabendAggregateOperation) gen.generateAggregate()); // TODO 更换成非聚合函数 @@ -79,16 +77,14 @@ public void check() throws SQLException { } private String createMetamorphicUnionQuery(DatabendSelect select, - NewFunctionNode aggregate, - List> from) { + DatabendFunctionOperation aggregate, List from) { String metamorphicQuery; - Node whereClause = DatabendExprToNode - .cast(gen.generateExpression(DatabendDataType.BOOLEAN)); - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + DatabendExpression whereClause = gen.generateExpression(DatabendDataType.BOOLEAN); + DatabendExpression negatedClause = new DatabendUnaryPrefixOperation(whereClause, DatabendUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + DatabendExpression notNullClause = new DatabendUnaryPostfixOperation(whereClause, DatabendUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); + List mappedAggregate = mapped(aggregate); DatabendSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DatabendSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DatabendSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); @@ -126,8 +122,7 @@ private String getAggregateResult(String queryString) throws SQLException { } } - private List> mapped( - NewFunctionNode aggregate) { + private List mapped(DatabendFunctionOperation aggregate) { DatabendCastOperation count; switch (aggregate.getFunc()) { case COUNT: @@ -136,22 +131,21 @@ private List> mapped( case SUM: return aliasArgs(Arrays.asList(aggregate)); case AVG: - NewFunctionNode sum = new NewFunctionNode<>( + DatabendFunctionOperation sum = new DatabendFunctionOperation<>( aggregate.getArgs(), DatabendAggregateFunction.SUM); count = new DatabendCastOperation( - new NewFunctionNode<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), + new DatabendFunctionOperation<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); return aliasArgs(Arrays.asList(sum, count)); case STDDEV_POP: - NewFunctionNode sumSquared = new NewFunctionNode<>( - Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DatabendFunctionOperation sumSquared = new DatabendFunctionOperation<>( + Arrays.asList(new DatabendBinaryOperation(aggregate.getArgs().get(0), aggregate.getArgs().get(0), DatabendBinaryArithmeticOperator.MULTIPLICATION)), DatabendAggregateFunction.SUM); count = new DatabendCastOperation( - new NewFunctionNode(aggregate.getArgs(), - DatabendAggregateFunction.COUNT), + new DatabendFunctionOperation<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); - NewFunctionNode avg = new NewFunctionNode<>( + DatabendFunctionOperation avg = new DatabendFunctionOperation<>( aggregate.getArgs(), DatabendAggregateFunction.AVG); return aliasArgs(Arrays.asList(sumSquared, count, avg)); default: @@ -159,16 +153,16 @@ private List> mapped( } } - private List> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode(expr, "agg" + i++)); + for (DatabendExpression expr : originalAggregateArgs) { + args.add(new DatabendAlias(expr, "agg" + i++)); } return args; } - private String getOuterAggregateFunction(NewFunctionNode aggregate) { + private String getOuterAggregateFunction(DatabendFunctionOperation aggregate) { switch (aggregate.getFunc()) { case STDDEV_POP: return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; @@ -181,16 +175,16 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, - Node whereClause, List> joinList) { + private DatabendSelect getSelect(List aggregates, List from, + DatabendExpression whereClause, List joinList) { DatabendSelect select = new DatabendSelect(); select.setFetchColumns(aggregates); select.setFromList(from); select.setWhereClause(whereClause); select.setJoinList(joinList); if (Randomly.getBooleanWithSmallProbability()) { - select.setGroupByExpressions(List.of(DatabendExprToNode.cast(gen.generateConstant(DatabendDataType.INT)))); // TODO - // 仍可加强 + select.setGroupByExpressions(List.of(gen.generateConstant(DatabendDataType.INT))); // TODO + // 仍可加强 } return select; } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 0042f83d6..50a02f0ff 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -7,9 +7,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -20,10 +17,12 @@ import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.gen.DatabendNewExpressionGenerator; public class DatabendQueryPartitioningBase @@ -35,7 +34,7 @@ public class DatabendQueryPartitioningBase DatabendNewExpressionGenerator gen; DatabendSelect select; - List> groupByExpression; + List groupByExpression; public DatabendQueryPartitioningBase(DatabendGlobalState state) { super(state); @@ -57,33 +56,33 @@ public void check() throws SQLException { .addAll(randomColumn.stream().map(c -> new DatabendColumnValue(c, null)).collect(Collectors.toList())); groupByExpression = new ArrayList<>(columnOfLeafNode); - select.setFetchColumns(randomColumn.stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList())); + select.setFetchColumns( + randomColumn.stream().map(c -> new DatabendColumnReference(c)).collect(Collectors.toList())); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); if (!DatabendBugs.bug9236) { - List> joins = DatabendJoin.getJoins(tableList, state); + List joins = DatabendJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); } select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); + columns.add(new DatabendColumnReference(new DatabendColumn("*", null, false, false))); } else { columns = generateRandomColumns(); } return columns; } - List> generateRandomColumns() { - List> columns; - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateRandomColumns() { + List columns; + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DatabendColumnReference(c)) + .collect(Collectors.toList()); return columns; } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java index 430c5cfc6..ce20ef214 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java @@ -7,7 +7,6 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendToStringVisitor; @@ -31,11 +30,11 @@ public void check() throws SQLException { if (Randomly.getBoolean()) { select.setDistinct(false); } - select.setWhereClause(DatabendExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java index 6ec148ae8..9d45291b3 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java @@ -7,13 +7,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendExpression; public class DatabendQueryPartitioningGroupByTester extends DatabendQueryPartitioningBase { @@ -32,11 +29,11 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setWhereClause(DatabendExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, @@ -46,9 +43,9 @@ public void check() throws SQLException { } @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DatabendColumnReference(c)) + .collect(Collectors.toList()); } } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java index dfde7755f..33de8c896 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java @@ -7,9 +7,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; @@ -27,14 +25,13 @@ public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); if (Randomly.getBoolean()) { - select.setWhereClause( - DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } // boolean orderBy = Randomly.getBoolean(); boolean orderBy = false; // 关闭order by if (orderBy) { // TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select // select.setOrderByClauses(gen.generateOrderBys()); - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add(new DatabendConstant.DatabendIntConstant( Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); @@ -44,11 +41,11 @@ public void check() throws SQLException { String originalQueryString = DatabendToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setHavingClause(DatabendExprToNode.cast(predicate)); + select.setHavingClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setHavingClause(DatabendExprToNode.cast(negatedPredicate)); + select.setHavingClause(negatedPredicate); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setHavingClause(DatabendExprToNode.cast(isNullPredicate)); + select.setHavingClause(isNullPredicate); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, @@ -63,8 +60,8 @@ protected DatabendExpression generatePredicate() { } @Override - List> generateFetchColumns() { - return Collections.singletonList(DatabendExprToNode.cast(gen.generateHavingClause())); + List generateFetchColumns() { + return Collections.singletonList(gen.generateHavingClause()); } } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index d6fb7f0e6..6ea6f5e57 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -6,7 +6,6 @@ import sqlancer.ComparatorHelper; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendToStringVisitor; @@ -30,11 +29,11 @@ public void check() throws SQLException { // if (orderBy) { //TODO 待开启 // select.setOrderByClauses(gen.generateOrderBys()); // } - select.setWhereClause(DatabendExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, diff --git a/src/sqlancer/doris/visitor/DorisExprToNode.java b/src/sqlancer/doris/visitor/DorisExprToNode.java deleted file mode 100644 index cec7aa061..000000000 --- a/src/sqlancer/doris/visitor/DorisExprToNode.java +++ /dev/null @@ -1,25 +0,0 @@ -package sqlancer.doris.visitor; - -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.ast.newast.Node; -import sqlancer.doris.ast.DorisExpression; - -public final class DorisExprToNode { - - private DorisExprToNode() { - - } - - @SuppressWarnings("unchecked") - public static Node cast(DorisExpression expression) { - return (Node) expression; - } - - @SuppressWarnings("unchecked") - public static List> casts(List expressions) { - return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); - } - -} From ecd3d35ab577cf79824de903d053852e5c234ef5 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 26 Aug 2024 17:57:46 +0800 Subject: [PATCH 0900/1284] [Doris] Update expressions to match new AST --- .../doris/ast/DorisAggregateOperation.java | 8 +- src/sqlancer/doris/ast/DorisAlias.java | 9 ++ .../doris/ast/DorisBetweenOperation.java | 9 +- .../ast/DorisBinaryArithmeticOperation.java | 7 +- .../ast/DorisBinaryComparisonOperation.java | 7 +- .../ast/DorisBinaryLogicalOperation.java | 7 +- .../doris/ast/DorisBinaryOperation.java | 10 ++ .../doris/ast/DorisCaseOperation.java | 4 +- .../doris/ast/DorisCastOperation.java | 13 ++- .../doris/ast/DorisColumnReference.java | 11 ++ src/sqlancer/doris/ast/DorisConstant.java | 3 +- src/sqlancer/doris/ast/DorisFunction.java | 11 ++ .../doris/ast/DorisFunctionOperation.java | 3 +- src/sqlancer/doris/ast/DorisInOperation.java | 3 +- src/sqlancer/doris/ast/DorisJoin.java | 56 +++++----- .../doris/ast/DorisLikeOperation.java | 7 +- src/sqlancer/doris/ast/DorisOrderByTerm.java | 3 +- src/sqlancer/doris/ast/DorisPostfixText.java | 9 ++ src/sqlancer/doris/ast/DorisSelect.java | 3 +- .../doris/ast/DorisTableReference.java | 11 ++ .../doris/ast/DorisUnaryPostfixOperation.java | 5 +- .../doris/ast/DorisUnaryPrefixOperation.java | 5 +- .../doris/gen/DorisInsertGenerator.java | 6 +- .../gen/DorisNewExpressionGenerator.java | 6 +- .../gen/DorisRandomQuerySynthesizer.java | 22 ++-- .../doris/gen/DorisTableGenerator.java | 6 +- .../doris/gen/DorisUpdateGenerator.java | 3 +- .../doris/oracle/DorisNoRECOracle.java | 36 +++---- .../DorisPivotedQuerySynthesisOracle.java | 27 +++-- ...DorisQueryPartitioningAggregateTester.java | 62 +++++------ .../tlp/DorisQueryPartitioningBase.java | 23 ++-- .../DorisQueryPartitioningDistinctTester.java | 7 +- .../DorisQueryPartitioningGroupByTester.java | 17 ++- .../DorisQueryPartitioningHavingTester.java | 17 ++- .../DorisQueryPartitioningWhereTester.java | 10 +- .../visitor/DorisExpectedValueVisitor.java | 102 +++++++++--------- .../doris/visitor/DorisToStringVisitor.java | 25 ++--- 37 files changed, 284 insertions(+), 289 deletions(-) create mode 100644 src/sqlancer/doris/ast/DorisAlias.java create mode 100644 src/sqlancer/doris/ast/DorisBinaryOperation.java create mode 100644 src/sqlancer/doris/ast/DorisColumnReference.java create mode 100644 src/sqlancer/doris/ast/DorisFunction.java create mode 100644 src/sqlancer/doris/ast/DorisPostfixText.java create mode 100644 src/sqlancer/doris/ast/DorisTableReference.java diff --git a/src/sqlancer/doris/ast/DorisAggregateOperation.java b/src/sqlancer/doris/ast/DorisAggregateOperation.java index de818c26d..4c60985a8 100644 --- a/src/sqlancer/doris/ast/DorisAggregateOperation.java +++ b/src/sqlancer/doris/ast/DorisAggregateOperation.java @@ -3,14 +3,12 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.doris.visitor.DorisExprToNode; -public class DorisAggregateOperation extends - NewFunctionNode implements DorisExpression { +public class DorisAggregateOperation extends DorisFunction + implements DorisExpression { public DorisAggregateOperation(List args, DorisAggregateFunction func) { - super(DorisExprToNode.casts(args), func); + super(args, func); } public enum DorisAggregateFunction { diff --git a/src/sqlancer/doris/ast/DorisAlias.java b/src/sqlancer/doris/ast/DorisAlias.java new file mode 100644 index 000000000..74acf1cff --- /dev/null +++ b/src/sqlancer/doris/ast/DorisAlias.java @@ -0,0 +1,9 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class DorisAlias extends NewAliasNode implements DorisExpression { + public DorisAlias(DorisExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/doris/ast/DorisBetweenOperation.java b/src/sqlancer/doris/ast/DorisBetweenOperation.java index 76488393b..96c0b428c 100644 --- a/src/sqlancer/doris/ast/DorisBetweenOperation.java +++ b/src/sqlancer/doris/ast/DorisBetweenOperation.java @@ -2,23 +2,22 @@ import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.doris.DorisSchema; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisBetweenOperation extends NewBetweenOperatorNode implements DorisExpression { public DorisBetweenOperation(DorisExpression left, DorisExpression middle, DorisExpression right, boolean isTrue) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(middle), DorisExprToNode.cast(right), isTrue); + super(left, middle, right, isTrue); } public DorisExpression getLeftExpr() { - return (DorisExpression) left; + return left; } public DorisExpression getMiddleExpr() { - return (DorisExpression) middle; + return middle; } public DorisExpression getRightExpr() { - return (DorisExpression) right; + return right; } @Override diff --git a/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java index 244404569..2e148c488 100644 --- a/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java +++ b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java @@ -5,12 +5,11 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisBinaryArithmeticOperation extends NewBinaryOperatorNode implements DorisExpression { public DorisBinaryArithmeticOperation(DorisExpression left, DorisExpression right, BinaryOperatorNode.Operator op) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + super(left, right, op); } public enum DorisBinaryArithmeticOperator implements BinaryOperatorNode.Operator { @@ -114,11 +113,11 @@ public String getTextRepresentation() { } public DorisExpression getLeftExpr() { - return (DorisExpression) super.getLeft(); + return super.getLeft(); } public DorisExpression getRightExpr() { - return (DorisExpression) super.getRight(); + return super.getRight(); } public DorisBinaryArithmeticOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java index 894d2ab9b..13355c7ff 100644 --- a/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java +++ b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java @@ -3,21 +3,20 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisBinaryComparisonOperation extends NewBinaryOperatorNode implements DorisExpression { public DorisBinaryComparisonOperation(DorisExpression left, DorisExpression right, DorisBinaryComparisonOperator op) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + super(left, right, op); } public DorisExpression getLeftExpression() { - return (DorisExpression) super.getLeft(); + return super.getLeft(); } public DorisExpression getRightExpression() { - return (DorisExpression) super.getRight(); + return super.getRight(); } public DorisBinaryComparisonOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java index b62ccd61b..858c3bf2f 100644 --- a/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java +++ b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java @@ -4,20 +4,19 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisBinaryLogicalOperation extends NewBinaryOperatorNode implements DorisExpression { public DorisBinaryLogicalOperation(DorisExpression left, DorisExpression right, DorisBinaryLogicalOperator op) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + super(left, right, op); } public DorisExpression getLeftExpr() { - return (DorisExpression) super.getLeft(); + return super.getLeft(); } public DorisExpression getRightExpr() { - return (DorisExpression) super.getRight(); + return super.getRight(); } public DorisBinaryLogicalOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisBinaryOperation.java b/src/sqlancer/doris/ast/DorisBinaryOperation.java new file mode 100644 index 000000000..ffe80b86d --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DorisBinaryOperation extends NewBinaryOperatorNode implements DorisExpression { + public DorisBinaryOperation(DorisExpression left, DorisExpression right, BinaryOperatorNode.Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/doris/ast/DorisCaseOperation.java b/src/sqlancer/doris/ast/DorisCaseOperation.java index e852e36bd..02641408a 100644 --- a/src/sqlancer/doris/ast/DorisCaseOperation.java +++ b/src/sqlancer/doris/ast/DorisCaseOperation.java @@ -2,9 +2,7 @@ import java.util.List; -import sqlancer.common.ast.newast.Node; - -public class DorisCaseOperation implements Node, DorisExpression { +public class DorisCaseOperation implements DorisExpression { private final DorisExpression expr; private final List conditions; diff --git a/src/sqlancer/doris/ast/DorisCastOperation.java b/src/sqlancer/doris/ast/DorisCastOperation.java index 1aacdaf06..e93b8b226 100644 --- a/src/sqlancer/doris/ast/DorisCastOperation.java +++ b/src/sqlancer/doris/ast/DorisCastOperation.java @@ -1,29 +1,28 @@ package sqlancer.doris.ast; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisSchema.DorisCompositeDataType; import sqlancer.doris.DorisSchema.DorisDataType; -public class DorisCastOperation implements Node, DorisExpression { - Node expr; +public class DorisCastOperation implements DorisExpression { + DorisExpression expr; DorisDataType type; - public DorisCastOperation(Node expr, DorisCompositeDataType type) { + public DorisCastOperation(DorisExpression expr, DorisCompositeDataType type) { this.expr = expr; this.type = type.getPrimitiveDataType(); } - public DorisCastOperation(Node expr, DorisDataType type) { + public DorisCastOperation(DorisExpression expr, DorisDataType type) { this.expr = expr; this.type = type; } - public Node getExpr() { + public DorisExpression getExpr() { return expr; } public DorisExpression getExpression() { - return (DorisExpression) expr; + return expr; } public DorisDataType getType() { diff --git a/src/sqlancer/doris/ast/DorisColumnReference.java b/src/sqlancer/doris/ast/DorisColumnReference.java new file mode 100644 index 000000000..ce6aee058 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.doris.DorisSchema; + +public class DorisColumnReference extends ColumnReferenceNode + implements DorisExpression { + public DorisColumnReference(DorisSchema.DorisColumn column) { + super(column); + } +} diff --git a/src/sqlancer/doris/ast/DorisConstant.java b/src/sqlancer/doris/ast/DorisConstant.java index 37374a4d7..2ad2500ed 100644 --- a/src/sqlancer/doris/ast/DorisConstant.java +++ b/src/sqlancer/doris/ast/DorisConstant.java @@ -1,10 +1,9 @@ package sqlancer.doris.ast; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.utils.DorisNumberUtils; -public abstract class DorisConstant implements Node, DorisExpression { +public abstract class DorisConstant implements DorisExpression { private DorisConstant() { } diff --git a/src/sqlancer/doris/ast/DorisFunction.java b/src/sqlancer/doris/ast/DorisFunction.java new file mode 100644 index 000000000..1f816a7e3 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisFunction.java @@ -0,0 +1,11 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DorisFunction extends NewFunctionNode implements DorisExpression { + public DorisFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/doris/ast/DorisFunctionOperation.java b/src/sqlancer/doris/ast/DorisFunctionOperation.java index 0a75892d0..05f5370b1 100644 --- a/src/sqlancer/doris/ast/DorisFunctionOperation.java +++ b/src/sqlancer/doris/ast/DorisFunctionOperation.java @@ -6,11 +6,10 @@ import java.util.stream.Stream; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.gen.DorisNewExpressionGenerator; -public class DorisFunctionOperation implements Node, DorisExpression { +public class DorisFunctionOperation implements DorisExpression { private DorisFunction function; private List args; diff --git a/src/sqlancer/doris/ast/DorisInOperation.java b/src/sqlancer/doris/ast/DorisInOperation.java index 31dcbe1ec..96ccd0998 100644 --- a/src/sqlancer/doris/ast/DorisInOperation.java +++ b/src/sqlancer/doris/ast/DorisInOperation.java @@ -4,7 +4,6 @@ import sqlancer.common.ast.newast.NewInOperatorNode; import sqlancer.doris.DorisSchema; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisInOperation extends NewInOperatorNode implements DorisExpression { @@ -12,7 +11,7 @@ public class DorisInOperation extends NewInOperatorNode impleme private final List rightExpr; public DorisInOperation(DorisExpression left, List right, boolean isNegated) { - super(DorisExprToNode.cast(left), DorisExprToNode.casts(right), isNegated); + super(left, right, isNegated); this.leftExpr = left; this.rightExpr = right; } diff --git a/src/sqlancer/doris/ast/DorisJoin.java b/src/sqlancer/doris/ast/DorisJoin.java index 01274449f..7c7a35c18 100644 --- a/src/sqlancer/doris/ast/DorisJoin.java +++ b/src/sqlancer/doris/ast/DorisJoin.java @@ -4,21 +4,17 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisColumn; -import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.gen.DorisNewExpressionGenerator; -import sqlancer.doris.visitor.DorisExprToNode; -public class DorisJoin implements Node { +public class DorisJoin implements DorisExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final DorisTableReference leftTable; + private final DorisTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final DorisExpression onCondition; public enum JoinType { INNER, STRAIGHT, LEFT, RIGHT; @@ -28,20 +24,19 @@ public static JoinType getRandom() { } } - public DorisJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public DorisJoin(DorisTableReference leftTable, DorisTableReference rightTable, JoinType joinType, + DorisExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public TableReferenceNode getLeftTable() { + public DorisTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public DorisTableReference getRightTable() { return rightTable; } @@ -49,35 +44,34 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public DorisExpression getOnCondition() { return onCondition; } - public static List> getJoins(List> tableList, - DorisGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DorisGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + DorisTableReference leftTable = tableList.remove(0); + DorisTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); DorisNewExpressionGenerator joinGen = new DorisNewExpressionGenerator(globalState).setColumns(columns); switch (DorisJoin.JoinType.getRandom()) { case INNER: joinExpressions.add(DorisJoin.createInnerJoin(leftTable, rightTable, - DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); break; case STRAIGHT: joinExpressions.add(DorisJoin.createStraightJoin(leftTable, rightTable, - DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); break; case LEFT: joinExpressions.add(DorisJoin.createLeftOuterJoin(leftTable, rightTable, - DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); break; case RIGHT: joinExpressions.add(DorisJoin.createRightOuterJoin(leftTable, rightTable, - DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); break; default: throw new AssertionError(); @@ -86,23 +80,23 @@ public static List> getJoins(List left, - TableReferenceNode right, Node predicate) { + public static DorisJoin createInnerJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { return new DorisJoin(left, right, JoinType.INNER, predicate); } - public static DorisJoin createStraightJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DorisJoin createStraightJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { return new DorisJoin(left, right, JoinType.STRAIGHT, predicate); } - public static DorisJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DorisJoin createRightOuterJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { return new DorisJoin(left, right, JoinType.RIGHT, predicate); } - public static DorisJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DorisJoin createLeftOuterJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { return new DorisJoin(left, right, JoinType.LEFT, predicate); } } diff --git a/src/sqlancer/doris/ast/DorisLikeOperation.java b/src/sqlancer/doris/ast/DorisLikeOperation.java index a3251ca7f..128a9299d 100644 --- a/src/sqlancer/doris/ast/DorisLikeOperation.java +++ b/src/sqlancer/doris/ast/DorisLikeOperation.java @@ -5,12 +5,11 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisLikeOperation extends NewBinaryOperatorNode implements DorisExpression { public DorisLikeOperation(DorisExpression left, DorisExpression right, DorisLikeOperator op) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + super(left, right, op); } @Override @@ -19,11 +18,11 @@ public DorisDataType getExpectedType() { } public DorisExpression getLeftExpr() { - return (DorisExpression) super.getLeft(); + return super.getLeft(); } public DorisExpression getRightExpr() { - return (DorisExpression) super.getRight(); + return super.getRight(); } public DorisLikeOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisOrderByTerm.java b/src/sqlancer/doris/ast/DorisOrderByTerm.java index 6207f4af2..01a7ab6d2 100644 --- a/src/sqlancer/doris/ast/DorisOrderByTerm.java +++ b/src/sqlancer/doris/ast/DorisOrderByTerm.java @@ -1,10 +1,9 @@ package sqlancer.doris.ast; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.Node; public class DorisOrderByTerm extends NewOrderingTerm implements DorisExpression { - public DorisOrderByTerm(Node expr, Ordering ordering) { + public DorisOrderByTerm(DorisExpression expr, Ordering ordering) { super(expr, ordering); } } diff --git a/src/sqlancer/doris/ast/DorisPostfixText.java b/src/sqlancer/doris/ast/DorisPostfixText.java new file mode 100644 index 000000000..889f4700b --- /dev/null +++ b/src/sqlancer/doris/ast/DorisPostfixText.java @@ -0,0 +1,9 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class DorisPostfixText extends NewPostfixTextNode implements DorisExpression { + public DorisPostfixText(DorisExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/doris/ast/DorisSelect.java b/src/sqlancer/doris/ast/DorisSelect.java index 283d474cc..6194a0498 100644 --- a/src/sqlancer/doris/ast/DorisSelect.java +++ b/src/sqlancer/doris/ast/DorisSelect.java @@ -2,9 +2,8 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class DorisSelect extends SelectBase> implements Node { +public class DorisSelect extends SelectBase implements DorisExpression { public enum DorisSelectDistinctType { diff --git a/src/sqlancer/doris/ast/DorisTableReference.java b/src/sqlancer/doris/ast/DorisTableReference.java new file mode 100644 index 000000000..b8a2cc2a2 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.doris.DorisSchema; + +public class DorisTableReference extends TableReferenceNode + implements DorisExpression { + public DorisTableReference(DorisSchema.DorisTable table) { + super(table); + } +} diff --git a/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java index 9a34e2c54..f6961c7b7 100644 --- a/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java +++ b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java @@ -4,17 +4,16 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements DorisExpression { public DorisUnaryPostfixOperation(DorisExpression expr, DorisUnaryPostfixOperator op) { - super(DorisExprToNode.cast(expr), op); + super(expr, op); } public DorisExpression getExpression() { - return (DorisExpression) getExpr(); + return getExpr(); } public DorisUnaryPostfixOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java index e7f7d91b0..ffd919fdc 100644 --- a/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java +++ b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java @@ -4,16 +4,15 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements DorisExpression { public DorisUnaryPrefixOperation(DorisExpression expr, DorisUnaryPrefixOperator op) { - super(DorisExprToNode.cast(expr), op); + super(expr, op); } public DorisExpression getExpression() { - return (DorisExpression) getExpr(); + return getExpr(); } public DorisUnaryPrefixOperator getOp() { diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java index 7ce4d2e60..50dc5cdec 100644 --- a/src/sqlancer/doris/gen/DorisInsertGenerator.java +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -11,7 +11,6 @@ import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisTable; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisInsertGenerator extends AbstractInsertGenerator { @@ -46,9 +45,8 @@ protected void insertValue(DorisColumn column) { if (column.hasDefaultValue() && Randomly.getBooleanWithRatherLowProbability()) { sb.append("DEFAULT"); } else { - String value = DorisToStringVisitor - .asString(DorisExprToNode.cast(new DorisNewExpressionGenerator(globalState) - .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); // 生成一个与column相同的常量类型 + String value = DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 sb.append(value); } } diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java index f283d0c39..13fe028ca 100644 --- a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -10,7 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.doris.DorisBugs; import sqlancer.doris.DorisProvider.DorisGlobalState; @@ -39,7 +38,6 @@ import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisNewExpressionGenerator extends TypedExpressionGenerator { @@ -97,7 +95,7 @@ private DorisExpression createColumnOfType(DorisDataType type) { return DorisColumnValue.create(column, value); } - public List> generateOrderBy() { + public List generateOrderBy() { List randomColumns = Randomly.subset(columns); return randomColumns.stream() .map(c -> new DorisOrderByTerm(new DorisColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) @@ -121,7 +119,7 @@ public DorisExpression generateExpression(DorisDataType type, int depth) { } if (!DorisBugs.bug36070 && type != DorisDataType.NULL && globalState.getDbmsSpecificOptions().testCasts && Randomly.getBooleanWithRatherLowProbability()) { - return new DorisCastOperation(DorisExprToNode.cast(generateExpression(getRandomType(), depth + 1)), type); + return new DorisCastOperation(generateExpression(getRandomType(), depth + 1), type); } if (!DorisBugs.bug36070 && globalState.getDbmsSpecificOptions().testCase && Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java index 1b85d4b34..518a204a5 100644 --- a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java +++ b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java @@ -6,8 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisTable; @@ -17,7 +15,7 @@ import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisJoin; import sqlancer.doris.ast.DorisSelect; -import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.ast.DorisTableReference; public final class DorisRandomQuerySynthesizer { @@ -33,33 +31,33 @@ public static DorisSelect generateSelect(DorisGlobalState globalState, int nrCol gen.setColumnOfLeafNode(columnOfLeafNode); int freeColumns = targetColumns.size(); select.setDistinct(DorisSelect.DorisSelectDistinctType.getRandomWithoutNull()); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { - Node column = null; + DorisExpression column = null; if (freeColumns > 0 && Randomly.getBoolean()) { column = new DorisColumnValue(targetColumns.get(freeColumns - 1), null); freeColumns -= 1; columnOfLeafNode.add((DorisColumnValue) column); } else { - column = DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); + column = gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN); } columns.add(column); } select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DorisJoin.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { - select.setHavingClause(DorisExprToNode.cast(gen.generateHavingClause())); + select.setHavingClause(gen.generateHavingClause()); } if (Randomly.getBoolean()) { - select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + select.setWhereClause(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); } - List> noExprColumns = new ArrayList<>(columnOfLeafNode); + List noExprColumns = new ArrayList<>(columnOfLeafNode); if (Randomly.getBoolean()) { select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); diff --git a/src/sqlancer/doris/gen/DorisTableGenerator.java b/src/sqlancer/doris/gen/DorisTableGenerator.java index 8a98739f5..74d2f956a 100644 --- a/src/sqlancer/doris/gen/DorisTableGenerator.java +++ b/src/sqlancer/doris/gen/DorisTableGenerator.java @@ -15,7 +15,6 @@ import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisCompositeDataType; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisTableGenerator { @@ -101,9 +100,8 @@ private static List getNewColumns(DorisGlobalState globalState) { && !isHllOrBitmap; String defaultValue = ""; if (hasDefaultValue) { - defaultValue = DorisToStringVisitor - .asString(DorisExprToNode.cast(new DorisNewExpressionGenerator(globalState) - .generateConstant(columnType.getPrimitiveDataType(), isNullable))); + defaultValue = DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + .generateConstant(columnType.getPrimitiveDataType(), isNullable)); } columns.add(new DorisColumn(columnName, columnType, iskey, isNullable, aggrType, hasDefaultValue, defaultValue)); diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java index 74fb9383d..906173921 100644 --- a/src/sqlancer/doris/gen/DorisUpdateGenerator.java +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -11,7 +11,6 @@ import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public final class DorisUpdateGenerator extends AbstractUpdateGenerator { @@ -45,7 +44,7 @@ private SQLQueryAdapter generate() { protected void updateValue(DorisColumn column) { if (Randomly.getBooleanWithSmallProbability()) { DorisExpression expr = gen.generateExpression(column.getType().getPrimitiveDataType()); - sb.append(DorisToStringVisitor.asString(DorisExprToNode.cast(expr))); + sb.append(DorisToStringVisitor.asString(expr)); } else { DorisExpression expr = gen.generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()); sb.append(DorisToStringVisitor.asString(expr)); diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java index 01fa11f2f..8840a299e 100644 --- a/src/sqlancer/doris/oracle/DorisNoRECOracle.java +++ b/src/sqlancer/doris/oracle/DorisNoRECOracle.java @@ -11,10 +11,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -28,12 +24,14 @@ import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.DorisSchema.DorisTables; import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisPostfixText; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.gen.DorisNewExpressionGenerator; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisNoRECOracle extends NoRECBase implements TestOracle { @@ -52,12 +50,11 @@ public void check() throws SQLException { DorisTables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = DorisExprToNode - .cast(gen.generateExpression(DorisDataType.BOOLEAN)); + DorisExpression randomWhereCondition = gen.generateExpression(DorisDataType.BOOLEAN); List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DorisJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, state); int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, @@ -71,11 +68,11 @@ public void check() throws SQLException { } } - private int getUnoptimizedQueryCount(List> tableList, - Node randomWhereCondition, List> joins) throws SQLException { + private int getUnoptimizedQueryCount(List tableList, DorisExpression randomWhereCondition, + List joins) throws SQLException { DorisSelect select = new DorisSelect(); - Node asText = new NewPostfixTextNode<>(new DorisCastOperation( - new NewPostfixTextNode(randomWhereCondition, + DorisExpression asText = new DorisPostfixText(new DorisCastOperation( + new DorisPostfixText(randomWhereCondition, " IS NOT NULL AND " + DorisToStringVisitor.asString(randomWhereCondition)), new DorisCompositeDataType(DorisDataType.INT, 8)), "as count"); select.setFetchColumns(Arrays.asList(asText)); @@ -100,18 +97,17 @@ private int getUnoptimizedQueryCount(List> tableList, return secondCount; } - private int getOptimizedQueryCount(SQLConnection con, List> tableList, - List columns, Node randomWhereCondition, List> joins) - throws SQLException { + private int getOptimizedQueryCount(SQLConnection con, List tableList, List columns, + DorisExpression randomWhereCondition, List joins) throws SQLException { DorisSelect select = new DorisSelect(); // select.setGroupByClause(groupBys); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List allColumns = columns.stream().map((c) -> new DorisColumnReference(c)) + .collect(Collectors.toList()); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); diff --git a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java index 4d8f3c012..a7e8b5b02 100644 --- a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java @@ -7,8 +7,6 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; @@ -17,17 +15,16 @@ import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.DorisSchema.DorisRowValue; -import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.DorisSchema.DorisTables; import sqlancer.doris.ast.DorisColumnValue; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.ast.DorisUnaryPostfixOperation; import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.gen.DorisNewExpressionGenerator; import sqlancer.doris.visitor.DorisExpectedValueVisitor; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisPivotedQuerySynthesisOracle @@ -53,22 +50,22 @@ protected Query getRectifiedQuery() throws Exception { selectStatement.setFetchColumns(fetchColumns.stream() .map(c -> new DorisColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) .collect(Collectors.toList())); - selectStatement.setFromList(randomTables.getTables().stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList())); + selectStatement.setFromList( + randomTables.getTables().stream().map(t -> new DorisTableReference(t)).collect(Collectors.toList())); DorisExpression whereClause = generateRectifiedExpression(columns, pivotRow); - selectStatement.setWhereClause(DorisExprToNode.cast(whereClause)); - List> groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); - Node limitClause = generateLimit(); + DorisExpression limitClause = generateLimit(); selectStatement.setLimitClause(limitClause); if (limitClause != null) { - Node offsetClause = generateOffset(); + DorisExpression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); gen.setColumns(columns); if (!isDistinct) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add(new DorisConstant.DorisIntConstant( Randomly.smallNumber() % selectStatement.getFetchColumns().size() + 1)); selectStatement.setOrderByClauses(constants); @@ -124,10 +121,10 @@ private DorisColumn getFetchValueAliasedColumn(DorisColumn c) { @Override protected String getExpectedValues(DorisExpression expr) { - return DorisExpectedValueVisitor.asExpectedValues(DorisExprToNode.cast(expr)); + return DorisExpectedValueVisitor.asExpectedValues(expr); } - private List> generateGroupByClause(List columns, DorisRowValue rowValue) { + private List generateGroupByClause(List columns, DorisRowValue rowValue) { if (Randomly.getBoolean()) { return columns.stream().map(c -> new DorisColumnValue(c, rowValue.getValues().get(c))) .collect(Collectors.toList()); @@ -136,7 +133,7 @@ private List> generateGroupByClause(List colu } } - private Node generateLimit() { + private DorisExpression generateLimit() { if (Randomly.getBoolean()) { return DorisConstant.createIntConstant(Integer.MAX_VALUE); } else { @@ -144,7 +141,7 @@ private Node generateLimit() { } } - private Node generateOffset() { + private DorisExpression generateOffset() { if (Randomly.getBoolean()) { return DorisConstant.createIntConstant(0); } else { diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java index 1f3fe66f1..b3289e5ad 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -8,12 +8,6 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; @@ -22,15 +16,19 @@ import sqlancer.doris.DorisSchema.DorisCompositeDataType; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.ast.DorisAggregateOperation; +import sqlancer.doris.ast.DorisAlias; import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; import sqlancer.doris.ast.DorisBinaryArithmeticOperation; +import sqlancer.doris.ast.DorisBinaryOperation; import sqlancer.doris.ast.DorisCastOperation; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunction; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningAggregateTester extends DorisQueryPartitioningBase @@ -53,16 +51,16 @@ public void check() throws SQLException { DorisAggregateFunction aggregateFunction = Randomly.fromOptions(DorisAggregateFunction.MAX, DorisAggregateFunction.MIN, DorisAggregateFunction.SUM, DorisAggregateFunction.COUNT, DorisAggregateFunction.AVG); - NewFunctionNode aggregate = (DorisAggregateOperation) gen + DorisFunction aggregate = (DorisAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); - List> fetchColumns = new ArrayList<>(); + List fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { fetchColumns.add((DorisAggregateOperation) gen.generateAggregate()); } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); @@ -94,15 +92,13 @@ public void check() throws SQLException { } - private String createMetamorphicUnionQuery(DorisSelect select, - NewFunctionNode aggregate, List> from) { + private String createMetamorphicUnionQuery(DorisSelect select, DorisFunction aggregate, + List from) { String metamorphicQuery; - Node whereClause = DorisExprToNode.cast(gen.generateExpression(DorisDataType.BOOLEAN)); - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, - DorisUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, - DorisUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); + DorisExpression whereClause = gen.generateExpression(DorisDataType.BOOLEAN); + DorisExpression negatedClause = new DorisUnaryPrefixOperation(whereClause, DorisUnaryPrefixOperator.NOT); + DorisExpression notNullClause = new DorisUnaryPostfixOperation(whereClause, DorisUnaryPostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); DorisSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DorisSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DorisSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); @@ -141,7 +137,7 @@ private String getAggregateResult(String queryString) throws SQLException { } } - private List> mapped(NewFunctionNode aggregate) { + private List mapped(DorisFunction aggregate) { DorisCastOperation count; switch (aggregate.getFunc()) { @@ -151,21 +147,19 @@ private List> mapped(NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + DorisFunction sum = new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.SUM); - count = new DorisCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), + count = new DorisCastOperation(new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), new DorisCompositeDataType(DorisDataType.FLOAT, 8)); return aliasArgs(Arrays.asList(sum, count)); case STDDEV_POP: - NewFunctionNode sumSquared = new NewFunctionNode<>( - Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DorisFunction sumSquared = new DorisFunction<>( + Arrays.asList(new DorisBinaryOperation(aggregate.getArgs().get(0), aggregate.getArgs().get(0), DorisBinaryArithmeticOperation.DorisBinaryArithmeticOperator.MULTIPLICATION)), DorisAggregateFunction.SUM); - count = new DorisCastOperation( - new NewFunctionNode(aggregate.getArgs(), - DorisAggregateFunction.COUNT), + count = new DorisCastOperation(new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), new DorisCompositeDataType(DorisDataType.FLOAT, 8)); - NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), + DorisFunction avg = new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.AVG); return aliasArgs(Arrays.asList(sumSquared, count, avg)); default: @@ -173,16 +167,16 @@ private List> mapped(NewFunctionNode> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode(expr, "agg" + i++)); + for (DorisExpression expr : originalAggregateArgs) { + args.add(new DorisAlias(expr, "agg" + i++)); } return args; } - private String getOuterAggregateFunction(NewFunctionNode aggregate) { + private String getOuterAggregateFunction(DorisFunction aggregate) { switch (aggregate.getFunc()) { case STDDEV_POP: return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; @@ -195,8 +189,8 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, - Node whereClause, List> joinList) { + private DorisSelect getSelect(List aggregates, List from, + DorisExpression whereClause, List joinList) { DorisSelect leftSelect = new DorisSelect(); leftSelect.setFetchColumns(aggregates); leftSelect.setFromList(from); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java index defc76a56..b3d3a7a6b 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java @@ -7,9 +7,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -19,10 +16,12 @@ import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisColumnValue; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisJoin; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.gen.DorisNewExpressionGenerator; public class DorisQueryPartitioningBase extends TernaryLogicPartitioningOracleBase @@ -33,7 +32,7 @@ public class DorisQueryPartitioningBase extends TernaryLogicPartitioningOracleBa DorisNewExpressionGenerator gen; DorisSelect select; - List> groupByExpression; + List groupByExpression; public DorisQueryPartitioningBase(DorisGlobalState state) { super(state); @@ -56,21 +55,21 @@ public void check() throws SQLException { groupByExpression = new ArrayList<>(allColumnValues); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DorisJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new DorisColumn("*", null, false, false))); + columns.add(new DorisColumnReference(new DorisColumn("*", null, false, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DorisColumnReference(c)) + .collect(Collectors.toList()); } return columns; } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java index 3bce110ff..dc350ea6f 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java @@ -7,7 +7,6 @@ import sqlancer.ComparatorHelper; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningDistinctTester extends DorisQueryPartitioningBase { @@ -26,11 +25,11 @@ public void check() throws SQLException { String originalQueryString = DorisToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setWhereClause(DorisExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DorisToStringVisitor.asString(select); List combinedString = new ArrayList<>(); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java index 8e173d7ae..97ade18e3 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java @@ -7,13 +7,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningGroupByTester extends DorisQueryPartitioningBase { @@ -33,11 +30,11 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setWhereClause(DorisExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DorisToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, @@ -47,9 +44,9 @@ public void check() throws SQLException { } @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DorisColumnReference(c)) + .collect(Collectors.toList()); } } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java index b3398f229..434ee01d5 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -6,14 +6,12 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningHavingTester extends DorisQueryPartitioningBase @@ -29,12 +27,12 @@ public DorisQueryPartitioningHavingTester(DorisGlobalState state) { public void check() throws SQLException { super.check(); if (Randomly.getBoolean()) { - select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + select.setWhereClause(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); } select.setFetchColumns(groupByExpression); boolean orderBy = Randomly.getBoolean(); if (orderBy) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); @@ -44,11 +42,11 @@ public void check() throws SQLException { String originalQueryString = DorisToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setHavingClause(DorisExprToNode.cast(predicate)); + select.setHavingClause(predicate); String firstQueryString = DorisToStringVisitor.asString(select); - select.setHavingClause(DorisExprToNode.cast(negatedPredicate)); + select.setHavingClause(negatedPredicate); String secondQueryString = DorisToStringVisitor.asString(select); - select.setHavingClause(DorisExprToNode.cast(isNullPredicate)); + select.setHavingClause(isNullPredicate); String thirdQueryString = DorisToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, @@ -63,10 +61,9 @@ protected DorisExpression generatePredicate() { } @Override - List> generateFetchColumns() { + List generateFetchColumns() { gen.setAllowAggregateFunctions(true); - List> expressions = DorisExprToNode - .casts(gen.generateExpressions(Randomly.smallNumber() + 1)); + List expressions = gen.generateExpressions(Randomly.smallNumber() + 1); gen.setAllowAggregateFunctions(false); return expressions; } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java index 34ade1d3a..5a593a573 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -6,12 +6,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningWhereTester extends DorisQueryPartitioningBase { @@ -32,16 +30,16 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); } - select.setWhereClause(DorisExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DorisToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, diff --git a/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java index 57cb29ae0..10c90e47e 100644 --- a/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java +++ b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java @@ -2,60 +2,57 @@ import java.util.List; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.ast.DorisAlias; +import sqlancer.doris.ast.DorisBetweenOperation; +import sqlancer.doris.ast.DorisBinaryOperation; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunction; +import sqlancer.doris.ast.DorisInOperation; import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisOrderByTerm; +import sqlancer.doris.ast.DorisPostfixText; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; public class DorisExpectedValueVisitor { protected final StringBuilder sb = new StringBuilder(); - private void print(Node expr) { + private void print(DorisExpression expr) { sb.append(DorisToStringVisitor.asString(expr)); sb.append(" -- "); sb.append(((DorisExpression) expr).getExpectedValue()); sb.append("\n"); } - @SuppressWarnings("unchecked") - public void visit(Node expr) { + public void visit(DorisExpression expr) { assert expr != null; - if (expr instanceof ColumnReferenceNode) { - visit((ColumnReferenceNode) expr); - } else if (expr instanceof NewUnaryPostfixOperatorNode) { - visit((NewUnaryPostfixOperatorNode) expr); - } else if (expr instanceof NewUnaryPrefixOperatorNode) { - visit((NewUnaryPrefixOperatorNode) expr); - } else if (expr instanceof NewBinaryOperatorNode) { - visit((NewBinaryOperatorNode) expr); - } else if (expr instanceof TableReferenceNode) { - visit((TableReferenceNode) expr); - } else if (expr instanceof NewFunctionNode) { - visit((NewFunctionNode) expr); - } else if (expr instanceof NewBetweenOperatorNode) { - visit((NewBetweenOperatorNode) expr); - } else if (expr instanceof NewInOperatorNode) { - visit((NewInOperatorNode) expr); - } else if (expr instanceof NewOrderingTerm) { - visit((NewOrderingTerm) expr); - } else if (expr instanceof NewAliasNode) { - visit((NewAliasNode) expr); - } else if (expr instanceof NewPostfixTextNode) { - visit((NewPostfixTextNode) expr); + if (expr instanceof DorisColumnReference) { + visit((DorisColumnReference) expr); + } else if (expr instanceof DorisUnaryPostfixOperation) { + visit((DorisUnaryPostfixOperation) expr); + } else if (expr instanceof DorisUnaryPrefixOperation) { + visit((DorisUnaryPrefixOperation) expr); + } else if (expr instanceof DorisBinaryOperation) { + visit((DorisBinaryOperation) expr); + } else if (expr instanceof DorisTableReference) { + visit((DorisTableReference) expr); + } else if (expr instanceof DorisFunction) { + visit((DorisFunction) expr); + } else if (expr instanceof DorisBetweenOperation) { + visit((DorisBetweenOperation) expr); + } else if (expr instanceof DorisInOperation) { + visit((DorisInOperation) expr); + } else if (expr instanceof DorisOrderByTerm) { + visit((DorisOrderByTerm) expr); + } else if (expr instanceof DorisAlias) { + visit((DorisAlias) expr); + } else if (expr instanceof DorisPostfixText) { + visit((DorisPostfixText) expr); } else if (expr instanceof DorisConstant) { visit((DorisConstant) expr); } else if (expr instanceof DorisSelect) { @@ -67,65 +64,64 @@ public void visit(Node expr) { } } - public void visit(ColumnReferenceNode c) { + public void visit(DorisColumnReference c) { print(c); } - public void visit(NewUnaryPostfixOperatorNode op) { + public void visit(DorisUnaryPostfixOperation op) { print(op); visit(op.getExpr()); } - public void visit(NewUnaryPrefixOperatorNode op) { + public void visit(DorisUnaryPrefixOperation op) { print(op); visit(op.getExpr()); } - public void visit(NewBinaryOperatorNode op) { - print(op); + public void visit(DorisBinaryOperation op) { visit(op.getLeft()); visit(op.getRight()); } - public void visit(TableReferenceNode t) { + public void visit(DorisTableReference t) { print(t); } - public void visit(NewFunctionNode fun) { + public void visit(DorisFunction fun) { print(fun); visit(fun.getArgs()); } - public void visit(List> expressions) { - for (Node expression : expressions) { + public void visit(List expressions) { + for (DorisExpression expression : expressions) { visit(expression); } } - public void visit(NewBetweenOperatorNode op) { + public void visit(DorisBetweenOperation op) { print(op); visit(op.getLeft()); visit(op.getMiddle()); visit(op.getRight()); } - public void visit(NewInOperatorNode op) { + public void visit(DorisInOperation op) { print(op); visit(op.getLeft()); visit(op.getRight()); } - public void visit(NewOrderingTerm op) { + public void visit(DorisOrderByTerm op) { print(op); visit(op.getExpr()); } - public void visit(NewAliasNode op) { + public void visit(DorisAlias op) { print(op); visit(op.getExpr()); } - public void visit(NewPostfixTextNode postFixText) { + public void visit(DorisPostfixText postFixText) { print(postFixText); visit(postFixText.getExpr()); } @@ -146,7 +142,7 @@ public String get() { return sb.toString(); } - public static String asExpectedValues(Node expr) { + public static String asExpectedValues(DorisExpression expr) { DorisExpectedValueVisitor v = new DorisExpectedValueVisitor(); v.visit(expr); return v.get(); diff --git a/src/sqlancer/doris/visitor/DorisToStringVisitor.java b/src/sqlancer/doris/visitor/DorisToStringVisitor.java index f7770058f..fcea9ab69 100644 --- a/src/sqlancer/doris/visitor/DorisToStringVisitor.java +++ b/src/sqlancer/doris/visitor/DorisToStringVisitor.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.ast.DorisCaseOperation; import sqlancer.doris.ast.DorisCastOperation; import sqlancer.doris.ast.DorisConstant; @@ -14,7 +13,7 @@ public class DorisToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(DorisExpression expr) { if (expr instanceof DorisConstant) { visit((DorisConstant) expr); } else if (expr instanceof DorisSelect) { @@ -34,7 +33,7 @@ public void visitSpecific(Node expr) { private void visit(DorisJoin join) { sb.append(" "); - visit(join.getLeftTable()); + visit((DorisExpression) join.getLeftTable()); sb.append(" "); switch (join.getJoinType()) { case INNER: @@ -65,7 +64,7 @@ private void visit(DorisJoin join) { default: throw new AssertionError(); } - visit(join.getRightTable()); + visit((DorisExpression) join.getRightTable()); sb.append(" "); if (join.getOnCondition() != null) { sb.append("ON "); @@ -91,7 +90,7 @@ private void visit(DorisFunctionOperation func) { if (func.getArgs() != null) { for (int i = 0; i < func.getArgs().size(); i++) { - visit(DorisExprToNode.cast(func.getArgs().get(i))); + visit(func.getArgs().get(i)); if (i != func.getArgs().size() - 1) { sb.append(","); } @@ -102,20 +101,20 @@ private void visit(DorisFunctionOperation func) { private void visit(DorisCaseOperation cases) { sb.append("CASE "); - visit(DorisExprToNode.cast(cases.getExpr())); + visit(cases.getExpr()); sb.append(" "); for (int i = 0; i < cases.getConditions().size(); i++) { DorisExpression predicate = cases.getConditions().get(i); DorisExpression then = cases.getThenClauses().get(i); sb.append(" WHEN "); - visit(DorisExprToNode.cast(predicate)); + visit(predicate); sb.append(" THEN "); - visit(DorisExprToNode.cast(then)); + visit(then); sb.append(" "); } if (cases.getElseClause() != null) { sb.append("ELSE "); - visit(DorisExprToNode.cast(cases.getElseClause())); + visit(cases.getElseClause()); sb.append(" "); } sb.append("END "); @@ -161,15 +160,9 @@ private void visit(DorisSelect select) { } } - public static String asString(Node expr) { - DorisToStringVisitor visitor = new DorisToStringVisitor(); - visitor.visit(expr); - return visitor.get(); - } - public static String asString(DorisExpression expr) { DorisToStringVisitor visitor = new DorisToStringVisitor(); - visitor.visit(DorisExprToNode.cast(expr)); + visitor.visit(expr); return visitor.get(); } } From bc08478432b34c25bd0689c3df0c93cd08770227 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 10:03:42 +0800 Subject: [PATCH 0901/1284] [H2] Update expressions to match new AST --- src/sqlancer/h2/H2CastNode.java | 24 ------- src/sqlancer/h2/H2ExpressionGenerator.java | 65 +++++++++---------- src/sqlancer/h2/H2QueryPartitioningBase.java | 24 +++---- src/sqlancer/h2/H2RandomQuerySynthesizer.java | 17 +++-- src/sqlancer/h2/H2Select.java | 8 --- src/sqlancer/h2/H2ToStringVisitor.java | 14 ++-- src/sqlancer/h2/ast/H2BetweenOperation.java | 9 +++ src/sqlancer/h2/ast/H2BinaryOperation.java | 10 +++ src/sqlancer/h2/ast/H2CaseOperation.java | 12 ++++ src/sqlancer/h2/ast/H2CastNode.java | 23 +++++++ src/sqlancer/h2/ast/H2ColumnReference.java | 11 ++++ src/sqlancer/h2/{ => ast}/H2Constant.java | 18 +++-- src/sqlancer/h2/{ => ast}/H2Expression.java | 2 +- src/sqlancer/h2/ast/H2Function.java | 11 ++++ src/sqlancer/h2/ast/H2InOperation.java | 11 ++++ src/sqlancer/h2/{ => ast}/H2Join.java | 46 ++++++------- src/sqlancer/h2/ast/H2Select.java | 7 ++ src/sqlancer/h2/ast/H2TableReference.java | 10 +++ .../h2/ast/H2UnaryPostfixOperation.java | 10 +++ .../h2/ast/H2UnaryPrefixOperation.java | 10 +++ 20 files changed, 215 insertions(+), 127 deletions(-) delete mode 100644 src/sqlancer/h2/H2CastNode.java delete mode 100644 src/sqlancer/h2/H2Select.java create mode 100644 src/sqlancer/h2/ast/H2BetweenOperation.java create mode 100644 src/sqlancer/h2/ast/H2BinaryOperation.java create mode 100644 src/sqlancer/h2/ast/H2CaseOperation.java create mode 100644 src/sqlancer/h2/ast/H2CastNode.java create mode 100644 src/sqlancer/h2/ast/H2ColumnReference.java rename src/sqlancer/h2/{ => ast}/H2Constant.java (83%) rename src/sqlancer/h2/{ => ast}/H2Expression.java (59%) create mode 100644 src/sqlancer/h2/ast/H2Function.java create mode 100644 src/sqlancer/h2/ast/H2InOperation.java rename src/sqlancer/h2/{ => ast}/H2Join.java (55%) create mode 100644 src/sqlancer/h2/ast/H2Select.java create mode 100644 src/sqlancer/h2/ast/H2TableReference.java create mode 100644 src/sqlancer/h2/ast/H2UnaryPostfixOperation.java create mode 100644 src/sqlancer/h2/ast/H2UnaryPrefixOperation.java diff --git a/src/sqlancer/h2/H2CastNode.java b/src/sqlancer/h2/H2CastNode.java deleted file mode 100644 index 9d279c8dd..000000000 --- a/src/sqlancer/h2/H2CastNode.java +++ /dev/null @@ -1,24 +0,0 @@ -package sqlancer.h2; - -import sqlancer.common.ast.newast.Node; -import sqlancer.h2.H2Schema.H2CompositeDataType; - -public class H2CastNode implements Node { - - private final Node expression; - private final H2CompositeDataType type; - - public H2CastNode(Node expression, H2CompositeDataType type) { - this.expression = expression; - this.type = type; - } - - public Node getExpression() { - return expression; - } - - public H2CompositeDataType getType() { - return type; - } - -} diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index d04251409..d4a2240e6 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -2,22 +2,23 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewCaseOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2CompositeDataType; import sqlancer.h2.H2Schema.H2DataType; - -public class H2ExpressionGenerator extends UntypedExpressionGenerator, H2Column> { +import sqlancer.h2.ast.H2BetweenOperation; +import sqlancer.h2.ast.H2BinaryOperation; +import sqlancer.h2.ast.H2CaseOperation; +import sqlancer.h2.ast.H2CastNode; +import sqlancer.h2.ast.H2ColumnReference; +import sqlancer.h2.ast.H2Constant; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2InOperation; +import sqlancer.h2.ast.H2UnaryPostfixOperation; +import sqlancer.h2.ast.H2UnaryPrefixOperation; + +public class H2ExpressionGenerator extends UntypedExpressionGenerator { private final H2GlobalState globalState; @@ -31,7 +32,7 @@ private enum Expression { } @Override - protected Node generateExpression(int depth) { + protected H2Expression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); } @@ -39,37 +40,33 @@ protected Node generateExpression(int depth) { switch (expr) { case BINARY_COMPARISON: Operator op = H2BinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), - op); + return new H2BinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_LOGICAL: op = H2BinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), - op); + return new H2BinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); case UNARY_POSTFIX: op = H2UnaryPostfixOperator.getRandom(); - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), op); + return new H2UnaryPostfixOperation(generateExpression(depth + 1), op); case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), - H2UnaryPrefixOperator.getRandom()); + return new H2UnaryPrefixOperation(generateExpression(depth + 1), H2UnaryPrefixOperator.getRandom()); case IN: - return new NewInOperatorNode(generateExpression(depth + 1), + return new H2InOperation(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case BETWEEN: - return new NewBetweenOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + return new H2BetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; - return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), - generateExpression(depth + 1)); + return new H2CaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), + return new H2BinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), H2BinaryArithmeticOperator.getRandom()); case CAST: return new H2CastNode(generateExpression(depth + 1), H2CompositeDataType.getRandom()); case FUNCTION: H2Function func = H2Function.getRandom(); - return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + return new sqlancer.h2.ast.H2Function<>(generateExpressions(func.getNrArgs()), func); default: throw new AssertionError(); } @@ -203,12 +200,12 @@ public int getNrArgs() { } @Override - protected Node generateColumn() { - return new ColumnReferenceNode(Randomly.fromList(columns)); + protected H2Expression generateColumn() { + return new H2ColumnReference(Randomly.fromList(columns)); } @Override - public Node generateConstant() { + public H2Expression generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return H2Constant.createNullConstant(); } @@ -330,13 +327,13 @@ public String getTextRepresentation() { } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, H2UnaryPrefixOperator.NOT); + public H2Expression negatePredicate(H2Expression predicate) { + return new H2UnaryPrefixOperation(predicate, H2UnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, H2UnaryPostfixOperator.IS_NULL); + public H2Expression isNull(H2Expression expr) { + return new H2UnaryPostfixOperation(expr, H2UnaryPostfixOperator.IS_NULL); } } diff --git a/src/sqlancer/h2/H2QueryPartitioningBase.java b/src/sqlancer/h2/H2QueryPartitioningBase.java index ec5af4e2e..34351f58f 100644 --- a/src/sqlancer/h2/H2QueryPartitioningBase.java +++ b/src/sqlancer/h2/H2QueryPartitioningBase.java @@ -5,9 +5,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -15,8 +12,13 @@ import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2Table; import sqlancer.h2.H2Schema.H2Tables; +import sqlancer.h2.ast.H2ColumnReference; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; +import sqlancer.h2.ast.H2TableReference; -public class H2QueryPartitioningBase extends TernaryLogicPartitioningOracleBase, H2GlobalState> +public class H2QueryPartitioningBase extends TernaryLogicPartitioningOracleBase implements TestOracle { H2Schema s; @@ -38,22 +40,22 @@ public void check() throws SQLException { select = new H2Select(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = H2Join.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new H2TableReference(t)) + .collect(Collectors.toList()); + List joins = H2Join.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); - columns.add(new ColumnReferenceNode<>(new H2Column("*", null))); + List generateFetchColumns() { + List columns = new ArrayList<>(); + columns.add(new H2ColumnReference(new H2Column("*", null))); return columns; } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } diff --git a/src/sqlancer/h2/H2RandomQuerySynthesizer.java b/src/sqlancer/h2/H2RandomQuerySynthesizer.java index dafa7835c..ba89bfd0b 100644 --- a/src/sqlancer/h2/H2RandomQuerySynthesizer.java +++ b/src/sqlancer/h2/H2RandomQuerySynthesizer.java @@ -5,11 +5,14 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; import sqlancer.h2.H2Schema.H2Tables; +import sqlancer.h2.ast.H2Constant; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; +import sqlancer.h2.ast.H2TableReference; public final class H2RandomQuerySynthesizer { @@ -20,16 +23,16 @@ public static H2Select generateSelect(H2GlobalState globalState, int nrColumns) H2Tables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState).setColumns(targetTables.getColumns()); H2Select select = new H2Select(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { - Node expression = gen.generateExpression(); + H2Expression expression = gen.generateExpression(); columns.add(expression); } select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = H2Join.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new H2TableReference(t)) + .collect(Collectors.toList()); + List joins = H2Join.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/h2/H2Select.java b/src/sqlancer/h2/H2Select.java deleted file mode 100644 index 07191659d..000000000 --- a/src/sqlancer/h2/H2Select.java +++ /dev/null @@ -1,8 +0,0 @@ -package sqlancer.h2; - -import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; - -public class H2Select extends SelectBase> implements Node { - -} diff --git a/src/sqlancer/h2/H2ToStringVisitor.java b/src/sqlancer/h2/H2ToStringVisitor.java index ad45c66f2..ab18ab235 100644 --- a/src/sqlancer/h2/H2ToStringVisitor.java +++ b/src/sqlancer/h2/H2ToStringVisitor.java @@ -1,12 +1,16 @@ package sqlancer.h2; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; +import sqlancer.h2.ast.H2CastNode; +import sqlancer.h2.ast.H2Constant; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; public class H2ToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(H2Expression expr) { if (expr instanceof H2Constant) { visit((H2Constant) expr); } else if (expr instanceof H2Select) { @@ -33,11 +37,11 @@ private void visit(H2CastNode cast) { } private void visit(H2Join join) { - visit(join.getLeftTable()); + visit((H2Expression) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" JOIN "); - visit(join.getRightTable()); + visit((H2Expression) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); @@ -81,7 +85,7 @@ public void visit(H2Select select) { } } - public static String asString(Node expr) { + public static String asString(H2Expression expr) { H2ToStringVisitor visitor = new H2ToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/h2/ast/H2BetweenOperation.java b/src/sqlancer/h2/ast/H2BetweenOperation.java new file mode 100644 index 000000000..77921d91e --- /dev/null +++ b/src/sqlancer/h2/ast/H2BetweenOperation.java @@ -0,0 +1,9 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class H2BetweenOperation extends NewBetweenOperatorNode implements H2Expression { + public H2BetweenOperation(H2Expression left, H2Expression middle, H2Expression right, boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/h2/ast/H2BinaryOperation.java b/src/sqlancer/h2/ast/H2BinaryOperation.java new file mode 100644 index 000000000..d523256fd --- /dev/null +++ b/src/sqlancer/h2/ast/H2BinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class H2BinaryOperation extends NewBinaryOperatorNode implements H2Expression { + public H2BinaryOperation(H2Expression left, H2Expression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/h2/ast/H2CaseOperation.java b/src/sqlancer/h2/ast/H2CaseOperation.java new file mode 100644 index 000000000..75e51fc4d --- /dev/null +++ b/src/sqlancer/h2/ast/H2CaseOperation.java @@ -0,0 +1,12 @@ +package sqlancer.h2.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class H2CaseOperation extends NewCaseOperatorNode implements H2Expression { + public H2CaseOperation(H2Expression switchCondition, List conditions, List expressions, + H2Expression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/h2/ast/H2CastNode.java b/src/sqlancer/h2/ast/H2CastNode.java new file mode 100644 index 000000000..5e7602311 --- /dev/null +++ b/src/sqlancer/h2/ast/H2CastNode.java @@ -0,0 +1,23 @@ +package sqlancer.h2.ast; + +import sqlancer.h2.H2Schema.H2CompositeDataType; + +public class H2CastNode implements H2Expression { + + private final H2Expression expression; + private final H2CompositeDataType type; + + public H2CastNode(H2Expression expression, H2CompositeDataType type) { + this.expression = expression; + this.type = type; + } + + public H2Expression getExpression() { + return expression; + } + + public H2CompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/h2/ast/H2ColumnReference.java b/src/sqlancer/h2/ast/H2ColumnReference.java new file mode 100644 index 000000000..5864f0b53 --- /dev/null +++ b/src/sqlancer/h2/ast/H2ColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.h2.H2Schema; + +public class H2ColumnReference extends ColumnReferenceNode implements H2Expression { + public H2ColumnReference(H2Schema.H2Column column) { + super(column); + } + +} diff --git a/src/sqlancer/h2/H2Constant.java b/src/sqlancer/h2/ast/H2Constant.java similarity index 83% rename from src/sqlancer/h2/H2Constant.java rename to src/sqlancer/h2/ast/H2Constant.java index 4390fed26..923bcb4a2 100644 --- a/src/sqlancer/h2/H2Constant.java +++ b/src/sqlancer/h2/ast/H2Constant.java @@ -1,8 +1,6 @@ -package sqlancer.h2; +package sqlancer.h2.ast; -import sqlancer.common.ast.newast.Node; - -public class H2Constant implements Node { +public class H2Constant implements H2Expression { private H2Constant() { } @@ -108,27 +106,27 @@ public String toString() { } - public static Node createIntConstant(long val) { + public static H2Expression createIntConstant(long val) { return new H2IntConstant(val); } - public static Node createNullConstant() { + public static H2Expression createNullConstant() { return new H2NullConstant(); } - public static Node createBoolConstant(boolean val) { + public static H2Expression createBoolConstant(boolean val) { return new H2BoolConstant(val); } - public static Node createStringConstant(String val) { + public static H2Expression createStringConstant(String val) { return new H2StringConstant(val); } - public static Node createDoubleConstant(double val) { + public static H2Expression createDoubleConstant(double val) { return new H2DoubleConstant(val); } - public static Node createBinaryConstant(long val) { + public static H2Expression createBinaryConstant(long val) { return new H2BinaryConstant(val); } diff --git a/src/sqlancer/h2/H2Expression.java b/src/sqlancer/h2/ast/H2Expression.java similarity index 59% rename from src/sqlancer/h2/H2Expression.java rename to src/sqlancer/h2/ast/H2Expression.java index 61b8c9658..ed41cdfab 100644 --- a/src/sqlancer/h2/H2Expression.java +++ b/src/sqlancer/h2/ast/H2Expression.java @@ -1,4 +1,4 @@ -package sqlancer.h2; +package sqlancer.h2.ast; public interface H2Expression { diff --git a/src/sqlancer/h2/ast/H2Function.java b/src/sqlancer/h2/ast/H2Function.java new file mode 100644 index 000000000..9442d6db7 --- /dev/null +++ b/src/sqlancer/h2/ast/H2Function.java @@ -0,0 +1,11 @@ +package sqlancer.h2.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class H2Function extends NewFunctionNode implements H2Expression { + public H2Function(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/h2/ast/H2InOperation.java b/src/sqlancer/h2/ast/H2InOperation.java new file mode 100644 index 000000000..1389fb18b --- /dev/null +++ b/src/sqlancer/h2/ast/H2InOperation.java @@ -0,0 +1,11 @@ +package sqlancer.h2.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class H2InOperation extends NewInOperatorNode implements H2Expression { + public H2InOperation(H2Expression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/h2/H2Join.java b/src/sqlancer/h2/ast/H2Join.java similarity index 55% rename from src/sqlancer/h2/H2Join.java rename to src/sqlancer/h2/ast/H2Join.java index 1b566c5f0..e777e4fc9 100644 --- a/src/sqlancer/h2/H2Join.java +++ b/src/sqlancer/h2/ast/H2Join.java @@ -1,21 +1,19 @@ -package sqlancer.h2; +package sqlancer.h2.ast; import java.util.ArrayList; import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.h2.H2ExpressionGenerator; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; -import sqlancer.h2.H2Schema.H2Table; -public class H2Join implements Node { +public class H2Join implements H2Expression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final H2TableReference leftTable; + private final H2TableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final H2Expression onCondition; public enum JoinType { INNER, CROSS, NATURAL, LEFT, RIGHT; @@ -25,20 +23,19 @@ public static JoinType getRandom() { } } - public H2Join(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public H2Join(H2TableReference leftTable, H2TableReference rightTable, JoinType joinType, + H2Expression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public TableReferenceNode getLeftTable() { + public H2TableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public H2TableReference getRightTable() { return rightTable; } @@ -46,16 +43,15 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public H2Expression getOnCondition() { return onCondition; } - public static List> getJoins(List> tableList, - H2GlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, H2GlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + H2TableReference leftTable = tableList.remove(0); + H2TableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); H2ExpressionGenerator joinGen = new H2ExpressionGenerator(globalState).setColumns(columns); @@ -83,23 +79,19 @@ public static List> getJoins(List left, - TableReferenceNode right, Node predicate) { + public static H2Join createRightOuterJoin(H2TableReference left, H2TableReference right, H2Expression predicate) { return new H2Join(left, right, JoinType.RIGHT, predicate); } - public static H2Join createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static H2Join createLeftOuterJoin(H2TableReference left, H2TableReference right, H2Expression predicate) { return new H2Join(left, right, JoinType.LEFT, predicate); } - public static H2Join createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static H2Join createInnerJoin(H2TableReference left, H2TableReference right, H2Expression predicate) { return new H2Join(left, right, JoinType.INNER, predicate); } - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right) { + public static H2Expression createNaturalJoin(H2TableReference left, H2TableReference right) { return new H2Join(left, right, JoinType.NATURAL, null); } diff --git a/src/sqlancer/h2/ast/H2Select.java b/src/sqlancer/h2/ast/H2Select.java new file mode 100644 index 000000000..9658ebca0 --- /dev/null +++ b/src/sqlancer/h2/ast/H2Select.java @@ -0,0 +1,7 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.SelectBase; + +public class H2Select extends SelectBase implements H2Expression { + +} diff --git a/src/sqlancer/h2/ast/H2TableReference.java b/src/sqlancer/h2/ast/H2TableReference.java new file mode 100644 index 000000000..459337c41 --- /dev/null +++ b/src/sqlancer/h2/ast/H2TableReference.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.h2.H2Schema; + +public class H2TableReference extends TableReferenceNode implements H2Expression { + public H2TableReference(H2Schema.H2Table table) { + super(table); + } +} diff --git a/src/sqlancer/h2/ast/H2UnaryPostfixOperation.java b/src/sqlancer/h2/ast/H2UnaryPostfixOperation.java new file mode 100644 index 000000000..5c43effbe --- /dev/null +++ b/src/sqlancer/h2/ast/H2UnaryPostfixOperation.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class H2UnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements H2Expression { + public H2UnaryPostfixOperation(H2Expression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/h2/ast/H2UnaryPrefixOperation.java b/src/sqlancer/h2/ast/H2UnaryPrefixOperation.java new file mode 100644 index 000000000..24085dcb6 --- /dev/null +++ b/src/sqlancer/h2/ast/H2UnaryPrefixOperation.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class H2UnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements H2Expression { + public H2UnaryPrefixOperation(H2Expression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} From 2464a7303c6a997b5781951df3b24e493cd189d9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 10:27:55 +0800 Subject: [PATCH 0902/1284] [HSQLDB] Update expressions to match new AST --- .../hsqldb/HSQLDBToStringVisitor.java | 9 ++-- .../hsqldb/ast/HSQLDBBinaryOperation.java | 10 ++++ .../hsqldb/ast/HSQLDBColumnReference.java | 3 +- src/sqlancer/hsqldb/ast/HSQLDBConstant.java | 22 ++++----- src/sqlancer/hsqldb/ast/HSQLDBJoin.java | 47 +++++++++---------- src/sqlancer/hsqldb/ast/HSQLDBSelect.java | 3 +- .../hsqldb/ast/HSQLDBTableReference.java | 11 +++++ .../ast/HSQLDBUnaryPostfixOperation.java | 8 ++-- .../ast/HSQLDBUnaryPrefixOperation.java | 6 +-- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 24 +++++----- .../hsqldb/gen/HSQLDBInsertGenerator.java | 4 +- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 3 +- .../hsqldb/test/HSQLDBNoRECOracle.java | 25 +++++----- .../test/HSQLDBQueryPartitioningBase.java | 21 ++++----- 14 files changed, 99 insertions(+), 97 deletions(-) create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBTableReference.java diff --git a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java index 8caf7b867..99fc50687 100644 --- a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java +++ b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.hsqldb; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; import sqlancer.hsqldb.ast.HSQLDBJoin; @@ -10,7 +9,7 @@ public class HSQLDBToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(HSQLDBExpression expr) { if (expr instanceof HSQLDBConstant) { visit((HSQLDBConstant) expr); } else if (expr instanceof HSQLDBSelect) { @@ -22,14 +21,14 @@ public void visitSpecific(Node expr) { } } - public static String asString(Node expr) { + public static String asString(HSQLDBExpression expr) { HSQLDBToStringVisitor visitor = new HSQLDBToStringVisitor(); visitor.visit(expr); return visitor.get(); } private void visit(HSQLDBJoin join) { - visit(join.getLeftTable()); + visit((HSQLDBExpression) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" "); @@ -37,7 +36,7 @@ private void visit(HSQLDBJoin join) { sb.append(join.getOuterType()); } sb.append(" JOIN "); - visit(join.getRightTable()); + visit((HSQLDBExpression) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); diff --git a/src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java new file mode 100644 index 000000000..b821a90e2 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class HSQLDBBinaryOperation extends NewBinaryOperatorNode implements HSQLDBExpression { + public HSQLDBBinaryOperation(HSQLDBExpression left, HSQLDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java index b2179eca3..cb579abac 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java @@ -3,7 +3,8 @@ import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.hsqldb.HSQLDBSchema; -public class HSQLDBColumnReference extends ColumnReferenceNode { +public class HSQLDBColumnReference extends ColumnReferenceNode + implements HSQLDBExpression { public HSQLDBColumnReference(HSQLDBSchema.HSQLDBColumn column) { super(column); diff --git a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java index f6eed5e13..67997d6bb 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java @@ -3,9 +3,7 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.common.ast.newast.Node; - -public class HSQLDBConstant implements Node { +public class HSQLDBConstant implements HSQLDBExpression { private HSQLDBConstant() { } @@ -190,39 +188,39 @@ public String toString() { } - public static Node createStringConstant(String text, int size) { + public static HSQLDBExpression createStringConstant(String text, int size) { return new HSQLDBTextConstant(text, size); } - public static Node createFloatConstant(double val) { + public static HSQLDBExpression createFloatConstant(double val) { return new HSQLDBDoubleConstant(val); } - public static Node createIntConstant(long val) { + public static HSQLDBExpression createIntConstant(long val) { return new HSQLDBIntConstant(val); } - public static Node createNullConstant() { + public static HSQLDBExpression createNullConstant() { return new HSQLDBNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static HSQLDBExpression createBooleanConstant(boolean val) { return new HSQLDBBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static HSQLDBExpression createDateConstant(long integer) { return new HSQLDBDateConstant(integer); } - public static Node createTimeConstant(long integer, int size) { + public static HSQLDBExpression createTimeConstant(long integer, int size) { return new HSQLDBTimeConstant(integer); } - public static Node createTimestampConstant(long integer, int size) { + public static HSQLDBExpression createTimestampConstant(long integer, int size) { return new HSQLDBTimestampConstant(integer); } - public static Node createBinaryConstant(long nonCachedInteger, int size) { + public static HSQLDBExpression createBinaryConstant(long nonCachedInteger, int size) { return new HSQLDBBinaryConstant(nonCachedInteger, size); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java index ef715052d..595384edd 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java @@ -4,19 +4,16 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -public class HSQLDBJoin implements Node { +public class HSQLDBJoin implements HSQLDBExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final HSQLDBTableReference leftTable; + private final HSQLDBTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final HSQLDBExpression onCondition; private OuterType outerType; public enum JoinType { @@ -35,20 +32,19 @@ public static OuterType getRandom() { } } - public HSQLDBJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public HSQLDBJoin(HSQLDBTableReference leftTable, HSQLDBTableReference rightTable, JoinType joinType, + HSQLDBExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public TableReferenceNode getLeftTable() { + public HSQLDBTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public HSQLDBTableReference getRightTable() { return rightTable; } @@ -56,7 +52,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public HSQLDBExpression getOnCondition() { return onCondition; } @@ -68,12 +64,11 @@ public OuterType getOuterType() { return outerType; } - public static List> getJoins( - List> tableList, HSQLDBGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, HSQLDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + HSQLDBTableReference leftTable = tableList.remove(0); + HSQLDBTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(globalState).setColumns(columns); @@ -100,23 +95,23 @@ public static List> getJoins( return joinExpressions; } - public static HSQLDBJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static HSQLDBJoin createRightOuterJoin(HSQLDBTableReference left, HSQLDBTableReference right, + HSQLDBExpression predicate) { return new HSQLDBJoin(left, right, JoinType.RIGHT, predicate); } - public static HSQLDBJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static HSQLDBJoin createLeftOuterJoin(HSQLDBTableReference left, HSQLDBTableReference right, + HSQLDBExpression predicate) { return new HSQLDBJoin(left, right, JoinType.LEFT, predicate); } - public static HSQLDBJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static HSQLDBJoin createInnerJoin(HSQLDBTableReference left, HSQLDBTableReference right, + HSQLDBExpression predicate) { return new HSQLDBJoin(left, right, JoinType.INNER, predicate); } - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, OuterType naturalJoinType) { + public static HSQLDBExpression createNaturalJoin(HSQLDBTableReference left, HSQLDBTableReference right, + OuterType naturalJoinType) { HSQLDBJoin join = new HSQLDBJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; diff --git a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java index 05cd390f2..c75b0b88a 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java @@ -1,9 +1,8 @@ package sqlancer.hsqldb.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class HSQLDBSelect extends SelectBase> implements Node { +public class HSQLDBSelect extends SelectBase implements HSQLDBExpression { private boolean isDistinct; diff --git a/src/sqlancer/hsqldb/ast/HSQLDBTableReference.java b/src/sqlancer/hsqldb/ast/HSQLDBTableReference.java new file mode 100644 index 000000000..010a921a7 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBTableReference extends TableReferenceNode + implements HSQLDBExpression { + public HSQLDBTableReference(HSQLDBSchema.HSQLDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java index 9e63efee5..5ebca87fa 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java @@ -3,12 +3,12 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.hsqldb.HSQLDBSchema; -public class HSQLDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { +public class HSQLDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements HSQLDBExpression { - public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op) { + public HSQLDBUnaryPostfixOperation(HSQLDBExpression expr, HSQLDBUnaryPostfixOperator op) { super(expr, op); } @@ -45,7 +45,7 @@ public String getTextRepresentation() { } - public Node getExpression() { + public HSQLDBExpression getExpression() { return getExpr(); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java index b1327d0be..44b2beece 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java @@ -2,12 +2,12 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.hsqldb.HSQLDBSchema; -public class HSQLDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { +public class HSQLDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements HSQLDBExpression { - public HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperator operation, Node expression) { + public HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperator operation, HSQLDBExpression expression) { super(expression, operation); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index a978dfdc3..b5304c890 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -7,11 +7,10 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.ast.HSQLDBBinaryOperation; import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; @@ -19,7 +18,7 @@ import sqlancer.hsqldb.ast.HSQLDBUnaryPrefixOperation; public final class HSQLDBExpressionGenerator extends - TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { + TypedExpressionGenerator { private enum Expression { BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC; @@ -32,23 +31,23 @@ public HSQLDBExpressionGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { } @Override - public Node generatePredicate() { + public HSQLDBExpression generatePredicate() { return generateExpression( HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithType(HSQLDBSchema.HSQLDBDataType.BOOLEAN)); } @Override - public Node negatePredicate(Node predicate) { + public HSQLDBExpression negatePredicate(HSQLDBExpression predicate) { return new HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperation.HSQLDBUnaryPrefixOperator.NOT, predicate); } @Override - public Node isNull(Node expr) { + public HSQLDBExpression isNull(HSQLDBExpression expr) { return new HSQLDBUnaryPostfixOperation(expr, HSQLDBUnaryPostfixOperation.HSQLDBUnaryPostfixOperator.IS_NULL); } @Override - public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataType type) { + public HSQLDBExpression generateConstant(HSQLDBSchema.HSQLDBCompositeDataType type) { switch (type.getType()) { case NULL: return HSQLDBConstant.createNullConstant(); @@ -82,7 +81,7 @@ public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataT } @Override - protected Node generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { + protected HSQLDBExpression generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { if (depth >= hsqldbGlobalState.getOptions().getMaxExpressionDepth() || Randomly.getBooleanWithSmallProbability()) { return generateLeafNode(type); @@ -105,13 +104,12 @@ protected Node generateExpression(HSQLDBSchema.HSQLDBComposite throw new AssertionError(); } - return new NewBinaryOperatorNode<>(generateExpression(type, depth + 1), generateExpression(type, depth + 1), - op); + return new HSQLDBBinaryOperation(generateExpression(type, depth + 1), generateExpression(type, depth + 1), op); } @Override - protected Node generateColumn(HSQLDBSchema.HSQLDBCompositeDataType type) { + protected HSQLDBExpression generateColumn(HSQLDBSchema.HSQLDBCompositeDataType type) { HSQLDBSchema.HSQLDBColumn column = Randomly .fromList(columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList())); return new HSQLDBColumnReference(column); @@ -183,8 +181,8 @@ public String getTextRepresentation() { } @Override - public List> generateOrderBys() { - List> expressions = new ArrayList<>(); + public List generateOrderBys() { + List expressions = new ArrayList<>(); int nr = Randomly.smallNumber() + 1; ArrayList hsqldbColumns = new ArrayList<>(columns); for (int i = 0; i < nr && !hsqldbColumns.isEmpty(); i++) { diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index c21d7c786..1cc132190 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -41,8 +40,7 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(HSQLDBSchema.HSQLDBColumn column) { - Node expression = new HSQLDBExpressionGenerator(globalState) - .generateConstant(column.getType()); + HSQLDBExpression expression = new HSQLDBExpressionGenerator(globalState).generateConstant(column.getType()); String s = HSQLDBToStringVisitor.asString(expression); sb.append(s); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index bc1a220b1..e639e21b3 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBErrors; @@ -48,7 +47,7 @@ private SQLQueryAdapter generate() { @Override protected void updateValue(HSQLDBColumn column) { - Node expr; + HSQLDBExpression expr; expr = gen.generateConstant(column.getType()); sb.append(HSQLDBToStringVisitor.asString(expr)); } diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java index eb08d061d..e5cc786c3 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -10,9 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -29,6 +26,7 @@ import sqlancer.hsqldb.ast.HSQLDBExpression; import sqlancer.hsqldb.ast.HSQLDBJoin; import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.ast.HSQLDBTableReference; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { @@ -47,12 +45,12 @@ public void check() throws SQLException { List columns = tables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList()); HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen + HSQLDBExpression randomWhereCondition = gen .generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = HSQLDBJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new HSQLDBTableReference(t)) + .collect(Collectors.toList()); + List joins = HSQLDBJoin.getJoins(tableList, state); int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); // 禁用优化 int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { @@ -64,8 +62,8 @@ public void check() throws SQLException { } } - private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + private int getSecondQuery(List tableList, HSQLDBExpression randomWhereCondition, + List joins) throws SQLException { HSQLDBSelect select = new HSQLDBSelect(); HSQLDBColumn c = new HSQLDBColumn("COUNT(*)", null, null); select.setFetchColumns(List.of(new HSQLDBColumnReference(c))); @@ -91,12 +89,11 @@ private int getSecondQuery(List> tableList, Node> tableList, - List columns, Node randomWhereCondition, List> joins) - throws SQLException { + private int getFirstQueryCount(SQLConnection con, List tableList, List columns, + HSQLDBExpression randomWhereCondition, List joins) throws SQLException { HSQLDBSelect select = new HSQLDBSelect(); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List allColumns = columns.stream().map((c) -> new HSQLDBColumnReference(c)) + .collect(Collectors.toList()); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java index 5e62a39ac..b428fd02f 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -6,22 +6,21 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBExpression; //import sqlancer.hsqldb.ast.HSQLDBJoin; import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.ast.HSQLDBTableReference; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; public class HSQLDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> + extends TernaryLogicPartitioningOracleBase implements TestOracle { HSQLDBSelect select; @@ -36,7 +35,7 @@ public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return expressionGenerator; } @@ -49,8 +48,7 @@ public void check() throws Exception { initializeTernaryPredicateVariants(); select = new HSQLDBSelect(); select.setFetchColumns(generateFetchColumns()); - List> tableList = targetTables.stream() - .map(t -> new TableReferenceNode(t)) + List tableList = targetTables.stream().map(t -> new HSQLDBTableReference(t)) .collect(Collectors.toList()); // List> joins = HSQLDBJoin.getJoins(tableList, state); // select.setJoinList(joins.stream().collect(Collectors.toList())); @@ -61,14 +59,13 @@ public void check() throws Exception { } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new HSQLDBSchema.HSQLDBColumn("*", null, null))); + columns.add(new HSQLDBColumnReference(new HSQLDBSchema.HSQLDBColumn("*", null, null))); } else { columns = Randomly.nonEmptySubset(targetTables.stream().flatMap(t -> t.getColumns().stream()) - .map(c -> new ColumnReferenceNode(c)) - .collect(Collectors.toList())); + .map(c -> new HSQLDBColumnReference(c)).collect(Collectors.toList())); } return columns; } From 435aff08a049765c0b8878726a8a0be995b8aa75 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 11:54:31 +0800 Subject: [PATCH 0903/1284] [Presto] Update expressions to match new AST --- .../presto/PrestoExpressionToNode.java | 25 -- .../presto/PrestoToStringVisitor.java | 9 +- .../presto/ast/PrestoAggregateFunction.java | 37 ++- src/sqlancer/presto/ast/PrestoAlias.java | 9 + .../presto/ast/PrestoAtTimeZoneOperator.java | 14 +- .../presto/ast/PrestoBetweenOperation.java | 10 + .../presto/ast/PrestoBinaryOperation.java | 10 + .../presto/ast/PrestoCaseOperation.java | 12 + .../presto/ast/PrestoCastFunction.java | 9 +- .../presto/ast/PrestoColumnReference.java | 3 +- src/sqlancer/presto/ast/PrestoConstant.java | 58 ++--- .../presto/ast/PrestoDefaultFunction.java | 7 +- src/sqlancer/presto/ast/PrestoFunction.java | 15 +- .../presto/ast/PrestoFunctionNode.java | 11 + .../ast/PrestoFunctionWithoutParenthesis.java | 3 +- .../presto/ast/PrestoInOperation.java | 11 + src/sqlancer/presto/ast/PrestoJoin.java | 43 ++-- .../ast/PrestoMultiValuedComparison.java | 14 +- .../presto/ast/PrestoPostfixText.java | 9 + .../ast/PrestoQuantifiedComparison.java | 14 +- src/sqlancer/presto/ast/PrestoSelect.java | 3 +- .../presto/ast/PrestoTableReference.java | 3 +- src/sqlancer/presto/ast/PrestoTernary.java | 10 + .../ast/PrestoUnaryPostfixOperation.java | 8 +- .../ast/PrestoUnaryPrefixOperation.java | 6 +- .../presto/gen/PrestoIndexGenerator.java | 3 +- .../presto/gen/PrestoInsertGenerator.java | 3 +- .../gen/PrestoRandomQuerySynthesizer.java | 13 +- .../gen/PrestoTypedExpressionGenerator.java | 239 +++++++++--------- .../presto/gen/PrestoUpdateGenerator.java | 3 +- .../presto/test/PrestoNoRECOracle.java | 32 ++- ...restoQueryPartitioningAggregateTester.java | 41 ++- .../test/PrestoQueryPartitioningBase.java | 26 +- .../PrestoQueryPartitioningGroupByTester.java | 10 +- .../PrestoQueryPartitioningHavingTester.java | 5 +- .../ycql/ast/YCQLBetweenOperation.java | 10 + .../ycql/ast/YCQLBinaryOperation.java | 11 + .../ycql/ast/YCQLColumnReference.java | 13 + .../yugabyte/ycql/ast/YCQLFunction.java | 12 + .../yugabyte/ycql/ast/YCQLInOperation.java | 12 + .../yugabyte/ycql/ast/YCQLOrderingTerm.java | 10 + .../yugabyte/ycql/ast/YCQLTableReference.java | 12 + .../ycql/ast/YCQLUnaryPostfixOperation.java | 12 + .../ycql/ast/YCQLUnaryPrefixOperation.java | 12 + 44 files changed, 478 insertions(+), 354 deletions(-) delete mode 100644 src/sqlancer/presto/PrestoExpressionToNode.java create mode 100644 src/sqlancer/presto/ast/PrestoAlias.java create mode 100644 src/sqlancer/presto/ast/PrestoBetweenOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoBinaryOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoCaseOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoFunctionNode.java create mode 100644 src/sqlancer/presto/ast/PrestoInOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoPostfixText.java create mode 100644 src/sqlancer/presto/ast/PrestoTernary.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java diff --git a/src/sqlancer/presto/PrestoExpressionToNode.java b/src/sqlancer/presto/PrestoExpressionToNode.java deleted file mode 100644 index 96497e47c..000000000 --- a/src/sqlancer/presto/PrestoExpressionToNode.java +++ /dev/null @@ -1,25 +0,0 @@ -package sqlancer.presto; - -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.ast.newast.Node; -import sqlancer.presto.ast.PrestoExpression; - -public final class PrestoExpressionToNode { - - private PrestoExpressionToNode() { - - } - - @SuppressWarnings("unchecked") - public static Node cast(PrestoExpression expression) { - return (Node) expression; - } - - @SuppressWarnings("unchecked") - public static List> casts(List expressions) { - return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); - } - -} diff --git a/src/sqlancer/presto/PrestoToStringVisitor.java b/src/sqlancer/presto/PrestoToStringVisitor.java index f840cf850..11c6d5fa6 100644 --- a/src/sqlancer/presto/PrestoToStringVisitor.java +++ b/src/sqlancer/presto/PrestoToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.presto; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.ast.PrestoAtTimeZoneOperator; import sqlancer.presto.ast.PrestoCastFunction; import sqlancer.presto.ast.PrestoConstant; @@ -14,14 +13,14 @@ public class PrestoToStringVisitor extends NewToStringVisitor { - public static String asString(Node expr) { + public static String asString(PrestoExpression expr) { PrestoToStringVisitor visitor = new PrestoToStringVisitor(); visitor.visit(expr); return visitor.get(); } @Override - public void visitSpecific(Node expr) { + public void visitSpecific(PrestoExpression expr) { if (expr instanceof PrestoConstant) { visit((PrestoConstant) expr); } else if (expr instanceof PrestoSelect) { @@ -44,7 +43,7 @@ public void visitSpecific(Node expr) { } private void visit(PrestoJoin join) { - visit(join.getLeftTable()); + visit((PrestoExpression) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" "); @@ -52,7 +51,7 @@ private void visit(PrestoJoin join) { sb.append(join.getOuterType()); } sb.append(" JOIN "); - visit(join.getRightTable()); + visit((PrestoExpression) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); diff --git a/src/sqlancer/presto/ast/PrestoAggregateFunction.java b/src/sqlancer/presto/ast/PrestoAggregateFunction.java index 0b118393f..25840933a 100644 --- a/src/sqlancer/presto/ast/PrestoAggregateFunction.java +++ b/src/sqlancer/presto/ast/PrestoAggregateFunction.java @@ -8,7 +8,6 @@ import java.util.stream.Stream; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; import sqlancer.presto.PrestoSchema.PrestoDataType; @@ -37,7 +36,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -131,7 +130,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -165,7 +164,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -211,7 +210,7 @@ public PrestoCompositeDataType getCompositeReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -248,7 +247,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -338,7 +337,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -427,9 +426,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); arguments.add(gen.generateExpression(returnType2, depth + 1)); arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); return arguments; @@ -441,7 +440,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -467,9 +466,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); arguments.add(gen.generateExpression(returnType2, depth + 1)); arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); if (Randomly.getBooleanWithRatherLowProbability()) { @@ -486,7 +485,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType @@ -523,9 +522,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); arguments.add(gen.generateExpression(returnType2, depth + 1)); arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); if (Randomly.getBooleanWithRatherLowProbability()) { @@ -542,7 +541,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType @@ -566,9 +565,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); arguments.add(gen.generateExpression(returnType2, depth + 1)); if (Randomly.getBooleanWithRatherLowProbability()) { arguments.add(new PrestoConstant.PrestoIntConstant(1)); @@ -590,7 +589,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType diff --git a/src/sqlancer/presto/ast/PrestoAlias.java b/src/sqlancer/presto/ast/PrestoAlias.java new file mode 100644 index 000000000..b327bed11 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoAlias.java @@ -0,0 +1,9 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class PrestoAlias extends NewAliasNode implements PrestoExpression { + public PrestoAlias(PrestoExpression expr, String alias) { + super(expr, alias); + } +} diff --git a/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java index 7bb01bb98..ac73a0a79 100644 --- a/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java +++ b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java @@ -1,22 +1,20 @@ package sqlancer.presto.ast; -import sqlancer.common.ast.newast.Node; +public class PrestoAtTimeZoneOperator implements PrestoExpression { -public class PrestoAtTimeZoneOperator implements Node { + private final PrestoExpression expr; + private final PrestoExpression timeZone; - private final Node expr; - private final Node timeZone; - - public PrestoAtTimeZoneOperator(Node expr, Node timeZone) { + public PrestoAtTimeZoneOperator(PrestoExpression expr, PrestoExpression timeZone) { this.expr = expr; this.timeZone = timeZone; } - public Node getExpr() { + public PrestoExpression getExpr() { return expr; } - public Node getTimeZone() { + public PrestoExpression getTimeZone() { return timeZone; } } diff --git a/src/sqlancer/presto/ast/PrestoBetweenOperation.java b/src/sqlancer/presto/ast/PrestoBetweenOperation.java new file mode 100644 index 000000000..394a5f675 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class PrestoBetweenOperation extends NewBetweenOperatorNode implements PrestoExpression { + public PrestoBetweenOperation(PrestoExpression left, PrestoExpression middle, PrestoExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/presto/ast/PrestoBinaryOperation.java b/src/sqlancer/presto/ast/PrestoBinaryOperation.java new file mode 100644 index 000000000..90f83f31a --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class PrestoBinaryOperation extends NewBinaryOperatorNode implements PrestoExpression { + public PrestoBinaryOperation(PrestoExpression left, PrestoExpression right, BinaryOperatorNode.Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/presto/ast/PrestoCaseOperation.java b/src/sqlancer/presto/ast/PrestoCaseOperation.java new file mode 100644 index 000000000..117c54bbb --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoCaseOperation.java @@ -0,0 +1,12 @@ +package sqlancer.presto.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class PrestoCaseOperation extends NewCaseOperatorNode implements PrestoExpression { + public PrestoCaseOperation(PrestoExpression switchCondition, List conditions, + List expressions, PrestoExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/presto/ast/PrestoCastFunction.java b/src/sqlancer/presto/ast/PrestoCastFunction.java index 89b139314..14798267d 100644 --- a/src/sqlancer/presto/ast/PrestoCastFunction.java +++ b/src/sqlancer/presto/ast/PrestoCastFunction.java @@ -1,19 +1,18 @@ package sqlancer.presto.ast; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; -public class PrestoCastFunction implements Node { +public class PrestoCastFunction implements PrestoExpression { - private final Node expr; + private final PrestoExpression expr; private final PrestoSchema.PrestoCompositeDataType type; - public PrestoCastFunction(Node expr, PrestoSchema.PrestoCompositeDataType type) { + public PrestoCastFunction(PrestoExpression expr, PrestoSchema.PrestoCompositeDataType type) { this.expr = expr; this.type = type; } - public Node getExpr() { + public PrestoExpression getExpr() { return expr; } diff --git a/src/sqlancer/presto/ast/PrestoColumnReference.java b/src/sqlancer/presto/ast/PrestoColumnReference.java index 4f96a013d..8bc6db0ec 100644 --- a/src/sqlancer/presto/ast/PrestoColumnReference.java +++ b/src/sqlancer/presto/ast/PrestoColumnReference.java @@ -3,7 +3,8 @@ import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.presto.PrestoSchema; -public class PrestoColumnReference extends ColumnReferenceNode { +public class PrestoColumnReference extends ColumnReferenceNode + implements PrestoExpression { public PrestoColumnReference(PrestoSchema.PrestoColumn column) { super(column); diff --git a/src/sqlancer/presto/ast/PrestoConstant.java b/src/sqlancer/presto/ast/PrestoConstant.java index 97b115a4e..b1d30a1ab 100644 --- a/src/sqlancer/presto/ast/PrestoConstant.java +++ b/src/sqlancer/presto/ast/PrestoConstant.java @@ -9,11 +9,10 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoConstantUtils; import sqlancer.presto.PrestoSchema; -public abstract class PrestoConstant implements Node, PrestoExpression { +public abstract class PrestoConstant implements PrestoExpression { private static final String[] TIME_ZONES = { "Africa/Abidjan", "Africa/Accra", "Africa/Addis_Ababa", "Africa/Algiers", "Africa/Asmara", "Africa/Asmera", "Africa/Bamako", "Africa/Bangui", "Africa/Banjul", @@ -127,44 +126,44 @@ public abstract class PrestoConstant implements Node, PrestoEx private PrestoConstant() { } - public static Node createStringConstant(String text) { + public static PrestoExpression createStringConstant(String text) { return new PrestoTextConstant(text); } - public static Node createStringConstant(String text, int size) { + public static PrestoExpression createStringConstant(String text, int size) { return new PrestoTextConstant(text, size); } - public static Node createJsonConstant() { + public static PrestoExpression createJsonConstant() { return new PrestoJsonConstant(); } - public static Node createFloatConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + public static PrestoExpression createFloatConstant(PrestoSchema.PrestoCompositeDataType type, double val) { assert type.getSize() == 4; float floatValue = (float) val; return new PrestoFloatConstant(floatValue); } - public static Node createFloatConstant(double val) { + public static PrestoExpression createFloatConstant(double val) { return new PrestoFloatConstant(val); } - public static Node createDecimalConstant(double val) { + public static PrestoExpression createDecimalConstant(double val) { return new PrestoDecimalConstant(val); } - public static Node createDecimalConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + public static PrestoExpression createDecimalConstant(PrestoSchema.PrestoCompositeDataType type, double val) { int scale = type.getScale(); int precision = type.getSize(); BigDecimal finalBD = PrestoConstantUtils.getDecimal(val, scale, precision); return new PrestoDecimalConstant(finalBD.doubleValue()); } - public static Node createIntConstant(long val) { + public static PrestoExpression createIntConstant(long val) { return new PrestoIntConstant(val); } - public static Node createIntConstant(PrestoSchema.PrestoCompositeDataType type, long val, + public static PrestoExpression createIntConstant(PrestoSchema.PrestoCompositeDataType type, long val, boolean castInteger) { PrestoIntConstant intConstant; assert List.of(1, 2, 4, 8).contains(type.getSize()); @@ -188,56 +187,56 @@ public static Node createIntConstant(PrestoSchema.PrestoCompos } } - public static Node createNullConstant() { + public static PrestoExpression createNullConstant() { return new PrestoNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static PrestoExpression createBooleanConstant(boolean val) { return new PrestoBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static PrestoExpression createDateConstant(long integer) { return new PrestoDateConstant(integer); } - public static Node createTimeConstant(long integer) { + public static PrestoExpression createTimeConstant(long integer) { return new PrestoTimeConstant(integer); } - public static Node createTimeWithTimeZoneConstant(long integer) { + public static PrestoExpression createTimeWithTimeZoneConstant(long integer) { return new PrestoTimeWithTimeZoneConstant(integer); } - public static Node createTimestampWithTimeZoneConstant(long integer) { + public static PrestoExpression createTimestampWithTimeZoneConstant(long integer) { return new PrestoTimestampWithTimezoneConstant(integer); } - public static Node createIntervalDayToSecond(long integer) { + public static PrestoExpression createIntervalDayToSecond(long integer) { return new PrestoIntervalDayToSecondConstant(); } - public static Node createIntervalYearToMonth(long integer) { + public static PrestoExpression createIntervalYearToMonth(long integer) { return new PrestoIntervalYearToMonthConstant(); } - public static Node createTimestampConstant(long integer) { + public static PrestoExpression createTimestampConstant(long integer) { return new PrestoTimestampConstant(integer); } - public static Node createVarbinaryConstant(String string) { + public static PrestoExpression createVarbinaryConstant(String string) { return new PrestoVarbinaryConstant(string); } - public static Node createTimezoneConstant() { + public static PrestoExpression createTimezoneConstant() { String string = Randomly.fromOptions(TIME_ZONES); return new PrestoTextConstant(string); } - public static Node createArrayConstant(PrestoSchema.PrestoCompositeDataType type) { + public static PrestoExpression createArrayConstant(PrestoSchema.PrestoCompositeDataType type) { PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); long size = Randomly.getNotCachedInteger(0, 10); - List> elements = new ArrayList<>(); + List elements = new ArrayList<>(); for (int i = 0; i <= size; i++) { if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { elements.add(createArrayConstant(elementType)); @@ -248,11 +247,11 @@ public static Node createArrayConstant(PrestoSchema.PrestoComp return new PrestoArrayConstant(elements); } - public static Node createMapConstant(PrestoSchema.PrestoCompositeDataType type) { + public static PrestoExpression createMapConstant(PrestoSchema.PrestoCompositeDataType type) { PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); long size = Randomly.getNotCachedInteger(0, 10); - List> elements = new ArrayList<>(); + List elements = new ArrayList<>(); for (int i = 0; i <= size; i++) { if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { elements.add(createArrayConstant(elementType)); @@ -263,8 +262,7 @@ public static Node createMapConstant(PrestoSchema.PrestoCompos return new PrestoArrayConstant(elements); } - public static Node generateConstant(PrestoSchema.PrestoCompositeDataType type, - boolean castInteger) { + public static PrestoExpression generateConstant(PrestoSchema.PrestoCompositeDataType type, boolean castInteger) { Randomly randomly = new Randomly(); switch (type.getPrimitiveDataType()) { case ARRAY: @@ -789,9 +787,9 @@ public boolean isBoolean() { public static class PrestoArrayConstant extends PrestoConstant { - private final List> elements; + private final List elements; - public PrestoArrayConstant(List> elements) { + public PrestoArrayConstant(List elements) { this.elements = new ArrayList<>(elements); } diff --git a/src/sqlancer/presto/ast/PrestoDefaultFunction.java b/src/sqlancer/presto/ast/PrestoDefaultFunction.java index 95b89091e..99b7f7fde 100644 --- a/src/sqlancer/presto/ast/PrestoDefaultFunction.java +++ b/src/sqlancer/presto/ast/PrestoDefaultFunction.java @@ -6,7 +6,6 @@ import java.util.stream.Stream; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; import sqlancer.presto.PrestoSchema.PrestoDataType; @@ -75,7 +74,7 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { return super.getArgumentsForReturnType(gen, depth, argumentTypes, returnType); } @@ -177,9 +176,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); // This is a workaround based on the assumption that array types should refer to the same element type. PrestoCompositeDataType savedArrayType = null; diff --git a/src/sqlancer/presto/ast/PrestoFunction.java b/src/sqlancer/presto/ast/PrestoFunction.java index c91759e2f..fc7ef5b79 100644 --- a/src/sqlancer/presto/ast/PrestoFunction.java +++ b/src/sqlancer/presto/ast/PrestoFunction.java @@ -4,11 +4,10 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public interface PrestoFunction { +public interface PrestoFunction extends PrestoExpression { String getFunctionName(); @@ -16,10 +15,10 @@ public interface PrestoFunction { PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType); - default List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + default List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoSchema.PrestoDataType[] argumentTypes, PrestoSchema.PrestoCompositeDataType returnType) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); // This is a workaround based on the assumption that array types should refer to // the same element type. @@ -56,17 +55,17 @@ default List> getArgumentsForReturnType(PrestoTypedExpres } else { dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); } - Node expression = gen.generateExpression(dataType, depth + 1); + PrestoExpression expression = gen.generateExpression(dataType, depth + 1); arguments.add(expression); } } return arguments; } - default List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + default List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoSchema.PrestoCompositeDataType returnType, boolean orderable) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); // This is a workaround based on the assumption that array types should refer to // the same element type. @@ -105,7 +104,7 @@ default List> getArgumentsForReturnType(PrestoTypedExpres } else { compositeDataType = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); } - Node expression = gen.generateExpression(compositeDataType, depth + 1); + PrestoExpression expression = gen.generateExpression(compositeDataType, depth + 1); arguments.add(expression); } } diff --git a/src/sqlancer/presto/ast/PrestoFunctionNode.java b/src/sqlancer/presto/ast/PrestoFunctionNode.java new file mode 100644 index 000000000..14409a825 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoFunctionNode.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class PrestoFunctionNode extends NewFunctionNode implements PrestoExpression { + public PrestoFunctionNode(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java index 5b184671b..88007053d 100644 --- a/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java +++ b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java @@ -1,9 +1,8 @@ package sqlancer.presto.ast; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; -public class PrestoFunctionWithoutParenthesis implements Node { +public class PrestoFunctionWithoutParenthesis implements PrestoExpression { private final PrestoSchema.PrestoCompositeDataType type; private final String expr; diff --git a/src/sqlancer/presto/ast/PrestoInOperation.java b/src/sqlancer/presto/ast/PrestoInOperation.java new file mode 100644 index 000000000..95edd7fc3 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoInOperation.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class PrestoInOperation extends NewInOperatorNode implements PrestoExpression { + public PrestoInOperation(PrestoExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/presto/ast/PrestoJoin.java b/src/sqlancer/presto/ast/PrestoJoin.java index 79bf7e1fc..0732e55be 100644 --- a/src/sqlancer/presto/ast/PrestoJoin.java +++ b/src/sqlancer/presto/ast/PrestoJoin.java @@ -4,37 +4,32 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoColumn; -import sqlancer.presto.PrestoSchema.PrestoTable; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoJoin implements Node { +public class PrestoJoin implements PrestoExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final PrestoTableReference leftTable; + private final PrestoTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final PrestoExpression onCondition; private OuterType outerType; - public PrestoJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public PrestoJoin(PrestoTableReference leftTable, PrestoTableReference rightTable, JoinType joinType, + PrestoExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public static List> getJoins( - List> tableList, PrestoGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, PrestoGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + PrestoTableReference leftTable = tableList.remove(0); + PrestoTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); PrestoTypedExpressionGenerator joinGen = new PrestoTypedExpressionGenerator(globalState) @@ -59,26 +54,26 @@ public static List> getJoins( return joinExpressions; } - public static PrestoJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static PrestoJoin createRightOuterJoin(PrestoTableReference left, PrestoTableReference right, + PrestoExpression predicate) { return new PrestoJoin(left, right, JoinType.RIGHT, predicate); } - public static PrestoJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static PrestoJoin createLeftOuterJoin(PrestoTableReference left, PrestoTableReference right, + PrestoExpression predicate) { return new PrestoJoin(left, right, JoinType.LEFT, predicate); } - public static PrestoJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static PrestoJoin createInnerJoin(PrestoTableReference left, PrestoTableReference right, + PrestoExpression predicate) { return new PrestoJoin(left, right, JoinType.INNER, predicate); } - public TableReferenceNode getLeftTable() { + public PrestoTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public PrestoTableReference getRightTable() { return rightTable; } @@ -86,7 +81,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public PrestoExpression getOnCondition() { return onCondition; } diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java index 47fc209fb..20ceaee79 100644 --- a/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java @@ -3,16 +3,14 @@ import java.util.ArrayList; import java.util.List; -import sqlancer.common.ast.newast.Node; +public class PrestoMultiValuedComparison implements PrestoExpression { -public class PrestoMultiValuedComparison implements Node { - - private final Node left; - private final List> right; + private final PrestoExpression left; + private final List right; private final PrestoMultiValuedComparisonType type; private final PrestoMultiValuedComparisonOperator op; - public PrestoMultiValuedComparison(Node left, List> right, + public PrestoMultiValuedComparison(PrestoExpression left, List right, PrestoMultiValuedComparisonType type, PrestoMultiValuedComparisonOperator op) { this.left = left; this.right = new ArrayList<>(right); @@ -20,7 +18,7 @@ public PrestoMultiValuedComparison(Node left, List getLeft() { + public PrestoExpression getLeft() { return left; } @@ -28,7 +26,7 @@ public PrestoMultiValuedComparisonOperator getOp() { return op; } - public List> getRight() { + public List getRight() { return new ArrayList<>(right); } diff --git a/src/sqlancer/presto/ast/PrestoPostfixText.java b/src/sqlancer/presto/ast/PrestoPostfixText.java new file mode 100644 index 000000000..de99fede4 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoPostfixText.java @@ -0,0 +1,9 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class PrestoPostfixText extends NewPostfixTextNode implements PrestoExpression { + public PrestoPostfixText(PrestoExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java index d51539645..3cbccc7b2 100644 --- a/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java +++ b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java @@ -1,23 +1,21 @@ package sqlancer.presto.ast; -import sqlancer.common.ast.newast.Node; +public class PrestoQuantifiedComparison implements PrestoExpression { -public class PrestoQuantifiedComparison implements Node { - - private final Node left; + private final PrestoExpression left; private final PrestoSelect right; private final PrestoMultiValuedComparisonType type; private final PrestoMultiValuedComparisonOperator op; - public PrestoQuantifiedComparison(Node left, PrestoSelect right, - PrestoMultiValuedComparisonType type, PrestoMultiValuedComparisonOperator op) { + public PrestoQuantifiedComparison(PrestoExpression left, PrestoSelect right, PrestoMultiValuedComparisonType type, + PrestoMultiValuedComparisonOperator op) { this.left = left; this.right = right; this.type = type; this.op = op; } - public Node getLeft() { + public PrestoExpression getLeft() { return left; } @@ -25,7 +23,7 @@ public PrestoMultiValuedComparisonOperator getOp() { return op; } - public Node getRight() { + public PrestoExpression getRight() { return right; } diff --git a/src/sqlancer/presto/ast/PrestoSelect.java b/src/sqlancer/presto/ast/PrestoSelect.java index c99c5c7d0..328c154c4 100644 --- a/src/sqlancer/presto/ast/PrestoSelect.java +++ b/src/sqlancer/presto/ast/PrestoSelect.java @@ -1,9 +1,8 @@ package sqlancer.presto.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class PrestoSelect extends SelectBase> implements Node { +public class PrestoSelect extends SelectBase implements PrestoExpression { private boolean isDistinct; diff --git a/src/sqlancer/presto/ast/PrestoTableReference.java b/src/sqlancer/presto/ast/PrestoTableReference.java index 4c0e0d789..7552ecc64 100644 --- a/src/sqlancer/presto/ast/PrestoTableReference.java +++ b/src/sqlancer/presto/ast/PrestoTableReference.java @@ -3,7 +3,8 @@ import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.presto.PrestoSchema; -public class PrestoTableReference extends TableReferenceNode { +public class PrestoTableReference extends TableReferenceNode + implements PrestoExpression { public PrestoTableReference(PrestoSchema.PrestoTable table) { super(table); diff --git a/src/sqlancer/presto/ast/PrestoTernary.java b/src/sqlancer/presto/ast/PrestoTernary.java new file mode 100644 index 000000000..618daa0fc --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoTernary.java @@ -0,0 +1,10 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewTernaryNode; + +public class PrestoTernary extends NewTernaryNode implements PrestoExpression { + public PrestoTernary(PrestoExpression left, PrestoExpression middle, PrestoExpression right, String leftStr, + String rightStr) { + super(left, middle, right, leftStr, rightStr); + } +} diff --git a/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java index 2bdc142e6..99677d3f8 100644 --- a/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java +++ b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java @@ -3,16 +3,16 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; -public class PrestoUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { +public class PrestoUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements PrestoExpression { - public PrestoUnaryPostfixOperation(Node expr, PrestoUnaryPostfixOperator op) { + public PrestoUnaryPostfixOperation(PrestoExpression expr, PrestoUnaryPostfixOperator op) { super(expr, op); } - public Node getExpression() { + public PrestoExpression getExpression() { return getExpr(); } diff --git a/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java index cb8313622..db7a08068 100644 --- a/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java +++ b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java @@ -3,12 +3,12 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; -public class PrestoUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { +public class PrestoUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements PrestoExpression { - public PrestoUnaryPrefixOperation(PrestoUnaryPrefixOperator operation, Node expression) { + public PrestoUnaryPrefixOperation(PrestoExpression expression, BinaryOperatorNode.Operator operation) { super(expression, operation); } diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java index 7a467e37d..c76283ee4 100644 --- a/src/sqlancer/presto/gen/PrestoIndexGenerator.java +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoGlobalState; @@ -46,7 +45,7 @@ public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { sb.append(")"); if (Randomly.getBoolean()) { sb.append(" WHERE "); - Node expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) + PrestoExpression expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); sb.append(PrestoToStringVisitor.asString(expr)); } diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index 102d2f1be..81e50e1d1 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -43,7 +42,7 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(PrestoColumn prestoColumn) { - Node constant = new PrestoTypedExpressionGenerator(globalState) + PrestoExpression constant = new PrestoTypedExpressionGenerator(globalState) .generateInsertConstant(prestoColumn.getType()); sb.append(PrestoToStringVisitor.asString(constant)); diff --git a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java index 2d18311b6..27a54d926 100644 --- a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java +++ b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java @@ -5,8 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoTable; @@ -15,6 +13,7 @@ import sqlancer.presto.ast.PrestoExpression; import sqlancer.presto.ast.PrestoJoin; import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; public final class PrestoRandomQuerySynthesizer { @@ -29,10 +28,10 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC // TODO: distinct // select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = gen + PrestoExpression expression = gen .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); columns.add(expression); // } else { @@ -41,9 +40,9 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC } select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = PrestoJoin.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, globalState); select.setJoinList(new ArrayList<>(joins)); select.setFromList(new ArrayList<>(tableList)); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java index 79e657fda..712ebdcc3 100644 --- a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -9,38 +9,35 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewCaseOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewTernaryNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.ast.PrestoAggregateFunction; import sqlancer.presto.ast.PrestoAtTimeZoneOperator; +import sqlancer.presto.ast.PrestoBetweenOperation; +import sqlancer.presto.ast.PrestoBinaryOperation; +import sqlancer.presto.ast.PrestoCaseOperation; import sqlancer.presto.ast.PrestoCastFunction; import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoConstant; import sqlancer.presto.ast.PrestoDefaultFunction; import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoFunctionNode; +import sqlancer.presto.ast.PrestoInOperation; import sqlancer.presto.ast.PrestoJoin; import sqlancer.presto.ast.PrestoMultiValuedComparison; import sqlancer.presto.ast.PrestoMultiValuedComparisonOperator; import sqlancer.presto.ast.PrestoMultiValuedComparisonType; import sqlancer.presto.ast.PrestoQuantifiedComparison; import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; +import sqlancer.presto.ast.PrestoTernary; import sqlancer.presto.ast.PrestoUnaryPostfixOperation; import sqlancer.presto.ast.PrestoUnaryPrefixOperation; public final class PrestoTypedExpressionGenerator extends - TypedExpressionGenerator, PrestoSchema.PrestoColumn, PrestoSchema.PrestoCompositeDataType> { + TypedExpressionGenerator { private final Randomly randomly; private final PrestoGlobalState globalState; @@ -53,24 +50,24 @@ public PrestoTypedExpressionGenerator(PrestoGlobalState globalState) { } @Override - public Node generatePredicate() { + public PrestoExpression generatePredicate() { return generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), randomly.getInteger(0, maxDepth)); } @Override - public Node negatePredicate(Node predicate) { - return new PrestoUnaryPrefixOperation(PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT, predicate); + public PrestoExpression negatePredicate(PrestoExpression predicate) { + return new PrestoUnaryPrefixOperation(predicate, PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { + public PrestoExpression isNull(PrestoExpression expr) { return new PrestoUnaryPostfixOperation(expr, PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); } @Override - public Node generateConstant(PrestoSchema.PrestoCompositeDataType type) { + public PrestoExpression generateConstant(PrestoSchema.PrestoCompositeDataType type) { if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { return PrestoConstant.createArrayConstant(type); // case MAP: @@ -79,7 +76,7 @@ public Node generateConstant(PrestoSchema.PrestoCompositeDataT return PrestoConstant.generateConstant(type, false); } - public Node generateInsertConstant(PrestoSchema.PrestoCompositeDataType type) { + public PrestoExpression generateInsertConstant(PrestoSchema.PrestoCompositeDataType type) { if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { return PrestoConstant.createArrayConstant(type); // case MAP: @@ -89,7 +86,7 @@ public Node generateInsertConstant(PrestoSchema.PrestoComposit } @Override - public Node generateExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + public PrestoExpression generateExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { if (allowAggregates && Randomly.getBoolean()) { return generateAggregate(type); } @@ -146,52 +143,52 @@ public Node generateExpression(PrestoSchema.PrestoCompositeDat } } - private Node generateJsonExpression(PrestoSchema.PrestoCompositeDataType type) { + private PrestoExpression generateJsonExpression(PrestoSchema.PrestoCompositeDataType type) { return generateLeafNode(type); } - private Node generateCast(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateCast(PrestoSchema.PrestoCompositeDataType type, int depth) { // check can cast - Node expressionNode = generateExpression(getRandomType(), depth + 1); + PrestoExpression expressionNode = generateExpression(getRandomType(), depth + 1); return new PrestoCastFunction(expressionNode, type); } @SuppressWarnings("unused") - private Node generateTry(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateTry(PrestoSchema.PrestoCompositeDataType type, int depth) { if (type.getPrimitiveDataType().isNumeric() && Randomly.getBooleanWithRatherLowProbability()) { - Node expression = generateExpression(type); - return new NewFunctionNode<>(List.of(expression), "try"); + PrestoExpression expression = generateExpression(type); + return new PrestoFunctionNode<>(List.of(expression), "try"); } List applicableFunctions = PrestoDefaultFunction.getFunctionsCompatibleWith(type); if (Randomly.getBooleanWithRatherLowProbability() && !applicableFunctions.isEmpty()) { PrestoDefaultFunction function = Randomly.fromList(applicableFunctions); - Node expression = generateFunction(type, depth, function); - return new NewFunctionNode<>(List.of(expression), "try"); + PrestoExpression expression = generateFunction(type, depth, function); + return new PrestoFunctionNode<>(List.of(expression), "try"); } - return new NewFunctionNode<>(List.of(generateCast(type, depth)), "try"); + return new PrestoFunctionNode<>(List.of(generateCast(type, depth)), "try"); } - private NewCaseOperatorNode getCase(PrestoSchema.PrestoCompositeDataType type, int depth) { - List> conditions = new ArrayList<>(); - List> cases = new ArrayList<>(); + private PrestoCaseOperation getCase(PrestoSchema.PrestoCompositeDataType type, int depth) { + List conditions = new ArrayList<>(); + List cases = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { conditions.add(generateExpression(type, depth + 1)); cases.add(generateExpression(type, depth + 1)); } - Node elseExpr = null; + PrestoExpression elseExpr = null; if (Randomly.getBoolean()) { elseExpr = generateExpression(type, depth + 1); } - Node expression = generateExpression(type); - return new NewCaseOperatorNode<>(expression, conditions, cases, elseExpr); + PrestoExpression expression = generateExpression(type); + return new PrestoCaseOperation(expression, conditions, cases, elseExpr); } - private Node generateFunction(PrestoSchema.PrestoCompositeDataType returnType, int depth, + private PrestoExpression generateFunction(PrestoSchema.PrestoCompositeDataType returnType, int depth, PrestoDefaultFunction function) { PrestoSchema.PrestoDataType[] argumentTypes = function.getArgumentTypes(returnType); - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); // This is a workaround based on the assumption that array types should refer to // the same element type. @@ -227,34 +224,34 @@ private Node generateFunction(PrestoSchema.PrestoCompositeData } else { dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); } - Node expression = generateExpression(dataType, depth + 1); + PrestoExpression expression = generateExpression(dataType, depth + 1); arguments.add(expression); } } - return new NewFunctionNode<>(arguments, function); + return new PrestoFunctionNode<>(arguments, function); } - private Node generateStringExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateStringExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(type); } return getStringOperation(depth); } - private NewBinaryOperatorNode getStringOperation(int depth) { + private PrestoBinaryOperation getStringOperation(int depth) { StringExpression exprType = Randomly.fromOptions(StringExpression.values()); if (Objects.requireNonNull(exprType) == StringExpression.CONCAT) { - Node left = generateExpression( + PrestoExpression left = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); - Node right = generateExpression( + PrestoExpression right = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); PrestBinaryStringOperator operator = PrestBinaryStringOperator.CONCAT; - return new NewBinaryOperatorNode<>(left, right, operator); + return new PrestoBinaryOperation(left, right, operator); } throw new AssertionError(exprType); } - private Node generateBooleanExpression(int depth) { + private PrestoExpression generateBooleanExpression(int depth) { List booleanExpressions = Arrays.stream(BooleanExpression.values()) .collect(Collectors.toList()); if (!globalState.getDbmsSpecificOptions().testBetween) { @@ -292,7 +289,7 @@ private Node generateBooleanExpression(int depth) { } } - private Node getMultiValuedComparison(int depth) { + private PrestoExpression getMultiValuedComparison(int depth) { PrestoSchema.PrestoCompositeDataType type; do { @@ -304,7 +301,7 @@ private Node getMultiValuedComparison(int depth) { PrestoMultiValuedComparisonType comparisonType = PrestoMultiValuedComparisonType.getRandom(); PrestoMultiValuedComparisonOperator comparisonOperator = PrestoMultiValuedComparisonOperator .getRandomForType(type); - Node left = generateExpression(type, depth + 1); + PrestoExpression left = generateExpression(type, depth + 1); // sub-query PrestoSchema.PrestoCompositeDataType finalType = type; List columnsOfType = columns.stream().filter(c -> c.getType() == finalType) @@ -315,7 +312,7 @@ private Node getMultiValuedComparison(int depth) { return new PrestoQuantifiedComparison(left, subquery, comparisonType, comparisonOperator); } int nr = Randomly.smallNumber() + 2; - List> rightList = new ArrayList<>(); + List rightList = new ArrayList<>(); for (int i = 0; i < nr; i++) { rightList.add(generateConstant(type)); } @@ -324,53 +321,50 @@ private Node getMultiValuedComparison(int depth) { private PrestoSelect generateSubquery(List columns) { PrestoSelect select = new PrestoSelect(); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)) + List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) .collect(Collectors.toList()); select.setFetchColumns(allColumns); List tables = columns.stream().map(AbstractTableColumn::getTable) .collect(Collectors.toList()); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).distinct() + List tableList = tables.stream().map(t -> new PrestoTableReference(t)).distinct() .collect(Collectors.toList()); - List> tableNodeList = tables.stream() - .map(t -> new TableReferenceNode(t)) + List tableNodeList = tables.stream().map(t -> new PrestoTableReference(t)) .collect(Collectors.toList()); select.setFromList(tableNodeList); - TypedExpressionGenerator, PrestoSchema.PrestoColumn, PrestoSchema.PrestoCompositeDataType> typedExpressionGenerator = new PrestoTypedExpressionGenerator( + TypedExpressionGenerator typedExpressionGenerator = new PrestoTypedExpressionGenerator( globalState).setColumns(columns); - Node predicate = typedExpressionGenerator.generatePredicate(); + PrestoExpression predicate = typedExpressionGenerator.generatePredicate(); select.setWhereClause(predicate); if (Randomly.getBooleanWithSmallProbability()) { select.setOrderByClauses(typedExpressionGenerator.generateOrderBys()); } - List> joins = PrestoJoin.getJoins(tableList, globalState); + List joins = PrestoJoin.getJoins(tableList, globalState); select.setJoinList(joins); return select; } - private Node generateNumericExpression(int depth) { + private PrestoExpression generateNumericExpression(int depth) { PrestoSchema.PrestoDataType dataType = Randomly.fromList(PrestoSchema.PrestoDataType.getNumberTypes()); PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); if (Randomly.getBoolean()) { BinaryOperatorNode.Operator operator = PrestoBinaryArithmeticOperator.getRandom(); - Node left = generateExpression(type, depth); - Node right = generateExpression(type, depth); - return new NewBinaryOperatorNode<>(left, right, operator); + PrestoExpression left = generateExpression(type, depth); + PrestoExpression right = generateExpression(type, depth); + return new PrestoBinaryOperation(left, right, operator); } else { BinaryOperatorNode.Operator operator = PrestoUnaryArithmeticOperator.MINUS; - Node left = generateExpression(type, depth); - return new NewUnaryPrefixOperatorNode<>(left, operator); + PrestoExpression left = generateExpression(type, depth); + return new PrestoUnaryPrefixOperation(left, operator); } } - private Node generateTemporalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateTemporalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { if (Randomly.getBooleanWithSmallProbability()) { - Node left = generateExpression(type, depth); - Node right = generateExpression(PrestoSchema.PrestoCompositeDataType + PrestoExpression left = generateExpression(type, depth); + PrestoExpression right = generateExpression(PrestoSchema.PrestoCompositeDataType .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getIntervalTypes())), depth); BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); - return new NewBinaryOperatorNode<>(left, right, operator); + return new PrestoBinaryOperation(left, right, operator); } // timestamp at time zone @@ -383,11 +377,11 @@ private Node generateTemporalExpression(PrestoSchema.PrestoCom return generateLeafNode(type); } - private Node generateIntervalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateIntervalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { if (Randomly.getBooleanWithSmallProbability()) { - Node left = generateExpression(type, depth); + PrestoExpression left = generateExpression(type, depth); - Node right; + PrestoExpression right; if (Randomly.getBoolean()) { right = generateExpression(PrestoSchema.PrestoCompositeDataType .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getTemporalTypes())), depth); @@ -396,9 +390,9 @@ private Node generateIntervalExpression(PrestoSchema.PrestoCom } BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); if (Randomly.getBoolean()) { - return new NewBinaryOperatorNode<>(left, right, operator); + return new PrestoBinaryOperation(left, right, operator); } else { - return new NewBinaryOperatorNode<>(right, left, operator); + return new PrestoBinaryOperation(right, left, operator); } } return generateLeafNode(type); @@ -408,17 +402,17 @@ private Node generateIntervalExpression(PrestoSchema.PrestoCom // timestamp at time zone } - private Node getLike(int depth) { + private PrestoExpression getLike(int depth) { PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType .fromDataType(PrestoSchema.PrestoDataType.VARCHAR); - Node expression = generateExpression(type, depth + 1); - Node pattern = generateExpression(type, depth + 1); + PrestoExpression expression = generateExpression(type, depth + 1); + PrestoExpression pattern = generateExpression(type, depth + 1); if (Randomly.getBoolean()) { - return new NewBinaryOperatorNode<>(expression, pattern, PrestoLikeOperator.getRandom()); + return new PrestoBinaryOperation(expression, pattern, PrestoLikeOperator.getRandom()); } else { String randomlyString = randomly.getString(); String randomlyChar = randomly.getChar(); - Node escape = new PrestoConstant.PrestoTextConstant(randomlyChar, 1); + PrestoExpression escape = new PrestoConstant.PrestoTextConstant(randomlyChar, 1); int index = randomlyString.indexOf(randomlyChar); while (index > -1) { String wildcard = Randomly.fromOptions("%", "_"); @@ -427,73 +421,75 @@ private Node getLike(int depth) { index = randomlyString.indexOf(randomlyChar, index + 1); } PrestoConstant.PrestoTextConstant patternString = new PrestoConstant.PrestoTextConstant(randomlyString); - return new NewTernaryNode<>(expression, patternString, escape, "LIKE", "ESCAPE"); + return new PrestoTernary(expression, patternString, escape, "LIKE", "ESCAPE"); } } - private NewBinaryOperatorNode getRegex(int depth) { - Node left = generateExpression( + private PrestoBinaryOperation getRegex(int depth) { + PrestoExpression left = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); - Node right = generateExpression( + PrestoExpression right = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); - return new NewBinaryOperatorNode<>(left, right, PrestoBinaryLogicalOperator.getRandom()); + return new PrestoBinaryOperation(left, right, PrestoBinaryLogicalOperator.getRandom()); } - private NewBinaryOperatorNode getBinaryLogical(int depth) { + private PrestoBinaryOperation getBinaryLogical(int depth) { PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType .fromDataType(PrestoSchema.PrestoDataType.BOOLEAN); - Node left = generateExpression(type, depth + 1); - Node right = generateExpression(type, depth + 1); + PrestoExpression left = generateExpression(type, depth + 1); + PrestoExpression right = generateExpression(type, depth + 1); BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode<>(left, right, operator); + return new PrestoBinaryOperation(left, right, operator); } - private Node getBetween(int depth) { + private PrestoExpression getBetween(int depth) { PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getNumericTypes())); - Node expression = generateExpression(type, depth + 1); - Node left = generateExpression(type, depth + 1); - Node right = generateExpression(type, depth + 1); - return new NewBetweenOperatorNode<>(expression, left, right, Randomly.getBoolean()); + PrestoExpression expression = generateExpression(type, depth + 1); + PrestoExpression left = generateExpression(type, depth + 1); + PrestoExpression right = generateExpression(type, depth + 1); + return new PrestoBetweenOperation(expression, left, right, Randomly.getBoolean()); } - private Node getInOperation(int depth) { + private PrestoExpression getInOperation(int depth) { PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType .fromDataType(PrestoSchema.PrestoDataType.getRandomWithoutNull()); - Node left = generateExpression(type, depth + 1); - List> inList = generateExpressions(type, Randomly.smallNumber() + 1, depth + 1); + PrestoExpression left = generateExpression(type, depth + 1); + List inList = generateExpressions(type, Randomly.smallNumber() + 1, depth + 1); boolean isNegated = Randomly.getBoolean(); - return new NewInOperatorNode<>(left, inList, isNegated); + return new PrestoInOperation(left, inList, isNegated); } - private Node getAndOrChain(int depth) { - Node left = generateExpression( + private PrestoExpression getAndOrChain(int depth) { + PrestoExpression left = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - Node right = generateExpression( + PrestoExpression right = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); - left = new NewBinaryOperatorNode<>(left, right, operator); + left = new PrestoBinaryOperation(left, right, operator); } return left; } - private Node getBinaryComparison(int depth) { + private PrestoExpression getBinaryComparison(int depth) { PrestoSchema.PrestoCompositeDataType type = getRandomType(); BinaryOperatorNode.Operator op = PrestoBinaryComparisonOperator.getRandomForType(type); - Node left = generateExpression(type, depth + 1); - Node right = generateExpression(type, depth + 1); - return new NewBinaryOperatorNode<>(left, right, op); + PrestoExpression left = generateExpression(type, depth + 1); + PrestoExpression right = generateExpression(type, depth + 1); + return new PrestoBinaryOperation(left, right, op); } - private Node generateNOT(int depth) { + private PrestoExpression generateNOT(int depth) { PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator operator = PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT; - return new PrestoUnaryPrefixOperation(operator, generateExpression( - PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth)); + return new PrestoUnaryPrefixOperation( + generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth), + operator); } @Override - protected Node generateColumn(PrestoSchema.PrestoCompositeDataType type) { + protected PrestoExpression generateColumn(PrestoSchema.PrestoCompositeDataType type) { List columnList = columns.stream() .filter(c -> c.getType().getPrimitiveDataType() == type.getPrimitiveDataType()) .collect(Collectors.toList()); @@ -502,7 +498,7 @@ protected Node generateColumn(PrestoSchema.PrestoCompositeData } @Override - public Node generateLeafNode(PrestoSchema.PrestoCompositeDataType type) { + public PrestoExpression generateLeafNode(PrestoSchema.PrestoCompositeDataType type) { if (Randomly.getBoolean()) { return generateConstant(type); } else { @@ -534,13 +530,13 @@ protected boolean canGenerateColumnOfType(PrestoSchema.PrestoCompositeDataType t return columns.stream().anyMatch(c -> c.getType() == type); } - public Node generateAggregate() { + public PrestoExpression generateAggregate() { PrestoAggregateFunction aggregateFunction = PrestoAggregateFunction.getRandom(); - List> argsForAggregate = generateArgsForAggregate(aggregateFunction); - return new NewFunctionNode<>(argsForAggregate, aggregateFunction); + List argsForAggregate = generateArgsForAggregate(aggregateFunction); + return new PrestoFunctionNode<>(argsForAggregate, aggregateFunction); } - public List> generateArgsForAggregate(PrestoAggregateFunction aggregateFunction) { + public List generateArgsForAggregate(PrestoAggregateFunction aggregateFunction) { PrestoSchema.PrestoCompositeDataType returnType; do { returnType = aggregateFunction.getCompositeReturnType(); @@ -548,17 +544,17 @@ public List> generateArgsForAggregate(PrestoAggregateFunc return aggregateFunction.getArgumentsForReturnType(this, this.maxDepth - 1, returnType, false); } - private Node generateAggregate(PrestoSchema.PrestoCompositeDataType type) { + private PrestoExpression generateAggregate(PrestoSchema.PrestoCompositeDataType type) { PrestoAggregateFunction aggregateFunction = Randomly .fromList(PrestoAggregateFunction.getFunctionsCompatibleWith(type)); - List> argsForAggregate = generateArgsForAggregate(type, aggregateFunction); - return new NewFunctionNode<>(argsForAggregate, aggregateFunction); + List argsForAggregate = generateArgsForAggregate(type, aggregateFunction); + return new PrestoFunctionNode<>(argsForAggregate, aggregateFunction); } - public List> generateArgsForAggregate(PrestoSchema.PrestoCompositeDataType type, + public List generateArgsForAggregate(PrestoSchema.PrestoCompositeDataType type, PrestoAggregateFunction aggregateFunction) { List returnTypes = aggregateFunction.getReturnTypes(type.getPrimitiveDataType()); - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); allowAggregates = false; // for (PrestoSchema.PrestoDataType argumentType : returnTypes) { arguments.add(generateExpression(PrestoSchema.PrestoCompositeDataType.fromDataType(argumentType))); @@ -568,8 +564,8 @@ public List> generateArgsForAggregate(PrestoSchema.Presto } @Override - public List> generateOrderBys() { - List> expressions = new ArrayList<>(); + public List generateOrderBys() { + List expressions = new ArrayList<>(); int nr = Randomly.smallNumber() + 1; ArrayList prestoColumns = new ArrayList<>(columns); prestoColumns.removeIf(c -> !c.isOrderable()); @@ -582,15 +578,14 @@ public List> generateOrderBys() { return expressions; } - public Node generateHavingClause() { + public PrestoExpression generateHavingClause() { allowAggregates = true; - Node expr = generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + PrestoExpression expr = generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); allowAggregates = false; return expr; } - public Node generateExpressionWithColumns(List columns, - int remainingDepth) { + public PrestoExpression generateExpressionWithColumns(List columns, int remainingDepth) { if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { return generateConstant(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); } @@ -613,7 +608,7 @@ public Node generateExpressionWithColumns(List(generateExpression(column.getType(), remainingDepth - 1), + return new PrestoBinaryOperation(generateExpression(column.getType(), remainingDepth - 1), generateExpression(column.getType(), remainingDepth - 1), op); } diff --git a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java index bdd8cf2a3..a8afcc578 100644 --- a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java +++ b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoErrors; @@ -40,7 +39,7 @@ private SQLQueryAdapter generate() { @Override protected void updateValue(PrestoColumn column) { - Node expr; + PrestoExpression expr; if (Randomly.getBooleanWithSmallProbability()) { expr = gen.generateExpression(column.getType()); PrestoErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java index da09b5a01..606183828 100644 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -10,10 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -28,9 +24,12 @@ import sqlancer.presto.PrestoSchema.PrestoTables; import sqlancer.presto.PrestoToStringVisitor; import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoExpression; import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoPostfixText; import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; public class PrestoNoRECOracle extends NoRECBase implements TestOracle { @@ -50,12 +49,12 @@ public void check() throws SQLException { List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = PrestoJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, state); PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generatePredicate(); + PrestoExpression randomWhereCondition = gen.generatePredicate(); int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); @@ -68,14 +67,14 @@ public void check() throws SQLException { } } - private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + private int getSecondQuery(List tableList, PrestoExpression randomWhereCondition, + List joins) throws SQLException { PrestoSelect select = new PrestoSelect(); - Node asText = new NewPostfixTextNode<>( + PrestoExpression asText = new PrestoPostfixText( new PrestoCastFunction( - new NewPostfixTextNode<>(randomWhereCondition, + new PrestoPostfixText(randomWhereCondition, " IS NOT NULL AND " + PrestoToStringVisitor.asString(randomWhereCondition)), new PrestoCompositeDataType(PrestoDataType.INT, 8, 0)), "as count"); @@ -104,12 +103,11 @@ private int getSecondQuery(List> tableList, Node> tableList, - List columns, Node randomWhereCondition, - List> joins) { + private int getFirstQueryCount(SQLConnection con, List tableList, List columns, + PrestoExpression randomWhereCondition, List joins) { PrestoSelect select = new PrestoSelect(); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java index 0f10a22d4..7ab23286a 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java @@ -8,11 +8,6 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; @@ -22,8 +17,10 @@ import sqlancer.presto.PrestoSchema.PrestoDataType; import sqlancer.presto.PrestoToStringVisitor; import sqlancer.presto.ast.PrestoAggregateFunction; +import sqlancer.presto.ast.PrestoAlias; import sqlancer.presto.ast.PrestoCastFunction; import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoFunctionNode; import sqlancer.presto.ast.PrestoSelect; import sqlancer.presto.ast.PrestoUnaryPostfixOperation; import sqlancer.presto.ast.PrestoUnaryPrefixOperation; @@ -49,8 +46,8 @@ public void check() throws SQLException { PrestoAggregateFunction aggregateFunction = Randomly.fromOptions(PrestoAggregateFunction.MAX, PrestoAggregateFunction.MIN, PrestoAggregateFunction.SUM, PrestoAggregateFunction.COUNT, PrestoAggregateFunction.AVG/* , PrestoAggregateFunction.STDDEV_POP */); - List> aggregateArgs = gen.generateArgsForAggregate(aggregateFunction); - NewFunctionNode aggregate = new NewFunctionNode<>(aggregateArgs, + List aggregateArgs = gen.generateArgsForAggregate(aggregateFunction); + PrestoFunctionNode aggregate = new PrestoFunctionNode<>(aggregateArgs, aggregateFunction); select.setFetchColumns(List.of(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { @@ -84,14 +81,14 @@ public void check() throws SQLException { } private String createMetamorphicUnionQuery(PrestoSelect select, - NewFunctionNode aggregate, List> from) { + PrestoFunctionNode aggregate, List from) { String metamorphicQuery; - Node whereClause = gen.generatePredicate(); - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + PrestoExpression whereClause = gen.generatePredicate(); + PrestoExpression negatedClause = new PrestoUnaryPrefixOperation(whereClause, PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + PrestoExpression notNullClause = new PrestoUnaryPostfixOperation(whereClause, PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); + List mappedAggregate = mapped(aggregate); PrestoSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); PrestoSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); PrestoSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); @@ -151,7 +148,7 @@ private String getAggregateResultType(String queryString) { } } - private List> mapped(NewFunctionNode aggregate) { + private List mapped(PrestoFunctionNode aggregate) { PrestoCastFunction count; switch (aggregate.getFunc()) { case COUNT: @@ -160,9 +157,9 @@ private List> mapped(NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + PrestoFunctionNode sum = new PrestoFunctionNode<>(aggregate.getArgs(), PrestoAggregateFunction.SUM); - count = new PrestoCastFunction(new NewFunctionNode<>(aggregate.getArgs(), PrestoAggregateFunction.COUNT), + count = new PrestoCastFunction(new PrestoFunctionNode<>(aggregate.getArgs(), PrestoAggregateFunction.COUNT), new PrestoCompositeDataType(PrestoDataType.FLOAT, 8, 0)); return aliasArgs(Arrays.asList(sum, count)); default: @@ -170,16 +167,16 @@ private List> mapped(NewFunctionNode> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode<>(expr, "agg" + i++)); + for (PrestoExpression expr : originalAggregateArgs) { + args.add(new PrestoAlias(expr, "agg" + i++)); } return args; } - private String getOuterAggregateFunction(NewFunctionNode aggregate) { + private String getOuterAggregateFunction(PrestoFunctionNode aggregate) { switch (aggregate.getFunc()) { case AVG: return "SUM(CAST(agg0 AS DOUBLE))/CAST(SUM(agg1) AS DOUBLE)"; @@ -190,8 +187,8 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, - Node whereClause, List> joinList) { + private PrestoSelect getSelect(List aggregates, List from, + PrestoExpression whereClause, List joinList) { PrestoSelect leftSelect = new PrestoSelect(); leftSelect.setFetchColumns(aggregates); leftSelect.setFromList(from); diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java index 5ee56e1c8..9ac6ef9f4 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java @@ -6,9 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -18,13 +15,14 @@ import sqlancer.presto.PrestoSchema.PrestoColumn; import sqlancer.presto.PrestoSchema.PrestoTable; import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoExpression; import sqlancer.presto.ast.PrestoJoin; import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, PrestoGlobalState> +public class PrestoQueryPartitioningBase extends TernaryLogicPartitioningOracleBase implements TestOracle { PrestoSchema s; @@ -63,27 +61,27 @@ public void check() throws SQLException { select = new PrestoSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = PrestoJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, state); select.setJoinList(new ArrayList<>(joins)); select.setFromList(new ArrayList<>(tableList)); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new PrestoColumn("*", null, false, false))); + columns.add(new PrestoColumnReference(new PrestoColumn("*", null, false, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); } return columns; } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java index 5d5e2343f..6c3f4bf71 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java @@ -7,12 +7,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; -import sqlancer.presto.PrestoSchema.PrestoColumn; import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoExpression; public class PrestoQueryPartitioningGroupByTester extends PrestoQueryPartitioningBase { @@ -45,9 +43,9 @@ public void check() throws SQLException { } @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); } } diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java index ad11806bf..b53bfc07f 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java @@ -7,7 +7,6 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; @@ -52,12 +51,12 @@ public void check() throws SQLException { } @Override - protected Node generatePredicate() { + protected PrestoExpression generatePredicate() { return gen.generateHavingClause(); } @Override - List> generateFetchColumns() { + List generateFetchColumns() { return Collections.singletonList(gen.generateHavingClause()); } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java new file mode 100644 index 000000000..1f648ed35 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class YCQLBetweenOperator extends NewBetweenOperatorNode implements YCQLExpression { + public YCQLBetweenOperator(YCQLExpression left, YCQLExpression middle, YCQLExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java new file mode 100644 index 000000000..369884e77 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBBinaryOperation extends NewBinaryOperatorNode implements QuestDBExpression { + public QuestDBBinaryOperation(QuestDBExpression left, QuestDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java new file mode 100644 index 000000000..33ba8b619 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java @@ -0,0 +1,13 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.questdb.QuestDBSchema; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBColumnReference extends ColumnReferenceNode + implements QuestDBExpression { + public QuestDBColumnReference(QuestDBSchema.QuestDBColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java new file mode 100644 index 000000000..01c880509 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.duckdb.ast.DuckDBExpression; + +public class DuckDBFunction extends NewFunctionNode implements DuckDBExpression { + public DuckDBFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java new file mode 100644 index 000000000..b050276ea --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBInOperation extends NewInOperatorNode implements QuestDBExpression { + public QuestDBInOperation(QuestDBExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java new file mode 100644 index 000000000..575725a07 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java @@ -0,0 +1,10 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.duckdb.ast.DuckDBExpression; + +public class DuckDBOrderingTerm extends NewOrderingTerm implements DuckDBExpression { + public DuckDBOrderingTerm(DuckDBExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java new file mode 100644 index 000000000..1a5f5e638 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.questdb.QuestDBSchema; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBTableReference extends TableReferenceNode + implements QuestDBExpression { + public QuestDBTableReference(QuestDBSchema.QuestDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java new file mode 100644 index 000000000..a6f63860a --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPostfixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java new file mode 100644 index 000000000..e9546ba8b --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPrefixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} From b3f9c16595189c81f265ac4e601379ee73556393 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 13:31:58 +0800 Subject: [PATCH 0904/1284] [QuestDB] Update expressions to match new AST --- .../questdb/QuestDBToStringVisitor.java | 5 +-- .../questdb/ast/QuestDBBinaryOperation.java | 10 +++++ .../questdb/ast/QuestDBColumnReference.java | 12 ++++++ src/sqlancer/questdb/ast/QuestDBConstant.java | 14 +++---- .../questdb/ast/QuestDBInOperation.java | 11 +++++ src/sqlancer/questdb/ast/QuestDBSelect.java | 3 +- .../questdb/ast/QuestDBTableReference.java | 11 +++++ .../ast/QuestDBUnaryPostfixOperation.java | 11 +++++ .../ast/QuestDBUnaryPrefixOperation.java | 11 +++++ .../gen/QuestDBExpressionGenerator.java | 41 +++++++++---------- .../test/QuestDBQueryPartitioningBase.java | 22 +++++----- 11 files changed, 105 insertions(+), 46 deletions(-) create mode 100644 src/sqlancer/questdb/ast/QuestDBBinaryOperation.java create mode 100644 src/sqlancer/questdb/ast/QuestDBColumnReference.java create mode 100644 src/sqlancer/questdb/ast/QuestDBInOperation.java create mode 100644 src/sqlancer/questdb/ast/QuestDBTableReference.java create mode 100644 src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java create mode 100644 src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java diff --git a/src/sqlancer/questdb/QuestDBToStringVisitor.java b/src/sqlancer/questdb/QuestDBToStringVisitor.java index 7b2c889d2..9363b30b1 100644 --- a/src/sqlancer/questdb/QuestDBToStringVisitor.java +++ b/src/sqlancer/questdb/QuestDBToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.questdb; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.questdb.ast.QuestDBConstant; import sqlancer.questdb.ast.QuestDBExpression; import sqlancer.questdb.ast.QuestDBSelect; @@ -9,7 +8,7 @@ public class QuestDBToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(QuestDBExpression expr) { if (expr instanceof QuestDBConstant) { visit((QuestDBConstant) expr); } else if (expr instanceof QuestDBSelect) { @@ -59,7 +58,7 @@ private void visit(QuestDBSelect select) { } } - public static String asString(Node expr) { + public static String asString(QuestDBExpression expr) { QuestDBToStringVisitor visitor = new QuestDBToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/questdb/ast/QuestDBBinaryOperation.java b/src/sqlancer/questdb/ast/QuestDBBinaryOperation.java new file mode 100644 index 000000000..1b69b2c31 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class QuestDBBinaryOperation extends NewBinaryOperatorNode implements QuestDBExpression { + public QuestDBBinaryOperation(QuestDBExpression left, QuestDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBColumnReference.java b/src/sqlancer/questdb/ast/QuestDBColumnReference.java new file mode 100644 index 000000000..d627538a0 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.questdb.QuestDBSchema; + +public class QuestDBColumnReference extends ColumnReferenceNode + implements QuestDBExpression { + public QuestDBColumnReference(QuestDBSchema.QuestDBColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/questdb/ast/QuestDBConstant.java b/src/sqlancer/questdb/ast/QuestDBConstant.java index f7e5ab0f8..f2e65fbc0 100644 --- a/src/sqlancer/questdb/ast/QuestDBConstant.java +++ b/src/sqlancer/questdb/ast/QuestDBConstant.java @@ -1,8 +1,6 @@ package sqlancer.questdb.ast; -import sqlancer.common.ast.newast.Node; - -public class QuestDBConstant implements Node { +public class QuestDBConstant implements QuestDBExpression { private QuestDBConstant() { } @@ -67,7 +65,7 @@ public String getValue() { } } - public static Node createIntConstant(long val) { + public static QuestDBExpression createIntConstant(long val) { return new QuestDBIntConstant(val); } @@ -95,19 +93,19 @@ public String toString() { } - public static Node createBooleanConstant(boolean val) { + public static QuestDBExpression createBooleanConstant(boolean val) { return new QuestDBBooleanConstant(val); } - public static Node createNullConstant() { + public static QuestDBExpression createNullConstant() { return new QuestDBNullConstant(); } - public static Node createFloatConstant(double val) { + public static QuestDBExpression createFloatConstant(double val) { return new QuestDBDoubleConstant(val); } - public static Node createSymbolConstant(String val) { + public static QuestDBExpression createSymbolConstant(String val) { return new QuestDBSymbolConstant(val); } } diff --git a/src/sqlancer/questdb/ast/QuestDBInOperation.java b/src/sqlancer/questdb/ast/QuestDBInOperation.java new file mode 100644 index 000000000..ee4ba2b59 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBInOperation.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class QuestDBInOperation extends NewInOperatorNode implements QuestDBExpression { + public QuestDBInOperation(QuestDBExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBSelect.java b/src/sqlancer/questdb/ast/QuestDBSelect.java index 023384487..0684b4082 100644 --- a/src/sqlancer/questdb/ast/QuestDBSelect.java +++ b/src/sqlancer/questdb/ast/QuestDBSelect.java @@ -1,9 +1,8 @@ package sqlancer.questdb.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class QuestDBSelect extends SelectBase> implements Node { +public class QuestDBSelect extends SelectBase implements QuestDBExpression { private boolean isDistinct; public void setDistinct(boolean distinct) { diff --git a/src/sqlancer/questdb/ast/QuestDBTableReference.java b/src/sqlancer/questdb/ast/QuestDBTableReference.java new file mode 100644 index 000000000..8e4d387da --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.questdb.QuestDBSchema; + +public class QuestDBTableReference extends TableReferenceNode + implements QuestDBExpression { + public QuestDBTableReference(QuestDBSchema.QuestDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java b/src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java new file mode 100644 index 000000000..0301ba60a --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class QuestDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPostfixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java b/src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java new file mode 100644 index 000000000..8488a0de8 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class QuestDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPrefixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java index 73f57c055..cbb19e705 100644 --- a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java @@ -7,20 +7,19 @@ import sqlancer.Randomly; import sqlancer.Randomly.StringGenerationStrategy; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBDataType; +import sqlancer.questdb.ast.QuestDBBinaryOperation; +import sqlancer.questdb.ast.QuestDBColumnReference; import sqlancer.questdb.ast.QuestDBConstant; import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.questdb.ast.QuestDBInOperation; +import sqlancer.questdb.ast.QuestDBUnaryPostfixOperation; +import sqlancer.questdb.ast.QuestDBUnaryPrefixOperation; -public class QuestDBExpressionGenerator extends UntypedExpressionGenerator, QuestDBColumn> { +public class QuestDBExpressionGenerator extends UntypedExpressionGenerator { private final QuestDBGlobalState globalState; @@ -33,17 +32,17 @@ private enum Expression { } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, QuestDBUnaryPrefixOperator.NOT); + public QuestDBExpression negatePredicate(QuestDBExpression predicate) { + return new QuestDBUnaryPrefixOperation(predicate, QuestDBUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, QuestDBUnaryPostfixOperator.IS_NULL); + public QuestDBExpression isNull(QuestDBExpression expr) { + return new QuestDBUnaryPostfixOperation(expr, QuestDBUnaryPostfixOperator.IS_NULL); } @Override - public Node generateConstant() { + public QuestDBExpression generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return QuestDBConstant.createNullConstant(); } @@ -68,7 +67,7 @@ public Node generateConstant() { } @Override - protected Node generateExpression(int depth) { + protected QuestDBExpression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); } @@ -78,22 +77,22 @@ protected Node generateExpression(int depth) { switch (expr) { case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), + return new QuestDBUnaryPrefixOperation(generateExpression(depth + 1), QuestDBUnaryPrefixOperator.getRandom()); case UNARY_POSTFIX: - return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), + return new QuestDBUnaryPostfixOperation(generateExpression(depth + 1), QuestDBUnaryPostfixOperator.getRandom()); case BINARY_COMPARISON: - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + return new QuestDBBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), QuestDBBinaryComparisonOperator.getRandom()); case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + return new QuestDBBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), QuestDBBinaryArithmeticOperator.getRandom()); case BINARY_LOGICAL: - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + return new QuestDBBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), QuestDBBinaryLogicalOperator.getRandom()); case IN: - return new NewInOperatorNode<>(generateExpression(depth + 1), + return new QuestDBInOperation(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); default: throw new AssertionError("Expression generation failed, depth=" + depth); @@ -101,9 +100,9 @@ protected Node generateExpression(int depth) { } @Override - protected Node generateColumn() { + protected QuestDBExpression generateColumn() { QuestDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode<>(column); + return new QuestDBColumnReference(column); } public enum QuestDBUnaryPostfixOperator implements Operator { diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index 068abbd38..0716e9ccc 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -6,9 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -17,12 +14,14 @@ import sqlancer.questdb.QuestDBSchema; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBTable; +import sqlancer.questdb.ast.QuestDBColumnReference; import sqlancer.questdb.ast.QuestDBExpression; import sqlancer.questdb.ast.QuestDBSelect; +import sqlancer.questdb.ast.QuestDBTableReference; import sqlancer.questdb.gen.QuestDBExpressionGenerator; public class QuestDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, QuestDBGlobalState> + extends TernaryLogicPartitioningOracleBase implements TestOracle { QuestDBSchema s; @@ -35,20 +34,19 @@ protected QuestDBQueryPartitioningBase(QuestDBGlobalState state) { QuestDBErrors.addExpressionErrors(errors); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new QuestDBColumn("*", null, false))); + columns.add(new QuestDBColumnReference(new QuestDBColumn("*", null, false))); } else { - columns = Randomly.nonEmptySubset(targetTable.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)) + columns = Randomly.nonEmptySubset(targetTable.getColumns()).stream().map(c -> new QuestDBColumnReference(c)) .collect(Collectors.toList()); } return columns; } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } @@ -64,8 +62,8 @@ public void check() throws SQLException { select.setFetchColumns(generateFetchColumns()); List tables = new ArrayList<>(); tables.add(targetTable); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List tableList = tables.stream().map(t -> new QuestDBTableReference(t)) + .collect(Collectors.toList()); // Ignore JOINs for now select.setFromList(new ArrayList<>(tableList)); select.setWhereClause(null); From 0ddd0ee2c5aa0169c5dc5b7ddd123b6066fc2155 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 13:53:25 +0800 Subject: [PATCH 0905/1284] [YCQL] Update expressions to match new AST --- .../yugabyte/ycql/YCQLToStringVisitor.java | 5 +- .../ycql/ast/YCQLBetweenOperation.java | 5 +- .../ycql/ast/YCQLBinaryOperation.java | 5 +- .../ycql/ast/YCQLColumnReference.java | 9 +-- .../yugabyte/ycql/ast/YCQLConstant.java | 18 ++--- .../yugabyte/ycql/ast/YCQLFunction.java | 5 +- .../yugabyte/ycql/ast/YCQLInOperation.java | 5 +- .../yugabyte/ycql/ast/YCQLOrderingTerm.java | 5 +- .../yugabyte/ycql/ast/YCQLSelect.java | 3 +- .../yugabyte/ycql/ast/YCQLTableReference.java | 9 +-- .../ycql/ast/YCQLUnaryPostfixOperation.java | 6 +- .../ycql/ast/YCQLUnaryPrefixOperation.java | 6 +- .../ycql/gen/YCQLExpressionGenerator.java | 74 +++++++++---------- .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 3 +- .../ycql/gen/YCQLRandomQuerySynthesizer.java | 10 +-- .../ycql/gen/YCQLUpdateGenerator.java | 3 +- 16 files changed, 74 insertions(+), 97 deletions(-) diff --git a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java index 59adf3e6b..c9f357fe9 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java +++ b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.yugabyte.ycql; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; import sqlancer.yugabyte.ycql.ast.YCQLSelect; @@ -9,7 +8,7 @@ public class YCQLToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(YCQLExpression expr) { if (expr instanceof YCQLConstant) { visit((YCQLConstant) expr); } else if (expr instanceof YCQLSelect) { @@ -55,7 +54,7 @@ private void visit(YCQLSelect select) { } } - public static String asString(Node expr) { + public static String asString(YCQLExpression expr) { YCQLToStringVisitor visitor = new YCQLToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java index 1f648ed35..880007cd8 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java @@ -2,9 +2,8 @@ import sqlancer.common.ast.newast.NewBetweenOperatorNode; -public class YCQLBetweenOperator extends NewBetweenOperatorNode implements YCQLExpression { - public YCQLBetweenOperator(YCQLExpression left, YCQLExpression middle, YCQLExpression right, - boolean isTrue) { +public class YCQLBetweenOperation extends NewBetweenOperatorNode implements YCQLExpression { + public YCQLBetweenOperation(YCQLExpression left, YCQLExpression middle, YCQLExpression right, boolean isTrue) { super(left, middle, right, isTrue); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java index 369884e77..fac2af681 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java @@ -2,10 +2,9 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.questdb.ast.QuestDBExpression; -public class QuestDBBinaryOperation extends NewBinaryOperatorNode implements QuestDBExpression { - public QuestDBBinaryOperation(QuestDBExpression left, QuestDBExpression right, Operator op) { +public class YCQLBinaryOperation extends NewBinaryOperatorNode implements YCQLExpression { + public YCQLBinaryOperation(YCQLExpression left, YCQLExpression right, Operator op) { super(left, right, op); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java index 33ba8b619..9c81e70a5 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java @@ -1,12 +1,11 @@ package sqlancer.yugabyte.ycql.ast; import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.questdb.QuestDBSchema; -import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.yugabyte.ycql.YCQLSchema; -public class QuestDBColumnReference extends ColumnReferenceNode - implements QuestDBExpression { - public QuestDBColumnReference(QuestDBSchema.QuestDBColumn column) { +public class YCQLColumnReference extends ColumnReferenceNode + implements YCQLExpression { + public YCQLColumnReference(YCQLSchema.YCQLColumn column) { super(column); } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java index dd85b4b66..08f04bcfb 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java @@ -3,9 +3,7 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.common.ast.newast.Node; - -public class YCQLConstant implements Node { +public class YCQLConstant implements YCQLExpression { private YCQLConstant() { } @@ -142,31 +140,31 @@ public String toString() { } - public static Node createStringConstant(String text) { + public static YCQLExpression createStringConstant(String text) { return new YCQLTextConstant(text); } - public static Node createFloatConstant(double val) { + public static YCQLExpression createFloatConstant(double val) { return new YCQLDoubleConstant(val); } - public static Node createIntConstant(long val) { + public static YCQLExpression createIntConstant(long val) { return new YCQLIntConstant(val); } - public static Node createNullConstant() { + public static YCQLExpression createNullConstant() { return new YCQLNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static YCQLExpression createBooleanConstant(boolean val) { return new YCQLBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static YCQLExpression createDateConstant(long integer) { return new YCQLDateConstant(integer); } - public static Node createTimestampConstant(long integer) { + public static YCQLExpression createTimestampConstant(long integer) { return new YCQLTimestampConstant(integer); } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java index 01c880509..483fab4c4 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java @@ -3,10 +3,9 @@ import java.util.List; import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBFunction extends NewFunctionNode implements DuckDBExpression { - public DuckDBFunction(List args, F func) { +public class YCQLFunction extends NewFunctionNode implements YCQLExpression { + public YCQLFunction(List args, F func) { super(args, func); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java index b050276ea..49d43be11 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java @@ -3,10 +3,9 @@ import java.util.List; import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.questdb.ast.QuestDBExpression; -public class QuestDBInOperation extends NewInOperatorNode implements QuestDBExpression { - public QuestDBInOperation(QuestDBExpression left, List right, boolean isNegated) { +public class YCQLInOperation extends NewInOperatorNode implements YCQLExpression { + public YCQLInOperation(YCQLExpression left, List right, boolean isNegated) { super(left, right, isNegated); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java index 575725a07..5f647e721 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java @@ -1,10 +1,9 @@ package sqlancer.yugabyte.ycql.ast; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBOrderingTerm extends NewOrderingTerm implements DuckDBExpression { - public DuckDBOrderingTerm(DuckDBExpression expr, Ordering ordering) { +public class YCQLOrderingTerm extends NewOrderingTerm implements YCQLExpression { + public YCQLOrderingTerm(YCQLExpression expr, Ordering ordering) { super(expr, ordering); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java index 646182875..25257057e 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java @@ -1,9 +1,8 @@ package sqlancer.yugabyte.ycql.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class YCQLSelect extends SelectBase> implements Node { +public class YCQLSelect extends SelectBase implements YCQLExpression { private boolean isDistinct; diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java index 1a5f5e638..70c01b820 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java @@ -1,12 +1,11 @@ package sqlancer.yugabyte.ycql.ast; import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.questdb.QuestDBSchema; -import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.yugabyte.ycql.YCQLSchema; -public class QuestDBTableReference extends TableReferenceNode - implements QuestDBExpression { - public QuestDBTableReference(QuestDBSchema.QuestDBTable table) { +public class YCQLTableReference extends TableReferenceNode + implements YCQLExpression { + public YCQLTableReference(YCQLSchema.YCQLTable table) { super(table); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java index a6f63860a..b9165211a 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java @@ -2,11 +2,9 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.questdb.ast.QuestDBExpression; -public class QuestDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode - implements QuestDBExpression { - public QuestDBUnaryPostfixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator op) { +public class YCQLUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements YCQLExpression { + public YCQLUnaryPostfixOperation(YCQLExpression expr, BinaryOperatorNode.Operator op) { super(expr, op); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java index e9546ba8b..bd3b554ee 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java @@ -2,11 +2,9 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.questdb.ast.QuestDBExpression; -public class QuestDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode - implements QuestDBExpression { - public QuestDBUnaryPrefixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator operator) { +public class YCQLUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements YCQLExpression { + public YCQLUnaryPrefixOperation(YCQLExpression expr, BinaryOperatorNode.Operator operator) { super(expr, operator); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java index 7a0a7ed1b..0b0e4dee5 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java @@ -9,24 +9,23 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLDataType; +import sqlancer.yugabyte.ycql.ast.YCQLBetweenOperation; +import sqlancer.yugabyte.ycql.ast.YCQLBinaryOperation; +import sqlancer.yugabyte.ycql.ast.YCQLColumnReference; import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import sqlancer.yugabyte.ycql.ast.YCQLFunction; +import sqlancer.yugabyte.ycql.ast.YCQLInOperation; +import sqlancer.yugabyte.ycql.ast.YCQLOrderingTerm; +import sqlancer.yugabyte.ycql.ast.YCQLUnaryPostfixOperation; +import sqlancer.yugabyte.ycql.ast.YCQLUnaryPrefixOperation; -public final class YCQLExpressionGenerator extends UntypedExpressionGenerator, YCQLColumn> { +public final class YCQLExpressionGenerator extends UntypedExpressionGenerator { private final YCQLGlobalState globalState; @@ -39,37 +38,35 @@ private enum Expression { } @Override - protected Node generateExpression(int depth) { + protected YCQLExpression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); } if (allowAggregates && Randomly.getBoolean()) { YCQLAggregateFunction aggregate = YCQLAggregateFunction.getRandom(); allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); + return new YCQLFunction<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); Expression expr = Randomly.fromList(possibleOptions); switch (expr) { case BINARY_COMPARISON: Operator op = YCQLBinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new YCQLBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_LOGICAL: op = YCQLBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new YCQLBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), YCQLBinaryArithmeticOperator.getRandom()); + return new YCQLBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + YCQLBinaryArithmeticOperator.getRandom()); case FUNC: DBFunction func = DBFunction.getRandom(); - return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + return new YCQLFunction(generateExpressions(func.getNrArgs()), func); case BETWEEN: - return new NewBetweenOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + return new YCQLBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); case IN: - return new NewInOperatorNode(generateExpression(depth + 1), + return new YCQLInOperation(generateExpression(depth + 1), generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); default: throw new AssertionError(expr); @@ -77,13 +74,13 @@ protected Node generateExpression(int depth) { } @Override - protected Node generateColumn() { + protected YCQLExpression generateColumn() { YCQLColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode(column); + return new YCQLColumnReference(column); } @Override - public Node generateConstant() { + public YCQLExpression generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { if (bug14330) { throw new IgnoreMeException(); @@ -111,12 +108,12 @@ public Node generateConstant() { } @Override - public List> generateOrderBys() { - List> expr = super.generateOrderBys(); - List> newExpr = new ArrayList<>(expr.size()); - for (Node curExpr : expr) { + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (YCQLExpression curExpr : expr) { if (Randomly.getBoolean()) { - curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); + curExpr = new YCQLOrderingTerm(curExpr, Ordering.getRandom()); } newExpr.add(curExpr); } @@ -280,25 +277,24 @@ public String getTextRepresentation() { } - public NewFunctionNode generateArgsForAggregate( - YCQLAggregateFunction aggregateFunction) { - return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + public YCQLFunction generateArgsForAggregate(YCQLAggregateFunction aggregateFunction) { + return new YCQLFunction(generateExpressions(aggregateFunction.getNrArgs()), + aggregateFunction); } - public Node generateAggregate() { + public YCQLExpression generateAggregate() { YCQLAggregateFunction aggrFunc = YCQLAggregateFunction.getRandom(); return generateArgsForAggregate(aggrFunc); } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, YCQLUnaryPrefixOperator.NOT); + public YCQLExpression negatePredicate(YCQLExpression predicate) { + return new YCQLUnaryPrefixOperation(predicate, YCQLUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, YCQLUnaryPostfixOperator.IS_NULL); + public YCQLExpression isNull(YCQLExpression expr) { + return new YCQLUnaryPostfixOperation(expr, YCQLUnaryPostfixOperator.IS_NULL); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java index 2dc565196..ab03316cc 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; @@ -41,7 +40,7 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { sb.append(")"); if (Randomly.getBoolean()) { sb.append(" WHERE "); - Node expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) + YCQLExpression expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(); sb.append(YCQLToStringVisitor.asString(expr)); } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java index 05f9ec2dd..f7692ebb0 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java @@ -6,14 +6,13 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTables; import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; import sqlancer.yugabyte.ycql.ast.YCQLSelect; +import sqlancer.yugabyte.ycql.ast.YCQLTableReference; public final class YCQLRandomQuerySynthesizer { @@ -24,15 +23,14 @@ public static YCQLSelect generateSelect(YCQLGlobalState globalState, int nrColum YCQLTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); YCQLExpressionGenerator gen = new YCQLExpressionGenerator(globalState).setColumns(targetTables.getColumns()); YCQLSelect select = new YCQLSelect(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { - Node expression = gen.generateExpression(); + YCQLExpression expression = gen.generateExpression(); columns.add(expression); } select.setFetchColumns(columns); List tables = targetTables.getTables(); - Optional> table = tables.stream() - .map(t -> new TableReferenceNode(t)).findFirst(); + Optional table = tables.stream().map(t -> new YCQLTableReference(t)).findFirst(); select.setFromList(table.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression()); diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java index 695301595..a6c855cf3 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLErrors; @@ -47,7 +46,7 @@ private SQLQueryAdapter generate() { @Override protected void updateValue(YCQLColumn column) { - Node expr; + YCQLExpression expr; if (Randomly.getBooleanWithSmallProbability()) { expr = gen.generateExpression(); YCQLErrors.addExpressionErrors(errors); From 8f84c4eab86fc9df15a54fc1afd9a49d1d592183 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 14:33:52 +0800 Subject: [PATCH 0906/1284] [DataFusion] Update expressions to match new AST --- src/sqlancer/datafusion/DataFusionSchema.java | 3 +- .../datafusion/DataFusionToStringVisitor.java | 11 ++--- .../ast/DataFusionBinaryOperation.java | 11 +++++ .../ast/DataFusionColumnReference.java | 12 +++++ .../datafusion/ast/DataFusionConstant.java | 8 ++-- .../datafusion/ast/DataFusionFunction.java | 11 +++++ .../datafusion/ast/DataFusionJoin.java | 37 +++++++------- .../datafusion/ast/DataFusionSelect.java | 16 ++----- .../ast/DataFusionTableReference.java | 11 +++++ .../ast/DataFusionUnaryPostfixOperation.java | 11 +++++ .../ast/DataFusionUnaryPrefixOperation.java | 11 +++++ .../gen/DataFusionExpressionGenerator.java | 48 +++++++++---------- .../test/DataFusionQueryPartitioningBase.java | 5 +- 13 files changed, 122 insertions(+), 73 deletions(-) create mode 100644 src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionColumnReference.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionFunction.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionTableReference.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java diff --git a/src/sqlancer/datafusion/DataFusionSchema.java b/src/sqlancer/datafusion/DataFusionSchema.java index 4a5ed803a..b9f00a02e 100644 --- a/src/sqlancer/datafusion/DataFusionSchema.java +++ b/src/sqlancer/datafusion/DataFusionSchema.java @@ -13,7 +13,6 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.Node; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; @@ -123,7 +122,7 @@ public static DataFusionDataType parseFromDataFusionCatalog(String typeString) { } // TODO(datafusion) lots of hack here, should build our own Randomly later - public Node getRandomConstant(DataFusionGlobalState state) { + public DataFusionExpression getRandomConstant(DataFusionGlobalState state) { if (Randomly.getBooleanWithSmallProbability()) { return DataFusionConstant.createNullConstant(); } diff --git a/src/sqlancer/datafusion/DataFusionToStringVisitor.java b/src/sqlancer/datafusion/DataFusionToStringVisitor.java index 1f3030552..7d0d1b1b6 100644 --- a/src/sqlancer/datafusion/DataFusionToStringVisitor.java +++ b/src/sqlancer/datafusion/DataFusionToStringVisitor.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.datafusion.ast.DataFusionConstant; import sqlancer.datafusion.ast.DataFusionExpression; import sqlancer.datafusion.ast.DataFusionJoin; @@ -11,20 +10,20 @@ public class DataFusionToStringVisitor extends NewToStringVisitor { - public static String asString(Node expr) { + public static String asString(DataFusionExpression expr) { DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); visitor.visit(expr); return visitor.get(); } - public static String asString(List> exprs) { + public static String asString(List exprs) { DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); visitor.visit(exprs); return visitor.get(); } @Override - public void visitSpecific(Node expr) { + public void visitSpecific(DataFusionExpression expr) { if (expr instanceof DataFusionConstant) { visit((DataFusionConstant) expr); } else if (expr instanceof DataFusionSelect) { @@ -37,13 +36,13 @@ public void visitSpecific(Node expr) { } private void visit(DataFusionJoin join) { - visit(join.getLeftTable()); + visit((DataFusionExpression) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" "); sb.append(" JOIN "); - visit(join.getRightTable()); + visit((DataFusionExpression) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); diff --git a/src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java b/src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java new file mode 100644 index 000000000..e59676be8 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DataFusionBinaryOperation extends NewBinaryOperatorNode + implements DataFusionExpression { + public DataFusionBinaryOperation(DataFusionExpression left, DataFusionExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionColumnReference.java b/src/sqlancer/datafusion/ast/DataFusionColumnReference.java new file mode 100644 index 000000000..2391ef694 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.datafusion.DataFusionSchema; + +public class DataFusionColumnReference extends + ColumnReferenceNode implements DataFusionExpression { + public DataFusionColumnReference(DataFusionSchema.DataFusionColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionConstant.java b/src/sqlancer/datafusion/ast/DataFusionConstant.java index d123adb8a..90a997fcf 100644 --- a/src/sqlancer/datafusion/ast/DataFusionConstant.java +++ b/src/sqlancer/datafusion/ast/DataFusionConstant.java @@ -1,17 +1,15 @@ package sqlancer.datafusion.ast; -import sqlancer.common.ast.newast.Node; - -public class DataFusionConstant implements Node { +public class DataFusionConstant implements DataFusionExpression { private DataFusionConstant() { } - public static Node createIntConstant(long val) { + public static DataFusionExpression createIntConstant(long val) { return new DataFusionIntConstant(val); } - public static Node createNullConstant() { + public static DataFusionExpression createNullConstant() { return new DataFusionNullConstant(); } diff --git a/src/sqlancer/datafusion/ast/DataFusionFunction.java b/src/sqlancer/datafusion/ast/DataFusionFunction.java new file mode 100644 index 000000000..130fc04bf --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionFunction.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DataFusionFunction extends NewFunctionNode implements DataFusionExpression { + public DataFusionFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionJoin.java b/src/sqlancer/datafusion/ast/DataFusionJoin.java index 61cd9f556..430b2b812 100644 --- a/src/sqlancer/datafusion/ast/DataFusionJoin.java +++ b/src/sqlancer/datafusion/ast/DataFusionJoin.java @@ -5,8 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; @@ -16,32 +14,29 @@ /* NOT IMPLEMENTED YET */ -public class DataFusionJoin implements Node { +public class DataFusionJoin implements DataFusionExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final DataFusionTableReference leftTable; + private final DataFusionTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final DataFusionExpression onCondition; - public DataFusionJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public DataFusionJoin(DataFusionTableReference leftTable, DataFusionTableReference rightTable, JoinType joinType, + DataFusionExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public static List> getJoins(List tables, - DataFusionGlobalState globalState) { + public static List getJoins(List tables, DataFusionGlobalState globalState) { // [t1_join_t2, t1_join_t3, ...] - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)) + List tableList = tables.stream().map(t -> new DataFusionTableReference(t)) .collect(Collectors.toList()); - List> joinExpressions = new ArrayList<>(); + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + DataFusionTableReference leftTable = tableList.remove(0); + DataFusionTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); // TODO(datafusion) this `joinGen` can generate super chaotic exprsions, maybe we should make it more like a @@ -59,16 +54,16 @@ public static List> getJoins(List ta return joinExpressions; } - public static DataFusionJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DataFusionJoin createInnerJoin(DataFusionTableReference left, DataFusionTableReference right, + DataFusionExpression predicate) { return new DataFusionJoin(left, right, JoinType.INNER, predicate); } - public TableReferenceNode getLeftTable() { + public DataFusionTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public DataFusionTableReference getRightTable() { return rightTable; } @@ -76,7 +71,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public DataFusionExpression getOnCondition() { return onCondition; } diff --git a/src/sqlancer/datafusion/ast/DataFusionSelect.java b/src/sqlancer/datafusion/ast/DataFusionSelect.java index a80758c4a..6fda25144 100644 --- a/src/sqlancer/datafusion/ast/DataFusionSelect.java +++ b/src/sqlancer/datafusion/ast/DataFusionSelect.java @@ -6,16 +6,12 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema; -import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionTable; import sqlancer.datafusion.gen.DataFusionExpressionGenerator; -public class DataFusionSelect extends SelectBase> implements Node { +public class DataFusionSelect extends SelectBase implements DataFusionExpression { public Optional fetchColumnsString = Optional.empty(); // When available, override `fetchColumns` in base // class's `Node` representation (for display) public DataFusionExpressionGenerator exprGen; @@ -37,16 +33,14 @@ public static DataFusionSelect getRandomSelect(DataFusionGlobalState state) { // And generate a random expression which might contain those columns List randomColumns = DataFusionTable.getRandomColumns(randomTables); randomSelect.exprGen = new DataFusionExpressionGenerator(state).setColumns(randomColumns); - Node whereExpr = randomSelect.exprGen + DataFusionExpression whereExpr = randomSelect.exprGen .generateExpression(DataFusionSchema.DataFusionDataType.BOOLEAN); // Constructing result - List> randomTableNodes = randomTables.stream() - .map(t -> new TableReferenceNode(t)) - .collect(Collectors.toList()); - List> randomColumnNodes = randomColumns.stream() - .map((c) -> new ColumnReferenceNode(c)) + List randomTableNodes = randomTables.stream().map(t -> new DataFusionTableReference(t)) .collect(Collectors.toList()); + List randomColumnNodes = randomColumns.stream() + .map((c) -> new DataFusionColumnReference(c)).collect(Collectors.toList()); randomSelect.setFetchColumns(randomColumnNodes); randomSelect.setFromList(randomTableNodes); diff --git a/src/sqlancer/datafusion/ast/DataFusionTableReference.java b/src/sqlancer/datafusion/ast/DataFusionTableReference.java new file mode 100644 index 000000000..14445576a --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.datafusion.DataFusionSchema; + +public class DataFusionTableReference extends TableReferenceNode + implements DataFusionExpression { + public DataFusionTableReference(DataFusionSchema.DataFusionTable table) { + super(table); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java b/src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java new file mode 100644 index 000000000..ba5629460 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class DataFusionUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements DataFusionExpression { + public DataFusionUnaryPostfixOperation(DataFusionExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java b/src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java new file mode 100644 index 000000000..7109a12c2 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class DataFusionUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements DataFusionExpression { + public DataFusionUnaryPrefixOperation(DataFusionExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java index 3d7340615..88e486f47 100644 --- a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java @@ -12,22 +12,22 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.ast.DataFusionBinaryOperation; +import sqlancer.datafusion.ast.DataFusionColumnReference; import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionFunction; +import sqlancer.datafusion.ast.DataFusionUnaryPostfixOperation; +import sqlancer.datafusion.ast.DataFusionUnaryPrefixOperation; import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; public final class DataFusionExpressionGenerator - extends TypedExpressionGenerator, DataFusionColumn, DataFusionDataType> { + extends TypedExpressionGenerator { private final DataFusionGlobalState globalState; @@ -51,7 +51,7 @@ protected boolean canGenerateColumnOfType(DataFusionDataType type) { } @Override - protected Node generateExpression(DataFusionDataType type, int depth) { + protected DataFusionExpression generateExpression(DataFusionDataType type, int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { DataFusionDataType expectedType = type; if (Randomly.getBooleanWithRatherLowProbability()) { // ~10% @@ -85,8 +85,7 @@ protected Node generateExpression(DataFusionDataType type, argType = type; } - return new NewUnaryPrefixOperatorNode(generateExpression(argType, depth + 1), - randomExpr); + return new DataFusionUnaryPrefixOperation(generateExpression(argType, depth + 1), randomExpr); case UNARY_POSTFIX: dfAssert(randomExpr.argTypes.size() == 1 && randomExpr.nArgs == 1, "Unary expression should only have 1 argument" + randomExpr.argTypes); @@ -97,8 +96,7 @@ protected Node generateExpression(DataFusionDataType type, argType = type; } - return new NewUnaryPostfixOperatorNode(generateExpression(argType, depth + 1), - randomExpr); + return new DataFusionUnaryPostfixOperation(generateExpression(argType, depth + 1), randomExpr); case BINARY: dfAssert(randomExpr.argTypes.size() == 2 && randomExpr.nArgs == 2, "Binrary expression should only have 2 argument" + randomExpr.argTypes); @@ -119,7 +117,7 @@ protected Node generateExpression(DataFusionDataType type, } } - return new NewBinaryOperatorNode(generateExpression(argTypeList.get(0), depth + 1), + return new DataFusionBinaryOperation(generateExpression(argTypeList.get(0), depth + 1), generateExpression(argTypeList.get(1), depth + 1), randomExpr); case AGGREGATE: // Fall through @@ -133,13 +131,13 @@ protected Node generateExpression(DataFusionDataType type, return null; } - public Node generateFunctionExpression(DataFusionDataType type, int depth, + public DataFusionExpression generateFunctionExpression(DataFusionDataType type, int depth, DataFusionBaseExpr exprType) { if (exprType.isVariadic || Randomly.getBooleanWithSmallProbability()) { // TODO(datafusion) maybe add possible types. e.g. some function have signature variadic(INT/DOUBLE), then // only randomly pick from INT and DOUBLE int nArgs = Randomly.smallNumber(); // 0, 2, 4, ... smaller one is more likely - return new NewFunctionNode(generateExpressions(nArgs), exprType); + return new DataFusionFunction(generateExpressions(nArgs), exprType); } List funcArgTypeList = new ArrayList<>(); // types of current expression's input arguments @@ -161,13 +159,13 @@ public Node generateFunctionExpression(DataFusionDataType i++; } - List> argExpressions = new ArrayList<>(); + List argExpressions = new ArrayList<>(); for (DataFusionDataType dataType : funcArgTypeList) { argExpressions.add(generateExpression(dataType, depth + 1)); } - return new NewFunctionNode(argExpressions, exprType); + return new DataFusionFunction(argExpressions, exprType); } List filterColumns(DataFusionDataType type) { @@ -179,7 +177,7 @@ List filterColumns(DataFusionDataType type) { } @Override - protected Node generateColumn(DataFusionDataType type) { + protected DataFusionExpression generateColumn(DataFusionDataType type) { // HACK: if no col of such type exist, generate constant value instead List colsOfType = filterColumns(type); if (colsOfType.isEmpty()) { @@ -187,32 +185,32 @@ protected Node generateColumn(DataFusionDataType type) { } DataFusionColumn column = Randomly.fromList(colsOfType); - return new ColumnReferenceNode(column); + return new DataFusionColumnReference(column); } @Override - public Node generateConstant(DataFusionDataType type) { + public DataFusionExpression generateConstant(DataFusionDataType type) { return type.getRandomConstant(globalState); } @Override - public Node generatePredicate() { + public DataFusionExpression generatePredicate() { return generateExpression(DataFusionDataType.BOOLEAN, 0); } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, createExpr(DataFusionBaseExprType.NOT)); + public DataFusionExpression negatePredicate(DataFusionExpression predicate) { + return new DataFusionUnaryPrefixOperation(predicate, createExpr(DataFusionBaseExprType.NOT)); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, createExpr(DataFusionBaseExprType.IS_NULL)); + public DataFusionExpression isNull(DataFusionExpression expr) { + return new DataFusionUnaryPostfixOperation(expr, createExpr(DataFusionBaseExprType.IS_NULL)); } public static class DataFusionCastOperation extends NewUnaryPostfixOperatorNode { - public DataFusionCastOperation(Node expr, DataFusionDataType type) { + public DataFusionCastOperation(DataFusionExpression expr, DataFusionDataType type) { super(expr, new Operator() { @Override diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java index b304a4131..067b07677 100644 --- a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java +++ b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java @@ -2,7 +2,6 @@ import java.sql.SQLException; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -12,7 +11,7 @@ import sqlancer.datafusion.gen.DataFusionExpressionGenerator; public class DataFusionQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, DataFusionGlobalState> + extends TernaryLogicPartitioningOracleBase implements TestOracle { DataFusionGlobalState state; DataFusionExpressionGenerator gen; @@ -31,7 +30,7 @@ public void check() throws SQLException { } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } From 5a8300438f732843059c73deb101f4873d6282e4 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 14:34:26 +0800 Subject: [PATCH 0907/1284] Remove Node interface --- src/sqlancer/common/ast/newast/Node.java | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 src/sqlancer/common/ast/newast/Node.java diff --git a/src/sqlancer/common/ast/newast/Node.java b/src/sqlancer/common/ast/newast/Node.java deleted file mode 100644 index 310e875c0..000000000 --- a/src/sqlancer/common/ast/newast/Node.java +++ /dev/null @@ -1,5 +0,0 @@ -package sqlancer.common.ast.newast; - -public interface Node { - -} From f36e1466414f94a0906ea79d8a85fbf93f68a720 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 14:38:58 +0800 Subject: [PATCH 0908/1284] Fix linter errors --- src/sqlancer/databend/DatabendExpectedValueVisitor.java | 2 +- .../oracle/tlp/DorisQueryPartitioningAggregateTester.java | 4 ++-- src/sqlancer/mariadb/ast/MariaDBJoin.java | 1 - src/sqlancer/postgres/ast/PostgresJoin.java | 1 - src/sqlancer/sqlite3/ast/SQLite3Expression.java | 1 - 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/databend/DatabendExpectedValueVisitor.java b/src/sqlancer/databend/DatabendExpectedValueVisitor.java index e10ade97a..bfa6208a3 100644 --- a/src/sqlancer/databend/DatabendExpectedValueVisitor.java +++ b/src/sqlancer/databend/DatabendExpectedValueVisitor.java @@ -25,7 +25,7 @@ public class DatabendExpectedValueVisitor { private void print(DatabendExpression expr) { sb.append(DatabendToStringVisitor.asString(expr)); sb.append(" -- "); - sb.append((expr).getExpectedValue()); + sb.append(expr.getExpectedValue()); sb.append("\n"); } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java index b3289e5ad..afc1dde43 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -16,8 +16,8 @@ import sqlancer.doris.DorisSchema.DorisCompositeDataType; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.ast.DorisAggregateOperation; -import sqlancer.doris.ast.DorisAlias; import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; +import sqlancer.doris.ast.DorisAlias; import sqlancer.doris.ast.DorisBinaryArithmeticOperation; import sqlancer.doris.ast.DorisBinaryOperation; import sqlancer.doris.ast.DorisCastOperation; @@ -26,8 +26,8 @@ import sqlancer.doris.ast.DorisFunction; import sqlancer.doris.ast.DorisSelect; import sqlancer.doris.ast.DorisUnaryPostfixOperation; -import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; import sqlancer.doris.visitor.DorisToStringVisitor; diff --git a/src/sqlancer/mariadb/ast/MariaDBJoin.java b/src/sqlancer/mariadb/ast/MariaDBJoin.java index d846f4605..f1d0892e1 100644 --- a/src/sqlancer/mariadb/ast/MariaDBJoin.java +++ b/src/sqlancer/mariadb/ast/MariaDBJoin.java @@ -36,7 +36,6 @@ public MariaDBTable getTable() { return table; } - @Override public MariaDBExpression getOnClause() { return onClause; } diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index 7bfef3f6f..ef6dfab1f 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -100,7 +100,6 @@ public PostgresExpression getRightTable() { return rightTable; } - @Override public PostgresExpression getOnClause() { return onClause; } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index d84f399a4..7b131b6b5 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -165,7 +165,6 @@ public SQLite3Table getTable() { return table; } - @Override public SQLite3Expression getOnClause() { return onClause; } From df613543ec194e74cc7e89023869e543bcdacb86 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:59:11 +0800 Subject: [PATCH 0909/1284] Update readme with removed DBMSs (#964) * Update readme with removed DBMSs * Add reference PR for removed DBMBs --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index bb80d0794..b0fe4769b 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,16 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | | DataFusion | Preliminary | Typed | Only basic SQL features are supported. | +## Previously Supported DBMS + +Some DBMS were once supported but subsequently removed. + +| DBMS | Pull Request | Description | +| ---------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ArangoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because ArangoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. | # Using SQLancer From b55c7743f7a6e7e55fcedd71ee9d4031625ce5da Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 28 Aug 2024 17:00:01 +0800 Subject: [PATCH 0910/1284] [DuckDB] Implement NoREC expression generator interface --- src/sqlancer/duckdb/ast/DuckDBJoin.java | 6 +- src/sqlancer/duckdb/ast/DuckDBSelect.java | 26 ++++++- .../duckdb/gen/DuckDBExpressionGenerator.java | 76 ++++++++++++++++++- .../gen/DuckDBRandomQuerySynthesizer.java | 2 +- .../test/DuckDBQueryPartitioningBase.java | 2 +- 5 files changed, 105 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/duckdb/ast/DuckDBJoin.java b/src/sqlancer/duckdb/ast/DuckDBJoin.java index 82031d1e7..f677e9efc 100644 --- a/src/sqlancer/duckdb/ast/DuckDBJoin.java +++ b/src/sqlancer/duckdb/ast/DuckDBJoin.java @@ -66,8 +66,8 @@ public OuterType getOuterType() { return outerType; } - public static List getJoins(List tableList, DuckDBGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DuckDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { DuckDBTableReference leftTable = tableList.remove(0); DuckDBTableReference rightTable = tableList.remove(0); @@ -111,7 +111,7 @@ public static DuckDBJoin createInnerJoin(DuckDBTableReference left, DuckDBTableR return new DuckDBJoin(left, right, JoinType.INNER, predicate); } - public static DuckDBExpression createNaturalJoin(DuckDBTableReference left, DuckDBTableReference right, + public static DuckDBJoin createNaturalJoin(DuckDBTableReference left, DuckDBTableReference right, OuterType naturalJoinType) { DuckDBJoin join = new DuckDBJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); diff --git a/src/sqlancer/duckdb/ast/DuckDBSelect.java b/src/sqlancer/duckdb/ast/DuckDBSelect.java index 10d76fbbd..e18e57a4d 100644 --- a/src/sqlancer/duckdb/ast/DuckDBSelect.java +++ b/src/sqlancer/duckdb/ast/DuckDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.duckdb.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; +import sqlancer.duckdb.DuckDBSchema.DuckDBTable; +import sqlancer.duckdb.DuckDBToStringVisitor; -public class DuckDBSelect extends SelectBase implements DuckDBExpression { +public class DuckDBSelect extends SelectBase + implements Select, DuckDBExpression { private boolean isDistinct; @@ -14,4 +22,20 @@ public boolean isDistinct() { return isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DuckDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DuckDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DuckDBToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 49a390c07..000aa95d3 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -3,17 +3,22 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; +import sqlancer.duckdb.DuckDBSchema.DuckDBTable; +import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBBetweenOperator; import sqlancer.duckdb.ast.DuckDBBinaryOperator; import sqlancer.duckdb.ast.DuckDBCaseOperator; @@ -22,12 +27,18 @@ import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBFunction; import sqlancer.duckdb.ast.DuckDBInOperator; +import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBOrderingTerm; +import sqlancer.duckdb.ast.DuckDBPostFixText; +import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; import sqlancer.duckdb.ast.DuckDBTernary; -public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator { +public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator + implements NoRECGenerator { private final DuckDBGlobalState globalState; + private List tables; public DuckDBExpressionGenerator(DuckDBGlobalState globalState) { this.globalState = globalState; @@ -437,4 +448,67 @@ public DuckDBExpression isNull(DuckDBExpression expr) { return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(expr, DuckDBUnaryPostfixOperator.IS_NULL); } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public DuckDBExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public DuckDBSelect generateSelect() { + return new DuckDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + return DuckDBJoin.getJoins(tableList, globalState); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DuckDBTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DuckDBSelect select, DuckDBExpression whereCondition, + boolean shouldUseAggregate) { + List allColumns = columns.stream().map((c) -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); + if (shouldUseAggregate) { + DuckDBFunction aggr = new DuckDBFunction<>( + Arrays.asList(new DuckDBColumnReference( + new DuckDBColumn("*", new DuckDBCompositeDataType(DuckDBDataType.INT, 0), false, false))), + DuckDBAggregateFunction.COUNT); + select.setFetchColumns(Arrays.asList(aggr)); + } else { + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DuckDBSelect select, DuckDBExpression whereCondition) { + DuckDBExpression asText = new DuckDBPostFixText(new DuckDBCastOperation( + new DuckDBPostFixText(whereCondition, + " IS NOT NULL AND " + DuckDBToStringVisitor.asString(whereCondition)), + new DuckDBCompositeDataType(DuckDBDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } diff --git a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java index c4c0d7c95..d88d4f0b8 100644 --- a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java +++ b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java @@ -40,7 +40,7 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) .collect(Collectors.toList()); - List joins = DuckDBJoin.getJoins(tableList, globalState); + List joins = DuckDBJoin.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index b4858df4c..1e75be7b3 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -46,7 +46,7 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) .collect(Collectors.toList()); - List joins = DuckDBJoin.getJoins(tableList, state); + List joins = DuckDBJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); From 9fc9a57aed4bc110771ba31d348ce6505e168d85 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 28 Aug 2024 17:00:29 +0800 Subject: [PATCH 0911/1284] [DuckDB] Use common NoREC oracle --- .../duckdb/gen/DuckDBExpressionGenerator.java | 4 +- .../duckdb/test/DuckDBNoRECOracle.java | 123 +++--------------- 2 files changed, 19 insertions(+), 108 deletions(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 000aa95d3..3f6abc37f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -471,7 +471,9 @@ public DuckDBSelect generateSelect() { public List getRandomJoinClauses() { List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) .collect(Collectors.toList()); - return DuckDBJoin.getJoins(tableList, globalState); + List joins = DuckDBJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 63932492d..4c57c0920 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -1,135 +1,44 @@ package sqlancer.duckdb.test; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; -import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; -import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.DuckDBSchema.DuckDBTables; -import sqlancer.duckdb.DuckDBToStringVisitor; -import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; -import sqlancer.duckdb.ast.DuckDBPostFixText; import sqlancer.duckdb.ast.DuckDBSelect; -import sqlancer.duckdb.ast.DuckDBTableReference; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBCastOperation; -public class DuckDBNoRECOracle extends NoRECBase implements TestOracle { +public class DuckDBNoRECOracle implements TestOracle { - private final DuckDBSchema s; + NoRECOracle oracle; public DuckDBNoRECOracle(DuckDBGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - DuckDBErrors.addExpressionErrors(errors); + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - DuckDBTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(state).setColumns(columns); - DuckDBExpression randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) - .collect(Collectors.toList()); - List joins = DuckDBJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); - int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, - randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getSecondQuery(List tableList, DuckDBExpression randomWhereCondition, - List joins) throws SQLException { - DuckDBSelect select = new DuckDBSelect(); - // select.setGroupByClause(groupBys); - // DuckDBExpression isTrue = DuckDBPostfixOperation.create(randomWhereCondition, - // PostfixOperator.IS_TRUE); - DuckDBExpression asText = new DuckDBPostFixText(new DuckDBCastOperation( - new DuckDBPostFixText(randomWhereCondition, - " IS NOT NULL AND " + DuckDBToStringVisitor.asString(randomWhereCondition)), - new DuckDBCompositeDataType(DuckDBDataType.INT, 8)), "as count"); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(tableList); - // select.setSelectType(SelectType.ALL); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + DuckDBToStringVisitor.asString(select) + ") as res"; - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getFirstQueryCount(SQLConnection con, List tableList, List columns, - DuckDBExpression randomWhereCondition, List joins) throws SQLException { - DuckDBSelect select = new DuckDBSelect(); - // select.setGroupByClause(groupBys); - // DuckDBAggregate aggr = new DuckDBAggregate( - List allColumns = columns.stream().map((c) -> new DuckDBColumnReference(c)) - .collect(Collectors.toList()); - // DuckDBAggregateFunction.COUNT); - // select.setFetchColumns(Arrays.asList(aggr)); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new DuckDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - // select.setSelectType(SelectType.ALL); - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = DuckDBToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From 68835ed60bb09450a52533cfbf8b69ec88733720 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:11:19 +0800 Subject: [PATCH 0912/1284] Update the DuckDB version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9bee83485..129908302 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 0.4.0 + 1.0.0 com.facebook.presto From 0b228603e3e82f20a8d9c56fc83b101faf9d43cb Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:12:21 +0800 Subject: [PATCH 0913/1284] Disable partial indexes Consider the following SQL statements: CREATE TABLE t0(c0 INT); CREATE INDEX i0 on t0(c0) WHERE c0 > 0; This will result in an error "Not implemented Error: Creating partial indexes is not supported currently (exit status 1)". --- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index daa22da5f..312fe5828 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -8,8 +8,6 @@ import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.DuckDBToStringVisitor; -import sqlancer.duckdb.ast.DuckDBExpression; public final class DuckDBIndexGenerator { @@ -42,12 +40,6 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } } sb.append(")"); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - DuckDBExpression expr = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()) - .generateExpression(); - sb.append(DuckDBToStringVisitor.asString(expr)); - } errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("Cannot create an index on the rowid!"); From d421418ece6d4660f336de1d557a7cd72d073103 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:18:08 +0800 Subject: [PATCH 0914/1284] Change the expected error message for duplicate key constraint violations This affects statements such as the following: CREATE TABLE t0(c0 INT UNIQUE); INSERT INTO t0(c0) VALUES (0); INSERT INTO t0(c0) VALUES (0); Error: Constraint Error: Duplicate key "c0: 0" violates unique constraint. If this is an unexpected constraint violation please double check with the known index limitations section in our documentation (https://duckdb.org/docs/sql/indexes). (exit status 1) --- src/sqlancer/duckdb/DuckDBErrors.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 5c2909f4d..1cadb3a2a 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -114,7 +114,7 @@ public static List getInsertErrors() { errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); - errors.add("duplicate key"); + errors.add("Duplicate key"); errors.add("can't be cast because the value is out of range for the destination type"); errors.add("Could not convert string"); errors.add("Unimplemented type for cast"); From 3eda3423e14522b6fee942cbadc9b019d3105bb3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:32:32 +0800 Subject: [PATCH 0915/1284] Do not insert into or update the rowid See the example errors below CREATE TABLE t0(c0 INT); INSERT INTO t0 VALUES (0), (1); SELECT rowid FROM t0; -- 0, 1 INSERT INTO t0(rowid) VALUES (0); -- Binder Error: Table "t0" does not have a column with name "rowid" (exit status 1) UPDATE t0 SET rowid=5; -- Binder Error: Referenced update column rowid not found in table! (exit status 1) --- src/sqlancer/common/schema/AbstractTable.java | 4 ++++ src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java | 2 +- src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 4a57fa846..89d568867 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -83,6 +83,10 @@ public List getRandomNonEmptyColumnSubset() { return Randomly.nonEmptySubset(getColumns()); } + public List getRandomNonEmptyColumnSubsetFilter(Predicate predicate) { + return Randomly.nonEmptySubset(getColumns().stream().filter(predicate).collect(Collectors.toList())); + } + public List getRandomNonEmptyColumnSubset(int size) { return Randomly.nonEmptySubset(getColumns(), size); } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 4fd3f390c..6793d2b51 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -29,7 +29,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { private SQLQueryAdapter generate() { sb.append("INSERT INTO "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyColumnSubset(); + List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); sb.append(table.getName()); sb.append("("); sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 3bb22b2ea..b4ffd0140 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -27,7 +27,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { private SQLQueryAdapter generate() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyColumnSubset(); + List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append("UPDATE "); sb.append(table.getName()); From 7701068472adf647236c00f1b2dd1aeed98d54b7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:38:39 +0800 Subject: [PATCH 0916/1284] Add an expected error for mismatched expression types in the CASE operator SELECT 1 BETWEEN 1.0 AND true; -- Binder Error: Cannot mix values of type DECIMAL(11,1) and BOOLEAN in BETWEEN clause - an explicit cast is required --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 1cadb3a2a..58b5eb110 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -63,6 +63,7 @@ public static List getExpressionErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); + errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); } private static List getRegexErrors() { From 7db4a6527576f8119ec493d4901bd9db60a02013 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:45:48 +0800 Subject: [PATCH 0917/1284] Add an expected error for mismatched expressions in CASE expressions SELECT CASE WHEN false THEN 5.0 ELSE true END; -- Binder Error: Cannot mix values of type BOOLEAN and DECIMAL(2,1) in CASE expression - an explicit cast is requiredSELECT CASE WHEN false THEN 5.0 ELSE true END; -- Binder Error: Cannot mix values of type BOOLEAN and DECIMAL(2,1) in CASE expression - an explicit cast is required --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 58b5eb110..4adce2b7b 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -64,6 +64,7 @@ public static List getExpressionErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); + errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in CASE expression"); } private static List getRegexErrors() { From 2f78758f22f8b04527a5b2272756ad07f95edbb5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:51:19 +0800 Subject: [PATCH 0918/1284] Add an expected error for mismatch expression types in comparisons SELECT 1 >= ('a' || 'b'); -- Binder Error: Cannot compare values of type BIGINT and type VARCHAR - an explicit cast is required SELECT 1 >= 'a'; -- Conversion Error: Could not convert string 'a' to INT32 --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 4adce2b7b..5510d49e4 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -65,6 +65,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in CASE expression"); + errors.addRegexString("Cannot compare values of type .* and type .*"); } private static List getRegexErrors() { From 3d87be0fe5e7d66c63ad1674176ae48d6fcde09d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:56:22 +0800 Subject: [PATCH 0919/1284] Change the expected error message when creating unique indexes on duplicate data CREATE TABLE t0(c0 INT); INSERT INTO t0 VALUES (0); INSERT INTO t0 VALUES (0); CREATE UNIQUE INDEX i0 on t0(c0); -- Constraint Error: Data contains duplicates on indexed column(s) (exit status 1) --- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 312fe5828..80bbfd61c 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -19,7 +19,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("CREATE "); if (Randomly.getBoolean()) { - errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + errors.add("Data contains duplicates on indexed column(s)"); sb.append("UNIQUE "); } sb.append("INDEX "); From 3e513f2203db7be28bec7769541a2ba0ec57622d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 23:18:53 +0800 Subject: [PATCH 0920/1284] Avoid assigning a rowid column to views CREATE VIEW v0 AS SELECT 1; SELECT rowid FROM v0; -- Binder Error: Referenced column "rowid" not found in FROM clause! --- src/sqlancer/common/schema/AbstractSchema.java | 4 ++++ src/sqlancer/duckdb/DuckDBErrors.java | 3 --- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java index 1c66363e7..053c9540a 100644 --- a/src/sqlancer/common/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -126,6 +126,10 @@ public String getFreeTableName() { } + public static boolean matchesViewName(String relationName) { + return relationName.startsWith("v"); + } + public String getFreeViewName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 5510d49e4..03f9c4b5a 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -46,9 +46,6 @@ public static List getExpressionErrors() { errors.add("Like pattern must not end with escape character!"); // LIKE - errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a - // table supports rowids - errors.add("does not have a column named"); // TODO: this only happens for views whose underlying table has a // removed column errors.add("Contents of view were altered: types don't match!"); diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 6b94ce137..8e48bf89f 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -269,7 +269,7 @@ private static List getTableColumns(SQLConnection con, String tabl } } } - if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { + if (columns.stream().noneMatch(c -> c.isPrimaryKey()) && !AbstractSchema.matchesViewName(tableName)) { // https://github.com/cwida/duckdb/issues/589 // https://github.com/cwida/duckdb/issues/588 // TODO: implement an option to enable/disable rowids From 993dd64c7676d1bdc28356fdfffd87ce178294ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 23:22:20 +0800 Subject: [PATCH 0921/1284] Add an expected error for mismatched expression types in the coalesce operator SELECT COALESCE(null, 'a', 5); -- Binder Error: Cannot mix values of type VARCHAR and INTEGER_LITERAL in COALESCE operator - an explicit cast is required (exit status 1) --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 03f9c4b5a..d3cc6f310 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -62,6 +62,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in CASE expression"); + errors.addRegexString("Cannot mix values of type .* and .* in COALESCE operator"); errors.addRegexString("Cannot compare values of type .* and type .*"); } From a2f3f0a8d49c4ccaa3fe1938aa5bced9d7078d22 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 23:31:30 +0800 Subject: [PATCH 0922/1284] Update expected regular expression errors SELECT 'a' SIMILAR TO 'a)'; -- Invalid Input Error: unexpected ): a) SELECT 'a' SIMILAR TO '[a'; -- Invalid Input Error: missing ]: [a --- src/sqlancer/duckdb/DuckDBErrors.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index d3cc6f310..29fc2f0fe 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -69,15 +69,8 @@ public static void addExpressionErrors(ExpectedErrors errors) { private static List getRegexErrors() { ArrayList errors = new ArrayList<>(); - errors.add("missing ]"); - errors.add("missing )"); - errors.add("invalid escape sequence"); - errors.add("no argument for repetition operator: "); - errors.add("bad repetition operator"); - errors.add("trailing \\"); - errors.add("invalid perl operator"); - errors.add("invalid character class range"); - errors.add("width is not integer"); + errors.add("Invalid Input Error:"); + errors.add("Invalid Input Error:"); return errors; } From fc156a0cce46180732b527181c758eba4cf6c40b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 23:32:22 +0800 Subject: [PATCH 0923/1284] Add another version for the GROUP BY clause error --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 29fc2f0fe..d7e0aadca 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -131,6 +131,7 @@ public static void addInsertErrors(ExpectedErrors errors) { public static List getGroupByErrors() { ArrayList errors = new ArrayList<>(); + errors.add("must appear in the GROUP BY clause or must be part of an aggregate function"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY term out of range"); From fdf853ff7cc48e0cf56a41ed0550f6ee54ea223c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 1 Sep 2024 00:05:06 +0800 Subject: [PATCH 0924/1284] Update the name of the NULL data type string for views --- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 8e48bf89f..379576fd8 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -203,7 +203,7 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { case "TIMESTAMP": primitiveType = DuckDBDataType.TIMESTAMP; break; - case "NULL": + case "\"NULL\"": primitiveType = DuckDBDataType.NULL; break; case "INTERVAL": From 8a935f18da5511f60cd381d6d591b19481a247b6 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 15:19:10 +0800 Subject: [PATCH 0925/1284] [TimescaleDB] Remove implementation --- src/sqlancer/Main.java | 2 - src/sqlancer/timescaledb/TimescaleDBBugs.java | 7 --- .../timescaledb/TimescaleDBGlobalState.java | 12 ----- .../timescaledb/TimescaleDBOptions.java | 27 ---------- .../timescaledb/TimescaleDBProvider.java | 30 ----------- .../timescaledb/TimescaleDBSchema.java | 52 ------------------- .../timescaledb/gen/TimescaleDBCommon.java | 20 ------- 7 files changed, 150 deletions(-) delete mode 100644 src/sqlancer/timescaledb/TimescaleDBBugs.java delete mode 100644 src/sqlancer/timescaledb/TimescaleDBGlobalState.java delete mode 100644 src/sqlancer/timescaledb/TimescaleDBOptions.java delete mode 100644 src/sqlancer/timescaledb/TimescaleDBProvider.java delete mode 100644 src/sqlancer/timescaledb/TimescaleDBSchema.java delete mode 100644 src/sqlancer/timescaledb/gen/TimescaleDBCommon.java diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 5ffc3e94c..46841095b 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -44,7 +44,6 @@ import sqlancer.questdb.QuestDBProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; -import sqlancer.timescaledb.TimescaleDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; import sqlancer.yugabyte.ysql.YSQLProvider; @@ -746,7 +745,6 @@ private static void checkForIssue799(List> providers) providers.add(new QuestDBProvider()); providers.add(new SQLite3Provider()); providers.add(new TiDBProvider()); - providers.add(new TimescaleDBProvider()); providers.add(new YCQLProvider()); providers.add(new YSQLProvider()); } diff --git a/src/sqlancer/timescaledb/TimescaleDBBugs.java b/src/sqlancer/timescaledb/TimescaleDBBugs.java deleted file mode 100644 index 079dfee74..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBBugs.java +++ /dev/null @@ -1,7 +0,0 @@ -package sqlancer.timescaledb; - -public final class TimescaleDBBugs { - private TimescaleDBBugs() { - - } -} diff --git a/src/sqlancer/timescaledb/TimescaleDBGlobalState.java b/src/sqlancer/timescaledb/TimescaleDBGlobalState.java deleted file mode 100644 index 519001b90..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBGlobalState.java +++ /dev/null @@ -1,12 +0,0 @@ -package sqlancer.timescaledb; - -import java.sql.SQLException; - -import sqlancer.postgres.PostgresGlobalState; - -public class TimescaleDBGlobalState extends PostgresGlobalState { - @Override - public TimescaleDBSchema readSchema() throws SQLException { - return TimescaleDBSchema.fromConnection(getConnection(), getDatabaseName()); - } -} diff --git a/src/sqlancer/timescaledb/TimescaleDBOptions.java b/src/sqlancer/timescaledb/TimescaleDBOptions.java deleted file mode 100644 index 03b479662..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBOptions.java +++ /dev/null @@ -1,27 +0,0 @@ -package sqlancer.timescaledb; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; - -import com.beust.jcommander.Parameter; - -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresOptions; -import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; - -public class TimescaleDBOptions extends PostgresOptions { - @Parameter(names = "--timescaledboracle", description = "Specifies which test oracle should be used for TimeScaleDB extension to PostgreSQL") - public List timescaleDBOracle = Arrays.asList(TimescaleDBOracleFactory.PQS); - - public enum TimescaleDBOracleFactory implements OracleFactory { - PQS { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresPivotedQuerySynthesisOracle(globalState); - } - }, - } -} diff --git a/src/sqlancer/timescaledb/TimescaleDBProvider.java b/src/sqlancer/timescaledb/TimescaleDBProvider.java deleted file mode 100644 index 756669128..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBProvider.java +++ /dev/null @@ -1,30 +0,0 @@ -package sqlancer.timescaledb; - -import com.google.auto.service.AutoService; - -import sqlancer.DatabaseProvider; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresOptions; -import sqlancer.postgres.PostgresProvider; -import sqlancer.timescaledb.gen.TimescaleDBCommon; - -@AutoService(DatabaseProvider.class) -public class TimescaleDBProvider extends PostgresProvider { - @SuppressWarnings("unchecked") - public TimescaleDBProvider() { - super((Class) (Object) TimescaleDBGlobalState.class, - (Class) (Object) TimescaleDBOptions.class); - } - - @Override - public String getDBMSName() { - return "timescaledb"; - } - - public static ExpectedErrors getTimescaleDBErrors() { - ExpectedErrors errors = new ExpectedErrors(); - TimescaleDBCommon.addTimescaleDBErrors(errors); - return errors; - } -} diff --git a/src/sqlancer/timescaledb/TimescaleDBSchema.java b/src/sqlancer/timescaledb/TimescaleDBSchema.java deleted file mode 100644 index 0c09d579c..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBSchema.java +++ /dev/null @@ -1,52 +0,0 @@ -package sqlancer.timescaledb; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.SQLIntegrityConstraintViolationException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.SQLConnection; -import sqlancer.postgres.PostgresSchema; - -public class TimescaleDBSchema extends PostgresSchema { - - public TimescaleDBSchema(List databaseTables, String databaseName) { - super(new ArrayList<>(databaseTables), databaseName); - } - - public static class TimescaleDBTable extends PostgresTable { - public TimescaleDBTable(String tableName, List columns, List indexes, - TableType tableType, List statistics, boolean isView, boolean isInsertable) { - super(tableName, columns, indexes, tableType, statistics, isView, isInsertable); - } - - public TimescaleDBTable(PostgresTable table) { - super(table.getName(), table.getColumns(), table.getIndexes(), table.getTableType(), table.getStatistics(), - table.isView(), table.isInsertable()); - } - } - - public static TimescaleDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { - PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); - List databaseTables = new ArrayList<>(); - try (Statement s = con.createStatement(); - ResultSet rs = s.executeQuery("SELECT table_name FROM information_schema.tables")) { - while (rs.next()) { - String tableName = rs.getString("table_name"); - - PostgresTable t = schema.getDatabaseTable(tableName); - if (t == null) { - continue; - } - - TimescaleDBTable table = new TimescaleDBTable(t); - databaseTables.add(table); - } - } catch (SQLIntegrityConstraintViolationException e) { - throw new AssertionError(e); - } - return new TimescaleDBSchema(databaseTables, databaseName); - } -} diff --git a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java deleted file mode 100644 index 69c4148de..000000000 --- a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.timescaledb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.common.query.ExpectedErrors; - -public final class TimescaleDBCommon { - private TimescaleDBCommon() { - - } - - public static List getTimescaleDBErrors() { - return new ArrayList<>(); - } - - public static void addTimescaleDBErrors(ExpectedErrors errors) { - errors.addAll(getTimescaleDBErrors()); - } -} From 8a1cad34158d3dd42f81c570ea8ad2a64f44c418 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 2 Sep 2024 15:33:53 +0800 Subject: [PATCH 0926/1284] Remove duplicate line --- src/sqlancer/duckdb/DuckDBErrors.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index d7e0aadca..1cc7bd96b 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -70,8 +70,6 @@ private static List getRegexErrors() { ArrayList errors = new ArrayList<>(); errors.add("Invalid Input Error:"); - errors.add("Invalid Input Error:"); - return errors; } From 98d1284b90933180cee461750b1028e039dceeaa Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 16:25:06 +0800 Subject: [PATCH 0927/1284] [CockroachDB] Implement NoREC expression generator interface --- .../ast/CockroachDBExpression.java | 5 +- .../cockroachdb/ast/CockroachDBJoin.java | 9 +- .../cockroachdb/ast/CockroachDBSelect.java | 27 +++++- .../gen/CockroachDBExpressionGenerator.java | 95 ++++++++++++++++++- .../oracle/CockroachDBCERTOracle.java | 2 +- 5 files changed, 130 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java b/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java index 555fb97fd..d0ac07310 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java @@ -1,5 +1,8 @@ package sqlancer.cockroachdb.ast; -public interface CockroachDBExpression { +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.common.ast.newast.Expression; + +public interface CockroachDBExpression extends Expression { } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java index b53c4f681..a8cba22f9 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java @@ -3,8 +3,12 @@ import java.util.Arrays; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.common.ast.newast.Join; -public class CockroachDBJoin implements CockroachDBExpression { +public class CockroachDBJoin + implements CockroachDBExpression, Join { private final CockroachDBExpression leftTable; private final CockroachDBExpression rightTable; @@ -49,7 +53,8 @@ public JoinType getJoinType() { return joinType; } - public void setOnCondition(CockroachDBExpression onCondition) { + @Override + public void setOnClause(CockroachDBExpression onCondition) { this.onCondition = onCondition; } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java index 2c7d33ff4..c043a9207 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.cockroachdb.ast; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; -public class CockroachDBSelect extends SelectBase implements CockroachDBExpression { +public class CockroachDBSelect extends SelectBase implements CockroachDBExpression, + Select { private boolean isDistinct; @@ -14,4 +22,21 @@ public void setDistinct(boolean isDistinct) { this.isDistinct = isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (CockroachDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (CockroachDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return CockroachDBVisitor.asString(this); + } + } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index 6326bcc95..d36e7673c 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -10,6 +10,8 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBCompositeDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.cockroachdb.ast.CockroachDBAggregate; import sqlancer.cockroachdb.ast.CockroachDBAggregate.CockroachDBAggregateFunction; import sqlancer.cockroachdb.ast.CockroachDBBetweenOperation; @@ -29,6 +31,7 @@ import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBFunction; import sqlancer.cockroachdb.ast.CockroachDBInOperation; +import sqlancer.cockroachdb.ast.CockroachDBJoin; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonOperator; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonType; @@ -36,14 +39,20 @@ import sqlancer.cockroachdb.ast.CockroachDBOrderingTerm; import sqlancer.cockroachdb.ast.CockroachDBRegexOperation; import sqlancer.cockroachdb.ast.CockroachDBRegexOperation.CockroachDBRegexOperator; +import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.ast.CockroachDBTypeAnnotation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; -public class CockroachDBExpressionGenerator - extends TypedExpressionGenerator { +public class CockroachDBExpressionGenerator extends + TypedExpressionGenerator implements + NoRECGenerator { + private List tables; private final CockroachDBGlobalState globalState; public CockroachDBExpressionGenerator(CockroachDBGlobalState globalState) { @@ -84,7 +93,8 @@ public List getOrderingTerms() { @Override public CockroachDBExpression generateExpression(CockroachDBCompositeDataType type, int depth) { - // if (type == CockroachDBDataType.FLOAT && Randomly.getBooleanWithRatherLowProbability()) { + // if (type == CockroachDBDataType.FLOAT && + // Randomly.getBooleanWithRatherLowProbability()) { // type = CockroachDBDataType.INT; // } if (allowAggregates && Randomly.getBoolean()) { @@ -358,4 +368,83 @@ public CockroachDBExpression isNull(CockroachDBExpression expr) { return new CockroachDBUnaryPostfixOperation(expr, CockroachDBUnaryPostfixOperator.IS_NULL); } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public CockroachDBExpression generateBooleanExpression() { + return generateExpression(CockroachDBDataType.BOOL.get()); + } + + @Override + public CockroachDBSelect generateSelect() { + return new CockroachDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinExpressions = new ArrayList<>(); + List tableReferences = tables.stream().map(t -> new CockroachDBTableReference(t)) + .collect(Collectors.toList()); + while (tableReferences.size() >= 2 && Randomly.getBoolean()) { + CockroachDBTableReference leftTable = tableReferences.remove(0); + CockroachDBTableReference rightTable = tableReferences.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + + tables = tableReferences.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joinExpressions; + } + + @Override + public List getTableRefs() { + List tableReferences = tables.stream().map(t -> new CockroachDBTableReference(t)) + .collect(Collectors.toList()); + + return CockroachDBCommon.getTableReferences(tableReferences); + } + + @Override + public String generateOptimizedQueryString(CockroachDBSelect select, CockroachDBExpression whereCondition, + boolean shouldUseAggregate) { + CockroachDBColumn c = new CockroachDBColumn("COUNT(*)", null, false, false); + select.setWhereClause(whereCondition); + if (shouldUseAggregate) { + CockroachDBAggregate aggr = new CockroachDBAggregate(CockroachDBAggregateFunction.COUNT, + List.of(new CockroachDBColumnReference(new CockroachDBColumn("*", + new CockroachDBCompositeDataType(CockroachDBDataType.INT, 0), false, false)))); + select.setFetchColumns(List.of(aggr)); + } else { + select.setFetchColumns(List.of(new CockroachDBColumnReference(c))); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(getOrderingTerms()); + } + } + return CockroachDBVisitor.asString(select); + } + + @Override + public String generateUnoptimizedQueryString(CockroachDBSelect select, CockroachDBExpression whereCondition) { + List tableList = select.getFromList(); + List joinList = select.getJoinList(); + String fromString = tableList.stream().map(t -> ((CockroachDBTableReference) t).getTable().getName()) + .collect(Collectors.joining(", ")); + if (!tableList.isEmpty() && !joinList.isEmpty()) { + fromString += ", "; + } + return "SELECT SUM(count) FROM (SELECT CAST(" + CockroachDBVisitor.asString(whereCondition) + + " IS TRUE AS INT) as count FROM " + fromString + " " + + joinList.stream().map(j -> CockroachDBVisitor.asString(j)).collect(Collectors.joining(", ")) + ")"; + } } diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java index 010188431..86d5d75ce 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -128,7 +128,7 @@ protected boolean mutateJoin() { columns.addAll(((CockroachDBTableReference) join.getLeftTable()).getTable().getColumns()); columns.addAll(((CockroachDBTableReference) join.getRightTable()).getTable().getColumns()); CockroachDBExpressionGenerator joinGen2 = new CockroachDBExpressionGenerator(state).setColumns(columns); - join.setOnCondition(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); + join.setOnClause(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); } JoinType newJoinType = CockroachDBJoin.JoinType.INNER; From 4819e48209bc126947413b725584a377af98e68b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 16:26:02 +0800 Subject: [PATCH 0928/1284] [CockroachDB] Use common NoREC oracle --- .../oracle/CockroachDBNoRECOracle.java | 120 ++++-------------- 1 file changed, 24 insertions(+), 96 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 2bc1873a0..3d0422dfd 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -2,66 +2,52 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.SQLGlobalState; -import sqlancer.cockroachdb.CockroachDBCommon; +import sqlancer.Reproducer; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.cockroachdb.CockroachDBSchema; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; -import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.cockroachdb.ast.CockroachDBColumnReference; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBJoin; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -public class CockroachDBNoRECOracle extends NoRECBase - implements TestOracle { +public class CockroachDBNoRECOracle implements TestOracle { - private CockroachDBExpressionGenerator gen; + NoRECOracle oracle; public CockroachDBNoRECOracle(CockroachDBGlobalState globalState) { - super(globalState); - CockroachDBErrors.addExpressionErrors(errors); - CockroachDBErrors.addTransactionErrors(errors); - errors.add("unable to vectorize execution plan"); // SET vectorize=experimental_always; - errors.add(" mismatched physical types at index"); // SET vectorize=experimental_always; + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with(CockroachDBErrors.getTransactionErrors()).with("unable to vectorize execution plan") // SET + // vectorize=experimental_always; + .with(" mismatched physical types at index") // SET vectorize=experimental_always; + .build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - CockroachDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); - List tableL = tables.getTables().stream().map(t -> new CockroachDBTableReference(t)) - .collect(Collectors.toList()); - List tableList = CockroachDBCommon.getTableReferences(tableL); - gen = new CockroachDBExpressionGenerator(state).setColumns(tables.getColumns()); - List joinExpressions = getJoins(tableList, state); - CockroachDBExpression whereCondition = gen.generateExpression(CockroachDBDataType.BOOL.get()); - int optimizableCount = getOptimizedResult(whereCondition, tableList, errors, joinExpressions); - if (optimizableCount == -1) { - throw new IgnoreMeException(); - } - int nonOptimizableCount = getNonOptimizedResult(whereCondition, tableList, errors, joinExpressions); - if (nonOptimizableCount == -1) { - throw new IgnoreMeException(); - } - if (optimizableCount != nonOptimizableCount) { - state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); - throw new AssertionError(CockroachDBVisitor.asString(whereCondition)); - } + oracle.check(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } public static List getJoins(List tableList, @@ -80,62 +66,4 @@ public static List getJoins(List t return joinExpressions; } - private int getOptimizedResult(CockroachDBExpression whereCondition, List tableList, - ExpectedErrors errors, List joinExpressions) throws SQLException { - CockroachDBSelect select = new CockroachDBSelect(); - CockroachDBColumn c = new CockroachDBColumn("COUNT(*)", null, false, false); - select.setFetchColumns(Arrays.asList(new CockroachDBColumnReference(c))); - select.setFromList(tableList); - select.setWhereClause(whereCondition); - select.setJoinList(joinExpressions); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.getOrderingTerms()); - } - String s = CockroachDBVisitor.asString(select); - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(s); - } - this.optimizedQueryString = s; - SQLQueryAdapter q = new SQLQueryAdapter(s, errors); - return getCount(state, q); - } - - private int getNonOptimizedResult(CockroachDBExpression whereCondition, List tableList, - ExpectedErrors errors, List joinList) throws SQLException { - String fromString = tableList.stream().map(t -> ((CockroachDBTableReference) t).getTable().getName()) - .collect(Collectors.joining(", ")); - if (!tableList.isEmpty() && !joinList.isEmpty()) { - fromString += ", "; - } - String s = "SELECT SUM(count) FROM (SELECT CAST(" + CockroachDBVisitor.asString(whereCondition) - + " IS TRUE AS INT) as count FROM " + fromString + " " - + joinList.stream().map(j -> CockroachDBVisitor.asString(j)).collect(Collectors.joining(", ")) + ")"; - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(s); - } - this.unoptimizedQueryString = s; - SQLQueryAdapter q = new SQLQueryAdapter(s, errors); - return getCount(state, q); - } - - private int getCount(SQLGlobalState globalState, SQLQueryAdapter q) throws AssertionError { - int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return -1; - } - if (rs.next()) { - count = rs.getInt(1); - } - } catch (Exception e) { - throw new AssertionError(q.getQueryString(), e); - } - return count; - } - - @Override - public String getLastQueryString() { - return optimizedQueryString; - } - } From 2fe7b6b00846aba7e975522f67255e14bafbb427 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 17:16:42 +0800 Subject: [PATCH 0929/1284] [CockroachDB] Add bug 85230 --- src/sqlancer/cockroachdb/CockroachDBBugs.java | 3 +++ .../cockroachdb/gen/CockroachDBTruncateGenerator.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 55045f637..6c34bcac4 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -56,6 +56,9 @@ public final class CockroachDBBugs { // https://github.com/cockroachdb/cockroach/issues/88037 public static boolean bug88037; + // https://github.com/cockroachdb/cockroach/issues/85230 + public static boolean bug85230 = true; + private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index 904ee3a9f..44979487b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -1,6 +1,7 @@ package sqlancer.cockroachdb.gen; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -14,6 +15,9 @@ private CockroachDBTruncateGenerator() { public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add("is referenced by foreign key"); + if (CockroachDBBugs.bug85230) { + errors.add("found in depended-on-by references, no such index in this relation"); + } StringBuilder sb = new StringBuilder(); sb.append("TRUNCATE"); From d18ce4c0d9e8aeb98b7bf801959806e1c2851de5 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 17:39:49 +0800 Subject: [PATCH 0930/1284] [CockroachDB] Update tests --- .github/workflows/main.yml | 4 +++- .../cockroachdb/CockroachDBErrors.java | 2 ++ ...roachDB.java => TestCockroachDBNoREC.java} | 10 ++++------ test/sqlancer/dbms/TestCockroachDBTLP.java | 19 +++++++++++++++++++ test/sqlancer/dbms/TestConfig.java | 1 + 5 files changed, 29 insertions(+), 7 deletions(-) rename test/sqlancer/dbms/{TestCockroachDB.java => TestCockroachDBNoREC.java} (58%) create mode 100644 test/sqlancer/dbms/TestCockroachDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68b02c443..5b50754b3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -144,7 +144,9 @@ jobs: - name: Create SQLancer user run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests - run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test + run: | + COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBNoREC test + COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBTLP test cockroachdb-qpg: name: QPG Tests (CockroachDB) diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index b52df11bd..7a06fe51e 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -206,6 +206,8 @@ public static List getExpressionErrors() { errors.add("argument of OFFSET must be type int, not type decimal"); errors.add("ERROR: for SELECT DISTINCT, ORDER BY expressions must appear in select list"); + errors.add("incompatible IF expressions"); + errors.addAll(getArrayErrors()); errors.addAll(getComputedColumnErrors()); diff --git a/test/sqlancer/dbms/TestCockroachDB.java b/test/sqlancer/dbms/TestCockroachDBNoREC.java similarity index 58% rename from test/sqlancer/dbms/TestCockroachDB.java rename to test/sqlancer/dbms/TestCockroachDBNoREC.java index 5eb701e3b..bf1a51193 100644 --- a/test/sqlancer/dbms/TestCockroachDB.java +++ b/test/sqlancer/dbms/TestCockroachDBNoREC.java @@ -7,15 +7,13 @@ import sqlancer.Main; -public class TestCockroachDB { +public class TestCockroachDBNoREC { @Test - public void testMySQL() { - String cockroachDB = System.getenv("COCKROACHDB_AVAILABLE"); - boolean cockroachDBIsAvailable = cockroachDB != null && cockroachDB.equalsIgnoreCase("true"); - assumeTrue(cockroachDBIsAvailable); + public void testCockroachDBNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb" })); + "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "NOREC" })); } } diff --git a/test/sqlancer/dbms/TestCockroachDBTLP.java b/test/sqlancer/dbms/TestCockroachDBTLP.java new file mode 100644 index 000000000..f916844ed --- /dev/null +++ b/test/sqlancer/dbms/TestCockroachDBTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCockroachDBTLP { + + @Test + public void testCockroachDBTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 084de35e7..1845aa422 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -4,6 +4,7 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; From 67ee096fbb6fc57a1ea3b74cb13d3513cd609322 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 10:10:40 +0800 Subject: [PATCH 0931/1284] [Databend] Implement NoREC expression generator interface --- .../databend/ast/DatabendExpression.java | 4 +- src/sqlancer/databend/ast/DatabendJoin.java | 16 ++-- src/sqlancer/databend/ast/DatabendSelect.java | 26 +++++- .../gen/DatabendNewExpressionGenerator.java | 85 ++++++++++++++++++- .../gen/DatabendRandomQuerySynthesizer.java | 4 +- .../tlp/DatabendQueryPartitioningBase.java | 2 +- 6 files changed, 124 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendExpression.java b/src/sqlancer/databend/ast/DatabendExpression.java index d53952140..d71e64ed7 100644 --- a/src/sqlancer/databend/ast/DatabendExpression.java +++ b/src/sqlancer/databend/ast/DatabendExpression.java @@ -1,8 +1,10 @@ package sqlancer.databend.ast; +import sqlancer.common.ast.newast.Expression; +import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; -public interface DatabendExpression { +public interface DatabendExpression extends Expression { default DatabendDataType getExpectedType() { return null; diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index b927eab92..9386c1fe1 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -4,6 +4,7 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; @@ -11,12 +12,12 @@ import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendJoin implements DatabendExpression { +public class DatabendJoin implements DatabendExpression, Join { private final DatabendTableReference leftTable; private final DatabendTableReference rightTable; private final JoinType joinType; - private final DatabendExpression onCondition; + private DatabendExpression onCondition; private OuterType outerType; public enum JoinType { @@ -67,9 +68,8 @@ public OuterType getOuterType() { return outerType; } - public static List getJoins(List tableList, - DatabendGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DatabendGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { DatabendTableReference leftTable = tableList.remove(0); DatabendTableReference rightTable = tableList.remove(0); @@ -116,11 +116,15 @@ public static DatabendJoin createInnerJoin(DatabendTableReference left, Databend return new DatabendJoin(left, right, JoinType.INNER, predicate); } - public static DatabendExpression createNaturalJoin(DatabendTableReference left, DatabendTableReference right, + public static DatabendJoin createNaturalJoin(DatabendTableReference left, DatabendTableReference right, OuterType naturalJoinType) { DatabendJoin join = new DatabendJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; } + @Override + public void setOnClause(DatabendExpression onClause) { + onCondition = onClause; + } } diff --git a/src/sqlancer/databend/ast/DatabendSelect.java b/src/sqlancer/databend/ast/DatabendSelect.java index 3b0854ede..adc75d1cc 100644 --- a/src/sqlancer/databend/ast/DatabendSelect.java +++ b/src/sqlancer/databend/ast/DatabendSelect.java @@ -1,8 +1,16 @@ package sqlancer.databend.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendToStringVisitor; -public class DatabendSelect extends SelectBase implements DatabendExpression { +public class DatabendSelect extends SelectBase + implements DatabendExpression, Select { private boolean isDistinct; @@ -14,4 +22,20 @@ public boolean isDistinct() { return isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DatabendExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DatabendJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DatabendToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 19a45eaa2..a3b7e996e 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -9,12 +9,17 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendSchema.DatabendRowValue; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendAggregateOperation; import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; import sqlancer.databend.ast.DatabendBetweenOperation; @@ -24,21 +29,29 @@ import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; import sqlancer.databend.ast.DatabendBinaryLogicalOperation; import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; +import sqlancer.databend.ast.DatabendCastOperation; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendInOperation; +import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendLikeOperation; import sqlancer.databend.ast.DatabendOrderByTerm; +import sqlancer.databend.ast.DatabendPostFixText; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; public class DatabendNewExpressionGenerator - extends TypedExpressionGenerator { + extends TypedExpressionGenerator + implements NoRECGenerator { private final DatabendGlobalState globalState; + private List tables; private final int maxDepth; private boolean allowAggregateFunctions; @@ -248,7 +261,8 @@ DatabendExpression getLike(int depth, DatabendDataType dataType) { } public DatabendExpression generateExpressionWithExpectedResult(DatabendDataType type) { - // DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + // DatabendNewExpressionGenerator gen = new + // DatabendNewExpressionGenerator(globalState).setColumns(columns); // gen.setRowValue(rowValue); DatabendExpression expr; do { @@ -340,4 +354,71 @@ public DatabendExpression generateHavingClause() { return expression; } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public DatabendExpression generateBooleanExpression() { + return generateExpression(DatabendDataType.BOOLEAN); + } + + @Override + public DatabendSelect generateSelect() { + return new DatabendSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); + List joins = DatabendJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DatabendTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DatabendSelect select, DatabendExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + DatabendExpression aggr = new DatabendAggregateOperation( + List.of(new DatabendColumnReference(new DatabendColumn("*", + new DatabendCompositeDataType(DatabendDataType.INT, 0), false, false))), + DatabendAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + List allColumns = columns.stream().map((c) -> new DatabendColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DatabendSelect select, DatabendExpression whereCondition) { + DatabendExpression asText = new DatabendPostFixText(new DatabendCastOperation( + new DatabendPostFixText(whereCondition, + " IS NOT NULL AND " + DatabendToStringVisitor.asString(whereCondition)), + new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); + select.setFetchColumns(List.of(asText)); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } + } diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 7b7d0e35b..875c5afee 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -51,8 +51,8 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DatabendTableReference(t)) .collect(Collectors.toList()); - List joins = DatabendJoin.getJoins(tableList, globalState); - select.setJoinList(joins); + List joins = DatabendJoin.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 50a02f0ff..611164902 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -62,7 +62,7 @@ public void check() throws SQLException { List tableList = tables.stream().map(t -> new DatabendTableReference(t)) .collect(Collectors.toList()); if (!DatabendBugs.bug9236) { - List joins = DatabendJoin.getJoins(tableList, state); + List joins = DatabendJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); } select.setFromList(tableList.stream().collect(Collectors.toList())); From f28934a01e7de4b73c872fc2e1c9542804b22e17 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 10:11:28 +0800 Subject: [PATCH 0932/1284] [Databend] Use common NoREC oracle --- .../databend/test/DatabendNoRECOracle.java | 141 ++---------------- 1 file changed, 16 insertions(+), 125 deletions(-) diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 54f17858c..4d0d29711 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -1,154 +1,45 @@ package sqlancer.databend.test; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; -import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.DatabendSchema.DatabendTables; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.ast.DatabendCastOperation; -import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; -import sqlancer.databend.ast.DatabendPostFixText; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendNoRECOracle extends NoRECBase implements TestOracle { +public class DatabendNoRECOracle implements TestOracle { - private final DatabendSchema s; + NoRECOracle oracle; public DatabendNoRECOracle(DatabendGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - DatabendErrors.addExpressionErrors(errors); + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - DatabendTables randomTables = s.getRandomTableNonEmptyAndViewTables(); // 随机获得nr张表 - List columns = randomTables.getColumns(); - if (columns.isEmpty()) { - debugColumns(columns, randomTables); // 调试代码，可忽略 - } - DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - DatabendExpression randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件 - List tables = randomTables.getTables(); - List tableList = tables.stream().map(t -> new DatabendTableReference(t)) - .collect(Collectors.toList()); - List joins = DatabendJoin.getJoins(tableList, state); - int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), - randomWhereCondition, joins); - int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, - randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getUnoptimizedQueryCount(List tableList, DatabendExpression randomWhereCondition, - List joins) throws SQLException { - DatabendSelect select = new DatabendSelect(); - // select.setGroupByClause(groupBys); - DatabendExpression asText = new DatabendPostFixText(new DatabendCastOperation( - new DatabendPostFixText(randomWhereCondition, - " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), - new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); - - select.setFetchColumns(List.of(asText)); - select.setFromList(tableList); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + DatabendToStringVisitor.asString(select) + ") as res"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(SQLConnection con, List tableList, - List columns, DatabendExpression randomWhereCondition, List joins) - throws SQLException { - DatabendSelect select = new DatabendSelect(); - // select.setGroupByClause(groupBys); - List allColumns = columns.stream().map((c) -> new DatabendColumnReference(c)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new DatabendNewExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = DatabendToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - void debugColumns(List columns, DatabendTables randomTables) { - DatabendTables test = new DatabendTables(s.getDatabaseTables()); - System.out.println(String.format("tables size: %d", test.getTables().size())); - for (DatabendTable table : test.getTables()) { - System.out.println(String.format("%s", table.getName())); - for (DatabendColumn column : table.getColumns()) { - System.out.println(String.format("%s %s", column.getName(), column.getType())); - } - System.out.println("------------------------"); - } - System.out.println("+++++++++++++++++++++++++++++"); - for (DatabendTable table : randomTables.getTables()) { - System.out.println(String.format("%s", table.getName())); - for (DatabendColumn column : table.getColumns()) { - System.out.println(String.format("%s %s", column.getName(), column.getType())); - } - System.out.println("------------------------"); - } - throw new AssertionError( - String.format("randomTables size: %d,column is empty", randomTables.getTables().size())); + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From 0f5157e8ff5d6385a07659b417164445df48caac Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 10:55:15 +0800 Subject: [PATCH 0933/1284] [Doris] Implement NoREC expression generator interfaces --- src/sqlancer/doris/ast/DorisExpression.java | 4 +- src/sqlancer/doris/ast/DorisJoin.java | 15 +++- src/sqlancer/doris/ast/DorisSelect.java | 26 +++++- .../gen/DorisNewExpressionGenerator.java | 85 ++++++++++++++++++- .../gen/DorisRandomQuerySynthesizer.java | 2 +- .../tlp/DorisQueryPartitioningBase.java | 2 +- 6 files changed, 125 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/doris/ast/DorisExpression.java b/src/sqlancer/doris/ast/DorisExpression.java index 559ecd8f3..ee5cc7a26 100644 --- a/src/sqlancer/doris/ast/DorisExpression.java +++ b/src/sqlancer/doris/ast/DorisExpression.java @@ -1,8 +1,10 @@ package sqlancer.doris.ast; +import sqlancer.common.ast.newast.Expression; import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; -public interface DorisExpression { +public interface DorisExpression extends Expression { default DorisSchema.DorisDataType getExpectedType() { return null; } diff --git a/src/sqlancer/doris/ast/DorisJoin.java b/src/sqlancer/doris/ast/DorisJoin.java index 7c7a35c18..c92555ec3 100644 --- a/src/sqlancer/doris/ast/DorisJoin.java +++ b/src/sqlancer/doris/ast/DorisJoin.java @@ -4,17 +4,19 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.gen.DorisNewExpressionGenerator; -public class DorisJoin implements DorisExpression { +public class DorisJoin implements DorisExpression, Join { private final DorisTableReference leftTable; private final DorisTableReference rightTable; private final JoinType joinType; - private final DorisExpression onCondition; + private DorisExpression onCondition; public enum JoinType { INNER, STRAIGHT, LEFT, RIGHT; @@ -48,8 +50,8 @@ public DorisExpression getOnCondition() { return onCondition; } - public static List getJoins(List tableList, DorisGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DorisGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { DorisTableReference leftTable = tableList.remove(0); DorisTableReference rightTable = tableList.remove(0); @@ -99,4 +101,9 @@ public static DorisJoin createLeftOuterJoin(DorisTableReference left, DorisTable DorisExpression predicate) { return new DorisJoin(left, right, JoinType.LEFT, predicate); } + + @Override + public void setOnClause(DorisExpression onClause) { + onCondition = onClause; + } } diff --git a/src/sqlancer/doris/ast/DorisSelect.java b/src/sqlancer/doris/ast/DorisSelect.java index 6194a0498..4921e0688 100644 --- a/src/sqlancer/doris/ast/DorisSelect.java +++ b/src/sqlancer/doris/ast/DorisSelect.java @@ -1,9 +1,17 @@ package sqlancer.doris.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.visitor.DorisToStringVisitor; -public class DorisSelect extends SelectBase implements DorisExpression { +public class DorisSelect extends SelectBase + implements DorisExpression, Select { public enum DorisSelectDistinctType { @@ -37,4 +45,20 @@ public boolean isDistinct() { || this.selectDistinctType == DorisSelectDistinctType.DISTINCTROW; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DorisExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DorisJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DorisToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java index 13fe028ca..7cea29f31 100644 --- a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -10,12 +10,16 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.doris.DorisBugs; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.DorisSchema.DorisRowValue; +import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.ast.DorisAggregateOperation; import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; import sqlancer.doris.ast.DorisBetweenOperation; @@ -27,21 +31,29 @@ import sqlancer.doris.ast.DorisBinaryLogicalOperation.DorisBinaryLogicalOperator; import sqlancer.doris.ast.DorisCaseOperation; import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisColumnValue; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisFunctionOperation.DorisFunction; import sqlancer.doris.ast.DorisInOperation; +import sqlancer.doris.ast.DorisJoin; import sqlancer.doris.ast.DorisLikeOperation; import sqlancer.doris.ast.DorisOrderByTerm; +import sqlancer.doris.ast.DorisPostfixText; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.ast.DorisUnaryPostfixOperation; import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; +import sqlancer.doris.visitor.DorisToStringVisitor; -public class DorisNewExpressionGenerator extends TypedExpressionGenerator { +public class DorisNewExpressionGenerator extends TypedExpressionGenerator + implements NoRECGenerator { private final DorisGlobalState globalState; + private List tables; private final int maxDepth; private boolean allowAggregateFunctions; @@ -449,4 +461,75 @@ public DorisExpression generateHavingClause() { public void setAllowAggregateFunctions(boolean allowAggregateFunctions) { this.allowAggregateFunctions = allowAggregateFunctions; } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public DorisExpression generateBooleanExpression() { + return generateExpression(DorisDataType.BOOLEAN); + } + + @Override + public DorisSelect generateSelect() { + return new DorisSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DorisTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DorisSelect select, DorisExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + DorisExpression aggr = new DorisAggregateOperation( + List.of(new DorisColumnReference( + new DorisColumn("*", new DorisCompositeDataType(DorisDataType.INT, 0), false, false))), + DorisAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + + } else { + List allColumns = columns.stream().map((c) -> new DorisColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + List constants = new ArrayList<>(); + constants.add(new DorisConstant.DorisIntConstant( + Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByClauses(constants); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DorisSelect select, DorisExpression whereCondition) { + DorisExpression asText = new DorisPostfixText(new DorisCastOperation( + new DorisPostfixText(whereCondition, + " IS NOT NULL AND " + DorisToStringVisitor.asString(whereCondition)), + new DorisCompositeDataType(DorisDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } diff --git a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java index 518a204a5..e1ec50eb2 100644 --- a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java +++ b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java @@ -47,7 +47,7 @@ public static DorisSelect generateSelect(DorisGlobalState globalState, int nrCol List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DorisTableReference(t)) .collect(Collectors.toList()); - List joins = DorisJoin.getJoins(tableList, globalState); + List joins = DorisJoin.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java index b3d3a7a6b..553e7739a 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java @@ -57,7 +57,7 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DorisTableReference(t)) .collect(Collectors.toList()); - List joins = DorisJoin.getJoins(tableList, state); + List joins = DorisJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); From c43052382a204314eca00140771b71a027b99dfd Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 10:55:41 +0800 Subject: [PATCH 0934/1284] [Doris] Use common NoREC oracle --- .../doris/oracle/DorisNoRECOracle.java | 121 +++--------------- 1 file changed, 16 insertions(+), 105 deletions(-) diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java index 8840a299e..9ce94b952 100644 --- a/src/sqlancer/doris/oracle/DorisNoRECOracle.java +++ b/src/sqlancer/doris/oracle/DorisNoRECOracle.java @@ -1,133 +1,44 @@ package sqlancer.doris.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisColumn; -import sqlancer.doris.DorisSchema.DorisCompositeDataType; -import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.DorisSchema.DorisTable; -import sqlancer.doris.DorisSchema.DorisTables; -import sqlancer.doris.ast.DorisCastOperation; -import sqlancer.doris.ast.DorisColumnReference; -import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisJoin; -import sqlancer.doris.ast.DorisPostfixText; import sqlancer.doris.ast.DorisSelect; -import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.gen.DorisNewExpressionGenerator; -import sqlancer.doris.visitor.DorisToStringVisitor; -public class DorisNoRECOracle extends NoRECBase implements TestOracle { +public class DorisNoRECOracle implements TestOracle { - private final DorisSchema s; + NoRECOracle oracle; public DorisNoRECOracle(DorisGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - DorisErrors.addExpressionErrors(errors); - DorisErrors.addInsertErrors(errors); + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - DorisTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state).setColumns(columns); - DorisExpression randomWhereCondition = gen.generateExpression(DorisDataType.BOOLEAN); - List tables = randomTables.getTables(); - List tableList = tables.stream().map(t -> new DorisTableReference(t)) - .collect(Collectors.toList()); - List joins = DorisJoin.getJoins(tableList, state); - int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), - randomWhereCondition, joins); - int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, - randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getUnoptimizedQueryCount(List tableList, DorisExpression randomWhereCondition, - List joins) throws SQLException { - DorisSelect select = new DorisSelect(); - DorisExpression asText = new DorisPostfixText(new DorisCastOperation( - new DorisPostfixText(randomWhereCondition, - " IS NOT NULL AND " + DorisToStringVisitor.asString(randomWhereCondition)), - new DorisCompositeDataType(DorisDataType.INT, 8)), "as count"); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(tableList); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + DorisToStringVisitor.asString(select) + ") as res"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getOptimizedQueryCount(SQLConnection con, List tableList, List columns, - DorisExpression randomWhereCondition, List joins) throws SQLException { - DorisSelect select = new DorisSelect(); - // select.setGroupByClause(groupBys); - List allColumns = columns.stream().map((c) -> new DorisColumnReference(c)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - List constants = new ArrayList<>(); - constants.add( - new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByClauses(constants); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = DorisToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From 3fc5c0c28956337a4f882bba2b7fb0899b40e2f7 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 11:36:41 +0800 Subject: [PATCH 0935/1284] [YSQL] Implement NoREC expression generator interfaces --- .../yugabyte/ysql/ast/YSQLExpression.java | 4 +- src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java | 11 +- .../yugabyte/ysql/ast/YSQLSelect.java | 12 +- .../ysql/gen/YSQLExpressionGenerator.java | 118 +++++++++++++++++- .../yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 55 ++++---- 5 files changed, 162 insertions(+), 38 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java index 9544c8838..5f68afd5d 100644 --- a/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java @@ -1,8 +1,10 @@ package sqlancer.yugabyte.ysql.ast; +import sqlancer.common.ast.newast.Expression; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; -public interface YSQLExpression { +public interface YSQLExpression extends Expression { default YSQLDataType getExpressionType() { return null; diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java index 746d46cfd..4edcbd11a 100644 --- a/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java @@ -1,12 +1,15 @@ package sqlancer.yugabyte.ysql.ast; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; -public class YSQLJoin implements YSQLExpression { +public class YSQLJoin implements YSQLExpression, Join { private final YSQLExpression tableReference; - private final YSQLExpression onClause; + private YSQLExpression onClause; private final YSQLJoinType type; public YSQLJoin(YSQLExpression tableReference, YSQLExpression onClause, YSQLJoinType type) { @@ -46,4 +49,8 @@ public static YSQLJoinType getRandom() { } + @Override + public void setOnClause(YSQLExpression onClause) { + this.onClause = onClause; + } } diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java index ce477c238..fbf96ee45 100644 --- a/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java @@ -5,10 +5,14 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; -public class YSQLSelect extends SelectBase implements YSQLExpression { +public class YSQLSelect extends SelectBase + implements YSQLExpression, Select { private SelectType selectOption = SelectType.ALL; private List joinClauses = Collections.emptyList(); @@ -32,10 +36,12 @@ public YSQLDataType getExpressionType() { return null; } + @Override public List getJoinClauses() { return joinClauses; } + @Override public void setJoinClauses(List joinStatements) { this.joinClauses = joinStatements; @@ -132,4 +138,8 @@ public YSQLDataType getExpressionType() { } } + @Override + public String asString() { + return YSQLVisitor.asString(this); + } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java index c6a554d67..759d4d3ba 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java @@ -11,13 +11,18 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.yugabyte.ysql.YSQLCompoundDataType; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLRowValue; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate.YSQLAggregateFunction; import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; import sqlancer.yugabyte.ysql.ast.YSQLBinaryArithmeticOperation; import sqlancer.yugabyte.ysql.ast.YSQLBinaryBitOperation; @@ -32,13 +37,17 @@ import sqlancer.yugabyte.ysql.ast.YSQLFunction; import sqlancer.yugabyte.ysql.ast.YSQLFunctionWithUnknownResult; import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; -public class YSQLExpressionGenerator implements ExpressionGenerator { +public class YSQLExpressionGenerator implements ExpressionGenerator, + NoRECGenerator { private final int maxDepth; @@ -46,6 +55,7 @@ public class YSQLExpressionGenerator implements ExpressionGenerator functionsAndTypes; private final List allowedFunctionTypes; private List columns; + private List tables; private YSQLRowValue rw; private boolean expectedResult; private YSQLGlobalState globalState; @@ -560,4 +570,110 @@ private enum IntExpression { UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION } + public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState, String name, YSQLTables tables) { + List columns = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + YSQLSelect select = new YSQLSelect(); + select.setFromList(tables.getTables().stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(YSQLSelect.ForClause.getRandom()); + } + return new YSQLSelect.YSQLSubquery(select, name); + } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public YSQLExpression generateBooleanExpression() { + return generateExpression(YSQLDataType.BOOLEAN); + } + + @Override + public YSQLSelect generateSelect() { + return new YSQLSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLTable table = Randomly.fromList(tables); + tables.remove(table); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLSelect.YSQLSubquery subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(YSQLSelect select, YSQLExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + YSQLAggregate aggr = new YSQLAggregate(List.of(new YSQLColumnValue(YSQLColumn.createDummy("*"), null)), + YSQLAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + YSQLColumnValue allColumns = new YSQLColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(Arrays.asList(allColumns)); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBy()); + } + select.setWhereClause(whereCondition); + } + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(YSQLSelect select, YSQLExpression whereCondition) { + YSQLCastOperation isTrue = new YSQLCastOperation(whereCondition, YSQLCompoundDataType.create(YSQLDataType.INT)); + YSQLPostfixText asText = new YSQLPostfixText(isTrue, " as count", null, YSQLDataType.INT); + select.setFetchColumns(Collections.singletonList(asText)); + select.setSelectType(YSQLSelect.SelectType.ALL); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java index fa63ffc1f..57fe246e3 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -18,12 +18,10 @@ import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; -import sqlancer.yugabyte.ysql.ast.YSQLConstant; import sqlancer.yugabyte.ysql.ast.YSQLExpression; import sqlancer.yugabyte.ysql.ast.YSQLJoin; import sqlancer.yugabyte.ysql.ast.YSQLSelect; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; -import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; public class YSQLTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { @@ -39,34 +37,6 @@ public YSQLTLPBase(YSQLGlobalState state) { YSQLErrors.addCommonFetchErrors(errors); } - public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState, String name, YSQLTables tables) { - List columns = new ArrayList<>(); - YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - columns.add(gen.generateExpression(0)); - } - YSQLSelect select = new YSQLSelect(); - select.setFromList(tables.getTables().stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) - .collect(Collectors.toList())); - select.setFetchColumns(columns); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - if (Randomly.getBoolean()) { - select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - if (Randomly.getBoolean()) { - select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - } - } - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setForClause(YSQLSelect.ForClause.getRandom()); - } - return new YSQLSelect.YSQLSubquery(select, name); - } - @Override public void check() throws SQLException { s = state.getSchema(); @@ -76,10 +46,29 @@ public void check() throws SQLException { generateSelectBase(tables, joins); } - protected List getJoinStatements(YSQLGlobalState globalState, List columns, + public static List getJoinStatements(YSQLGlobalState globalState, List columns, List tables) { - return YSQLNoRECOracle.getJoinStatements(state, columns, tables); - // TODO joins + List joinStatements = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLTable table = Randomly.fromList(tables); + tables.remove(table); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLSelect.YSQLSubquery subquery = YSQLExpressionGenerator.createSubquery(globalState, + String.format("sub%d", i), subqueryTables); + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; } protected void generateSelectBase(List tables, List joins) { From 4b0c923b2362686968317af80b59277318c68f03 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 11:37:17 +0800 Subject: [PATCH 0936/1284] [YSQL] Use common NoREC oracle --- .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 153 ++---------------- 1 file changed, 16 insertions(+), 137 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java index 37a6491ad..70804ce40 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -1,165 +1,44 @@ package sqlancer.yugabyte.ysql.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.common.query.ExpectedErrors; import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; -import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; -import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; -import sqlancer.yugabyte.ysql.YSQLVisitor; -import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; -import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; import sqlancer.yugabyte.ysql.ast.YSQLExpression; import sqlancer.yugabyte.ysql.ast.YSQLJoin; -import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; import sqlancer.yugabyte.ysql.ast.YSQLSelect; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; -import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPBase; -public class YSQLNoRECOracle extends NoRECBase implements TestOracle { +public class YSQLNoRECOracle implements TestOracle { - private final YSQLSchema s; + NoRECOracle oracle; public YSQLNoRECOracle(YSQLGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - YSQLErrors.addCommonExpressionErrors(errors); - YSQLErrors.addCommonFetchErrors(errors); - } - - public static List getJoinStatements(YSQLGlobalState globalState, List columns, - List tables) { - List joinStatements = new ArrayList<>(); - YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); - YSQLTable table = Randomly.fromList(tables); - tables.remove(table); - YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); - YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); - joinStatements.add(j); - } - // JOIN subqueries - for (int i = 0; i < Randomly.smallNumber(); i++) { - YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); - YSQLSelect.YSQLSubquery subquery = YSQLTLPBase.createSubquery(globalState, String.format("sub%d", i), - subqueryTables); - YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); - YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); - YSQLJoin j = new YSQLJoin(subquery, joinClause, options); - joinStatements.add(j); - } - return joinStatements; + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) + .with(YSQLErrors.getCommonFetchErrors()).with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - YSQLTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - YSQLExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); - - List joinStatements = getJoinStatements(state, columns, tables); - List fromTables = tables.stream() - .map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } - } - - private YSQLExpression getRandomWhereCondition(List columns) { - return new YSQLExpressionGenerator(state).setColumns(columns).generateExpression(YSQLDataType.BOOLEAN); + oracle.check(); } - private int getUnoptimizedQueryCount(List fromTables, YSQLExpression randomWhereCondition, - List joinStatements) throws SQLException { - YSQLSelect select = new YSQLSelect(); - YSQLCastOperation isTrue = new YSQLCastOperation(randomWhereCondition, - YSQLCompoundDataType.create(YSQLDataType.INT)); - YSQLPostfixText asText = new YSQLPostfixText(isTrue, " as count", null, YSQLDataType.INT); - select.setFetchColumns(Collections.singletonList(asText)); - select.setFromList(fromTables); - select.setSelectType(YSQLSelect.SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + YSQLVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getOptimizedQueryCount(List randomTables, List columns, - YSQLExpression randomWhereCondition, List joinStatements) throws SQLException { - YSQLSelect select = new YSQLSelect(); - YSQLColumnValue allColumns = new YSQLColumnValue(Randomly.fromList(columns), null); - select.setFetchColumns(Arrays.asList(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new YSQLExpressionGenerator(state).setColumns(columns).generateOrderBy()); - } - select.setSelectType(YSQLSelect.SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = YSQLVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From 1af322b4aaf46fe16778a6ebfb18a67372faaa99 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 11:49:24 +0800 Subject: [PATCH 0937/1284] [Yugabyte] Update tests --- .github/workflows/main.yml | 7 ++++-- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestYCQL.java | 18 ++++++++++++++++ test/sqlancer/dbms/TestYSQLNoREC.java | 19 ++++++++++++++++ test/sqlancer/dbms/TestYSQLPQS.java | 19 ++++++++++++++++ test/sqlancer/dbms/TestYSQLTLP.java | 19 ++++++++++++++++ test/sqlancer/dbms/TestYugabyte.java | 31 --------------------------- 7 files changed, 81 insertions(+), 33 deletions(-) create mode 100644 test/sqlancer/dbms/TestYCQL.java create mode 100644 test/sqlancer/dbms/TestYSQLNoREC.java create mode 100644 test/sqlancer/dbms/TestYSQLPQS.java create mode 100644 test/sqlancer/dbms/TestYSQLTLP.java delete mode 100644 test/sqlancer/dbms/TestYugabyte.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68b02c443..83c98666c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -522,8 +522,11 @@ jobs: docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false sleep 5 - name: Run Tests - run: YUGABYTE_AVAILABLE=true mvn -Dtest=TestYugabyte test - + run: | + YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLNoREC test + YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLTLP test + YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLPQS test + YUGABYTE_AVAILABLE=true mvn -Dtest=TestYCQL test doris: name: DBMS Tests (Apache Doris) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 084de35e7..510a34dd0 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -8,6 +8,7 @@ public class TestConfig { public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; + public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { String value = System.getenv(key); diff --git a/test/sqlancer/dbms/TestYCQL.java b/test/sqlancer/dbms/TestYCQL.java new file mode 100644 index 000000000..1198e2376 --- /dev/null +++ b/test/sqlancer/dbms/TestYCQL.java @@ -0,0 +1,18 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestYCQL { + @Test + public void testYCQL() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "cassandra", "--password", "cassandra", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ycql")); + } +} diff --git a/test/sqlancer/dbms/TestYSQLNoREC.java b/test/sqlancer/dbms/TestYSQLNoREC.java new file mode 100644 index 000000000..51f0487cc --- /dev/null +++ b/test/sqlancer/dbms/TestYSQLNoREC.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestYSQLNoREC { + @Test + public void testYSQLNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ysql", "--oracle", "NOREC")); + } +} diff --git a/test/sqlancer/dbms/TestYSQLPQS.java b/test/sqlancer/dbms/TestYSQLPQS.java new file mode 100644 index 000000000..6a21b6cb2 --- /dev/null +++ b/test/sqlancer/dbms/TestYSQLPQS.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestYSQLPQS { + @Test + public void testYSQLPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ysql", "--oracle", "PQS")); + } +} diff --git a/test/sqlancer/dbms/TestYSQLTLP.java b/test/sqlancer/dbms/TestYSQLTLP.java new file mode 100644 index 000000000..fdcdf372a --- /dev/null +++ b/test/sqlancer/dbms/TestYSQLTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestYSQLTLP { + @Test + public void testYSQLTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ysql", "--oracle", "QUERY_PARTITIONING")); + } +} diff --git a/test/sqlancer/dbms/TestYugabyte.java b/test/sqlancer/dbms/TestYugabyte.java deleted file mode 100644 index 10387948b..000000000 --- a/test/sqlancer/dbms/TestYugabyte.java +++ /dev/null @@ -1,31 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestYugabyte { - - String yugabyteAvailable = System.getenv("YUGABYTE_AVAILABLE"); - boolean yugabyteIsAvailable = yugabyteAvailable != null && yugabyteAvailable.equalsIgnoreCase("true"); - - @Test - public void testYCQL() { - assumeTrue(yugabyteIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "cassandra", "--password", "cassandra", "--num-threads", "1", "--num-queries", - TestConfig.NUM_QUERIES, "ycql")); - } - - @Test - public void testYSQL() { - assumeTrue(yugabyteIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", - TestConfig.NUM_QUERIES, "ysql")); - } -} From 1ef2f90192eb3179dc6aa50458adb06eca536088 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 13:41:13 +0800 Subject: [PATCH 0938/1284] [Materialize] Implement NoREC expression generator interfaces --- .../ast/MaterializeExpression.java | 4 +- .../materialize/ast/MaterializeJoin.java | 12 ++- .../materialize/ast/MaterializeSelect.java | 12 ++- .../gen/MaterializeExpressionGenerator.java | 100 +++++++++++++++++- 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/materialize/ast/MaterializeExpression.java b/src/sqlancer/materialize/ast/MaterializeExpression.java index 8a42ad3f6..70444d8ee 100644 --- a/src/sqlancer/materialize/ast/MaterializeExpression.java +++ b/src/sqlancer/materialize/ast/MaterializeExpression.java @@ -1,8 +1,10 @@ package sqlancer.materialize.ast; +import sqlancer.common.ast.newast.Expression; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; -public interface MaterializeExpression { +public interface MaterializeExpression extends Expression { default MaterializeDataType getExpressionType() { return null; diff --git a/src/sqlancer/materialize/ast/MaterializeJoin.java b/src/sqlancer/materialize/ast/MaterializeJoin.java index 967cdf420..ef8f1ac61 100644 --- a/src/sqlancer/materialize/ast/MaterializeJoin.java +++ b/src/sqlancer/materialize/ast/MaterializeJoin.java @@ -1,9 +1,13 @@ package sqlancer.materialize.ast; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; -public class MaterializeJoin implements MaterializeExpression { +public class MaterializeJoin + implements MaterializeExpression, Join { public enum MaterializeJoinType { INNER, LEFT, RIGHT, FULL, CROSS; @@ -14,8 +18,8 @@ public static MaterializeJoinType getRandom() { } + private MaterializeExpression onClause; private final MaterializeExpression tableReference; - private final MaterializeExpression onClause; private final MaterializeJoinType type; public MaterializeJoin(MaterializeExpression tableReference, MaterializeExpression onClause, @@ -47,4 +51,8 @@ public MaterializeConstant getExpectedValue() { throw new AssertionError(); } + @Override + public void setOnClause(MaterializeExpression onClause) { + this.onClause = onClause; + } } diff --git a/src/sqlancer/materialize/ast/MaterializeSelect.java b/src/sqlancer/materialize/ast/MaterializeSelect.java index cef761428..db5b85a2f 100644 --- a/src/sqlancer/materialize/ast/MaterializeSelect.java +++ b/src/sqlancer/materialize/ast/MaterializeSelect.java @@ -5,10 +5,14 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; -public class MaterializeSelect extends SelectBase implements MaterializeExpression { +public class MaterializeSelect extends SelectBase implements MaterializeExpression, + Select { private SelectType selectOption = SelectType.ALL; private List joinClauses = Collections.emptyList(); @@ -111,11 +115,13 @@ public MaterializeDataType getExpressionType() { return null; } + @Override public void setJoinClauses(List joinStatements) { this.joinClauses = joinStatements; } + @Override public List getJoinClauses() { return joinClauses; } @@ -132,4 +138,8 @@ public ForClause getForClause() { return forClause; } + @Override + public String asString() { + return MaterializeVisitor.asString(this); + } } diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java index 1396052a2..f3af50038 100644 --- a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -11,12 +11,16 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.materialize.MaterializeCompoundDataType; import sqlancer.materialize.MaterializeGlobalState; import sqlancer.materialize.MaterializeProvider; import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; import sqlancer.materialize.MaterializeSchema.MaterializeRowValue; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; import sqlancer.materialize.ast.MaterializeAggregate; import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; import sqlancer.materialize.ast.MaterializeBetweenOperation; @@ -36,6 +40,8 @@ import sqlancer.materialize.ast.MaterializeFunction.MaterializeFunctionWithResult; import sqlancer.materialize.ast.MaterializeFunctionWithUnknownResult; import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; import sqlancer.materialize.ast.MaterializeLikeOperation; import sqlancer.materialize.ast.MaterializeOrderByTerm; import sqlancer.materialize.ast.MaterializeOrderByTerm.MaterializeOrder; @@ -43,10 +49,17 @@ import sqlancer.materialize.ast.MaterializePOSIXRegularExpression.POSIXRegex; import sqlancer.materialize.ast.MaterializePostfixOperation; import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializePostfixText; import sqlancer.materialize.ast.MaterializePrefixOperation; import sqlancer.materialize.ast.MaterializePrefixOperation.PrefixOperator; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSelect.SelectType; +import sqlancer.materialize.oracle.tlp.MaterializeTLPBase; -public class MaterializeExpressionGenerator implements ExpressionGenerator { +public class MaterializeExpressionGenerator implements ExpressionGenerator, + NoRECGenerator { private final int maxDepth; @@ -54,6 +67,8 @@ public class MaterializeExpressionGenerator implements ExpressionGenerator columns; + private List tables; + private MaterializeRowValue rw; private boolean expectedResult; @@ -514,4 +529,87 @@ public MaterializeExpression isNull(MaterializeExpression expr) { return new MaterializePostfixOperation(expr, PostfixOperator.IS_NULL); } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public MaterializeExpression generateBooleanExpression() { + return generateExpression(MaterializeDataType.BOOLEAN); + } + + @Override + public MaterializeSelect generateSelect() { + return new MaterializeSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeTable table = Randomly.fromList(tables); + tables.remove(table); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, + options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); + joinStatements.add(j); + } + + return joinStatements; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new MaterializeFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(MaterializeSelect select, MaterializeExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + MaterializeAggregate aggr = new MaterializeAggregate( + List.of(new MaterializeColumnValue(MaterializeColumn.createDummy("*"), null)), + MaterializeAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + MaterializeColumnValue allColumns = new MaterializeColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(List.of(allColumns)); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBy()); + } + select.setSelectType(SelectType.ALL); + } + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(MaterializeSelect select, MaterializeExpression whereCondition) { + MaterializeCastOperation isTrue = new MaterializeCastOperation(whereCondition, + MaterializeCompoundDataType.create(MaterializeDataType.INT)); + MaterializePostfixText asText = new MaterializePostfixText(isTrue, " as count", null, MaterializeDataType.INT); + select.setFetchColumns(List.of(asText)); + select.setSelectType(SelectType.ALL); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } From c0d4fe541ed1f1ed04b91bc01b0de5d25b0ad156 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 13:41:36 +0800 Subject: [PATCH 0939/1284] [Materialize] Use common NoREC oracle --- .../oracle/MaterializeNoRECOracle.java | 160 ++---------------- .../oracle/tlp/MaterializeTLPBase.java | 27 ++- 2 files changed, 39 insertions(+), 148 deletions(-) diff --git a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java index bf9cc43fd..e0f665d93 100644 --- a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java +++ b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java @@ -1,175 +1,45 @@ package sqlancer.materialize.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.common.query.ExpectedErrors; import sqlancer.materialize.MaterializeGlobalState; import sqlancer.materialize.MaterializeSchema; import sqlancer.materialize.MaterializeSchema.MaterializeColumn; -import sqlancer.materialize.MaterializeSchema.MaterializeDataType; import sqlancer.materialize.MaterializeSchema.MaterializeTable; -import sqlancer.materialize.MaterializeSchema.MaterializeTables; -import sqlancer.materialize.MaterializeVisitor; -import sqlancer.materialize.ast.MaterializeCastOperation; -import sqlancer.materialize.ast.MaterializeColumnValue; import sqlancer.materialize.ast.MaterializeExpression; import sqlancer.materialize.ast.MaterializeJoin; -import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; -import sqlancer.materialize.ast.MaterializePostfixText; import sqlancer.materialize.ast.MaterializeSelect; -import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; -import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; -import sqlancer.materialize.ast.MaterializeSelect.SelectType; import sqlancer.materialize.gen.MaterializeCommon; import sqlancer.materialize.gen.MaterializeExpressionGenerator; -import sqlancer.materialize.oracle.tlp.MaterializeTLPBase; -public class MaterializeNoRECOracle extends NoRECBase - implements TestOracle { +public class MaterializeNoRECOracle implements TestOracle { - private final MaterializeSchema s; + NoRECOracle oracle; public MaterializeNoRECOracle(MaterializeGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - MaterializeCommon.addCommonExpressionErrors(errors); - MaterializeCommon.addCommonFetchErrors(errors); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) + .with(MaterializeCommon.getCommonFetchErrors()).with("canceling statement due to statement timeout") + .build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - MaterializeTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - MaterializeExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); - - List joinStatements = getJoinStatements(state, columns, tables); - List fromTables = tables.stream() - .map(t -> new MaterializeFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } - } - - public static List getJoinStatements(MaterializeGlobalState globalState, - List columns, List tables) { - List joinStatements = new ArrayList<>(); - MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); - MaterializeTable table = Randomly.fromList(tables); - tables.remove(table); - MaterializeJoinType options = MaterializeJoinType.getRandom(); - MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, - options); - joinStatements.add(j); - } - // JOIN subqueries - for (int i = 0; i < Randomly.smallNumber(); i++) { - MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); - MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), - subqueryTables); - MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); - MaterializeJoinType options = MaterializeJoinType.getRandom(); - MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); - joinStatements.add(j); - } - return joinStatements; - } - - private MaterializeExpression getRandomWhereCondition(List columns) { - return new MaterializeExpressionGenerator(state).setColumns(columns) - .generateExpression(MaterializeDataType.BOOLEAN); - } - - private int getUnoptimizedQueryCount(List fromTables, - MaterializeExpression randomWhereCondition, List joinStatements) throws SQLException { - MaterializeSelect select = new MaterializeSelect(); - MaterializeCastOperation isTrue = new MaterializeCastOperation(randomWhereCondition, - MaterializeCompoundDataType.create(MaterializeDataType.INT)); - MaterializePostfixText asText = new MaterializePostfixText(isTrue, " as count", null, MaterializeDataType.INT); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(fromTables); - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + MaterializeVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + oracle.check(); } - private int getOptimizedQueryCount(List randomTables, List columns, - MaterializeExpression randomWhereCondition, List joinStatements) throws SQLException { - MaterializeSelect select = new MaterializeSelect(); - MaterializeColumnValue allColumns = new MaterializeColumnValue(Randomly.fromList(columns), null); - select.setFetchColumns(Arrays.asList(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new MaterializeExpressionGenerator(state).setColumns(columns).generateOrderBy()); - } - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = MaterializeVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } @Override public String getLastQueryString() { - return optimizedQueryString; + return oracle.getLastQueryString(); } } diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java index dcf16dfad..36711997a 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java @@ -20,13 +20,13 @@ import sqlancer.materialize.ast.MaterializeConstant; import sqlancer.materialize.ast.MaterializeExpression; import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; import sqlancer.materialize.ast.MaterializeSelect; import sqlancer.materialize.ast.MaterializeSelect.ForClause; import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; import sqlancer.materialize.gen.MaterializeCommon; import sqlancer.materialize.gen.MaterializeExpressionGenerator; -import sqlancer.materialize.oracle.MaterializeNoRECOracle; public class MaterializeTLPBase extends TernaryLogicPartitioningOracleBase @@ -54,8 +54,29 @@ public void check() throws SQLException { protected List getJoinStatements(MaterializeGlobalState globalState, List columns, List tables) { - return MaterializeNoRECOracle.getJoinStatements(state, columns, tables); - // TODO joins + List joinStatements = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeTable table = Randomly.fromList(tables); + tables.remove(table); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, + options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); + joinStatements.add(j); + } + + return joinStatements; } protected void generateSelectBase(List tables, List joins) { From 7fddc4bce9c513dc5b47f1f44f2f34e7bec5f056 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 13:49:41 +0800 Subject: [PATCH 0940/1284] [Materialize] Update tests --- .github/workflows/main.yml | 3 ++- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestMaterializeNoREC.java | 21 +++++++++++++++++++ test/sqlancer/dbms/TestMaterializePQS.java | 5 +---- ...terialize.java => TestMaterializeTLP.java} | 9 +++----- 5 files changed, 28 insertions(+), 11 deletions(-) create mode 100644 test/sqlancer/dbms/TestMaterializeNoREC.java rename test/sqlancer/dbms/{TestMaterialize.java => TestMaterializeTLP.java} (67%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42845c307..aaafeb8cd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -323,7 +323,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterialize + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeNoREC + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeTLP MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializePQS materialize-qpg: diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 4ef95d907..0ed58e137 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -8,6 +8,7 @@ public class TestConfig { public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; + public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestMaterializeNoREC.java b/test/sqlancer/dbms/TestMaterializeNoREC.java new file mode 100644 index 000000000..e4ac980ce --- /dev/null +++ b/test/sqlancer/dbms/TestMaterializeNoREC.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMaterializeNoREC { + + @Test + public void test() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MATERIALIZE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--username", "materialize", + "materialize", "--oracle", "NOREC", "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/dbms/TestMaterializePQS.java b/test/sqlancer/dbms/TestMaterializePQS.java index 5d4d795e4..0c6be974d 100644 --- a/test/sqlancer/dbms/TestMaterializePQS.java +++ b/test/sqlancer/dbms/TestMaterializePQS.java @@ -9,12 +9,9 @@ public class TestMaterializePQS { - String materializeAvailable = System.getenv("MATERIALIZE_AVAILABLE"); - boolean materializeIsAvailable = materializeAvailable != null && materializeAvailable.equalsIgnoreCase("true"); - @Test public void test() { - assumeTrue(materializeIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MATERIALIZE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", diff --git a/test/sqlancer/dbms/TestMaterialize.java b/test/sqlancer/dbms/TestMaterializeTLP.java similarity index 67% rename from test/sqlancer/dbms/TestMaterialize.java rename to test/sqlancer/dbms/TestMaterializeTLP.java index 1818aa513..a98b3e053 100644 --- a/test/sqlancer/dbms/TestMaterialize.java +++ b/test/sqlancer/dbms/TestMaterializeTLP.java @@ -7,18 +7,15 @@ import sqlancer.Main; -public class TestMaterialize { - - String materializeAvailable = System.getenv("MATERIALIZE_AVAILABLE"); - boolean materializeIsAvailable = materializeAvailable != null && materializeAvailable.equalsIgnoreCase("true"); +public class TestMaterializeTLP { @Test public void test() { - assumeTrue(materializeIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MATERIALIZE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--username", "materialize", - "materialize", "--set-max-tables-mvs", "true" })); + "materialize", "--set-max-tables-mvs", "true", "--oracle", "QUERY_PARTITIONING" })); } } From fd1ba0ebb7c2acabf1842b5367a02ffdb6e49581 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 14:58:11 +0800 Subject: [PATCH 0941/1284] [DataFusion] Implement NoREC expression generator interfaces --- .../datafusion/ast/DataFusionExpression.java | 5 +- .../datafusion/ast/DataFusionJoin.java | 18 +++-- .../datafusion/ast/DataFusionSelect.java | 23 +++++- .../gen/DataFusionExpressionGenerator.java | 79 ++++++++++++++++++- 4 files changed, 113 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/datafusion/ast/DataFusionExpression.java b/src/sqlancer/datafusion/ast/DataFusionExpression.java index 6da95cd93..eaf84ba1d 100644 --- a/src/sqlancer/datafusion/ast/DataFusionExpression.java +++ b/src/sqlancer/datafusion/ast/DataFusionExpression.java @@ -1,5 +1,8 @@ package sqlancer.datafusion.ast; -public interface DataFusionExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; + +public interface DataFusionExpression extends Expression { } diff --git a/src/sqlancer/datafusion/ast/DataFusionJoin.java b/src/sqlancer/datafusion/ast/DataFusionJoin.java index 430b2b812..05718c5ad 100644 --- a/src/sqlancer/datafusion/ast/DataFusionJoin.java +++ b/src/sqlancer/datafusion/ast/DataFusionJoin.java @@ -2,9 +2,9 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; @@ -14,12 +14,13 @@ /* NOT IMPLEMENTED YET */ -public class DataFusionJoin implements DataFusionExpression { +public class DataFusionJoin + implements DataFusionExpression, Join { private final DataFusionTableReference leftTable; private final DataFusionTableReference rightTable; private final JoinType joinType; - private final DataFusionExpression onCondition; + private DataFusionExpression onCondition; public DataFusionJoin(DataFusionTableReference leftTable, DataFusionTableReference rightTable, JoinType joinType, DataFusionExpression whereCondition) { @@ -29,11 +30,10 @@ public DataFusionJoin(DataFusionTableReference leftTable, DataFusionTableReferen this.onCondition = whereCondition; } - public static List getJoins(List tables, DataFusionGlobalState globalState) { + public static List getJoins(List tableList, + DataFusionGlobalState globalState) { // [t1_join_t2, t1_join_t3, ...] - List tableList = tables.stream().map(t -> new DataFusionTableReference(t)) - .collect(Collectors.toList()); - List joinExpressions = new ArrayList<>(); + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { DataFusionTableReference leftTable = tableList.remove(0); DataFusionTableReference rightTable = tableList.remove(0); @@ -84,4 +84,8 @@ public static JoinType getRandom() { } } + @Override + public void setOnClause(DataFusionExpression onClause) { + onCondition = onClause; + } } diff --git a/src/sqlancer/datafusion/ast/DataFusionSelect.java b/src/sqlancer/datafusion/ast/DataFusionSelect.java index 6fda25144..10cf4859f 100644 --- a/src/sqlancer/datafusion/ast/DataFusionSelect.java +++ b/src/sqlancer/datafusion/ast/DataFusionSelect.java @@ -6,12 +6,16 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionToStringVisitor; import sqlancer.datafusion.gen.DataFusionExpressionGenerator; -public class DataFusionSelect extends SelectBase implements DataFusionExpression { +public class DataFusionSelect extends SelectBase implements DataFusionExpression, + Select { public Optional fetchColumnsString = Optional.empty(); // When available, override `fetchColumns` in base // class's `Node` representation (for display) public DataFusionExpressionGenerator exprGen; @@ -58,4 +62,21 @@ public static DataFusionSelect getRandomSelect(DataFusionGlobalState state) { public void setFetchColumnsString(String selectExpr) { this.fetchColumnsString = Optional.of(selectExpr); } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DataFusionExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DataFusionJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DataFusionToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java index 88e486f47..d56493fa2 100644 --- a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java @@ -13,22 +13,31 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionToStringVisitor; import sqlancer.datafusion.ast.DataFusionBinaryOperation; import sqlancer.datafusion.ast.DataFusionColumnReference; import sqlancer.datafusion.ast.DataFusionExpression; import sqlancer.datafusion.ast.DataFusionFunction; +import sqlancer.datafusion.ast.DataFusionJoin; +import sqlancer.datafusion.ast.DataFusionSelect; +import sqlancer.datafusion.ast.DataFusionTableReference; import sqlancer.datafusion.ast.DataFusionUnaryPostfixOperation; import sqlancer.datafusion.ast.DataFusionUnaryPrefixOperation; import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; public final class DataFusionExpressionGenerator - extends TypedExpressionGenerator { + extends TypedExpressionGenerator implements + NoRECGenerator { + private List tables; private final DataFusionGlobalState globalState; public DataFusionExpressionGenerator(DataFusionGlobalState globalState) { @@ -100,7 +109,8 @@ protected DataFusionExpression generateExpression(DataFusionDataType type, int d case BINARY: dfAssert(randomExpr.argTypes.size() == 2 && randomExpr.nArgs == 2, "Binrary expression should only have 2 argument" + randomExpr.argTypes); - List argTypeList = new ArrayList<>(); // types of current expression's input arguments + List argTypeList = new ArrayList<>(); // types of current expression's input + // arguments for (ArgumentType argumentType : randomExpr.argTypes) { if (argumentType instanceof ArgumentType.Fixed) { ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); @@ -134,7 +144,8 @@ protected DataFusionExpression generateExpression(DataFusionDataType type, int d public DataFusionExpression generateFunctionExpression(DataFusionDataType type, int depth, DataFusionBaseExpr exprType) { if (exprType.isVariadic || Randomly.getBooleanWithSmallProbability()) { - // TODO(datafusion) maybe add possible types. e.g. some function have signature variadic(INT/DOUBLE), then + // TODO(datafusion) maybe add possible types. e.g. some function have signature + // variadic(INT/DOUBLE), then // only randomly pick from INT and DOUBLE int nArgs = Randomly.smallNumber(); // 0, 2, 4, ... smaller one is more likely return new DataFusionFunction(generateExpressions(nArgs), exprType); @@ -222,4 +233,66 @@ public String getTextRepresentation() { } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + List randomTables = Randomly.nonEmptySubset(tables.getTables()); + int maxSize = Randomly.fromOptions(1, 2, 3, 4); + if (randomTables.size() > maxSize) { + randomTables = randomTables.subList(0, maxSize); + } + this.columns = DataFusionTable.getAllColumns(randomTables); + this.tables = randomTables; + + return this; + } + + @Override + public DataFusionExpression generateBooleanExpression() { + return generateExpression(DataFusionDataType.BOOLEAN); + } + + @Override + public DataFusionSelect generateSelect() { + return new DataFusionSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DataFusionTableReference(t)) + .collect(Collectors.toList()); + List joins = DataFusionJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DataFusionTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DataFusionSelect select, DataFusionExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + select.setFetchColumnsString("COUNT(*)"); + } else { + List allColumns = columns.stream().map((c) -> new DataFusionColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DataFusionSelect select, DataFusionExpression whereCondition) { + String fetchColumn = String.format("COUNT(CASE WHEN %S THEN 1 ELSE NULL END)", + DataFusionToStringVisitor.asString(whereCondition)); + select.setFetchColumnsString(fetchColumn); + select.setWhereClause(null); + + return select.asString(); + } } From dded5b9bca73322634b33147c9a9a3c4bcbb805b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 15:00:05 +0800 Subject: [PATCH 0942/1284] [DataFusion] Use common NoREC oracle --- src/sqlancer/datafusion/DataFusionErrors.java | 12 ++- .../test/DataFusionNoRECOracle.java | 99 +++++-------------- 2 files changed, 35 insertions(+), 76 deletions(-) diff --git a/src/sqlancer/datafusion/DataFusionErrors.java b/src/sqlancer/datafusion/DataFusionErrors.java index d4b208f5b..4be35017e 100644 --- a/src/sqlancer/datafusion/DataFusionErrors.java +++ b/src/sqlancer/datafusion/DataFusionErrors.java @@ -2,6 +2,9 @@ import static sqlancer.datafusion.DataFusionUtil.dfAssert; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class DataFusionErrors { @@ -17,7 +20,8 @@ private DataFusionErrors() { * Note now it's implemented this way for simplicity This way might cause false negative, because Q1 and Q2 should * both succeed or both fail TODO(datafusion): ensure both succeed or both fail */ - public static void registerExpectedExecutionErrors(ExpectedErrors errors) { + public static List getExpectedExecutionErrors() { + ArrayList errors = new ArrayList<>(); /* * Expected */ @@ -40,5 +44,11 @@ public static void registerExpectedExecutionErrors(ExpectedErrors errors) { errors.add("Physical plan does not support logical expression AggregateFunction"); // False positive: when aggr // is generated in where // clause + + return errors; + } + + public static void registerExpectedExecutionErrors(ExpectedErrors errors) { + errors.addAll(getExpectedExecutionErrors()); } } diff --git a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java index 55f18fecd..36ab97854 100644 --- a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java +++ b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java @@ -1,95 +1,44 @@ package sqlancer.datafusion.test; -import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.ERROR; -import static sqlancer.datafusion.ast.DataFusionSelect.getRandomSelect; - import java.sql.SQLException; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.datafusion.DataFusionErrors; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionToStringVisitor; -import sqlancer.datafusion.DataFusionUtil; +import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionJoin; import sqlancer.datafusion.ast.DataFusionSelect; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; -public class DataFusionNoRECOracle extends NoRECBase - implements TestOracle { +public class DataFusionNoRECOracle implements TestOracle { - private final DataFusionGlobalState state; + NoRECOracle oracle; public DataFusionNoRECOracle(DataFusionGlobalState globalState) { - super(globalState); - this.state = globalState; - DataFusionErrors.registerExpectedExecutionErrors(errors); + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } - /* - * Non-Optimizing Reference Engine Construction q1: SELECT [expr1] FROM [expr2] WHERE [expr3] q2: SELECT [expr3] - * FROM [expr2] - * - * Oracle Check: q1's result size equals to `true` count in q2's result set - */ @Override public void check() throws SQLException { - /* - * Setup Q1 and Q2 - */ - // generate a random: - // SELECT [expr1] FROM [expr2] WHERE [expr3] - DataFusionSelect randomSelect = getRandomSelect(state); - // Q1: SELECT count(*) FROM [expr2] WHERE [expr3] - DataFusionSelect q1 = new DataFusionSelect(); - q1.setFetchColumnsString("COUNT(*)"); - q1.setFromList(randomSelect.getFromList()); - q1.setWhereClause(randomSelect.getWhereClause()); - // Q2: SELECT count(case when [expr3] then 1 else null end) FROM [expr2] - DataFusionSelect q2 = new DataFusionSelect(); - String selectExpr = String.format("COUNT(CASE WHEN %S THEN 1 ELSE NULL END)", - DataFusionToStringVisitor.asString(randomSelect.getWhereClause())); - q2.setFetchColumnsString(selectExpr); - q2.setFromList(randomSelect.getFromList()); - q2.setWhereClause(null); - - /* - * Execute Q1 and Q2 - */ - String q1String = DataFusionToStringVisitor.asString(q1); - String q2String = DataFusionToStringVisitor.asString(q2); - List q1ResultSet = null; - List q2ResultSet = null; - try { - q1ResultSet = ComparatorHelper.getResultSetFirstColumnAsString(q1String, errors, state); - q2ResultSet = ComparatorHelper.getResultSetFirstColumnAsString(q2String, errors, state); - } catch (AssertionError e) { - // Append detailed error message - String replay = DataFusionUtil.getReplay(state.getDatabaseName()); - String newMessage = e.getMessage() + "\n" + e.getCause() + "\n" + replay + "\n"; - state.dfLogger.appendToLog(ERROR, newMessage); - - throw new AssertionError(newMessage); - } - - /* - * NoREC check - */ - int count1 = q1ResultSet != null ? Integer.parseInt(q1ResultSet.get(0)) : -1; - int count2 = q2ResultSet != null ? Integer.parseInt(q2ResultSet.get(0)) : -1; - if (count1 != count2) { - StringBuilder errorMessage = new StringBuilder().append("NoREC oracle violated:\n") - .append(" Q1(result size ").append(count1).append("):").append(q1String).append(";\n") - .append(" Q2(result size ").append(count2).append("):").append(q2String).append(";\n") - .append("=======================================\n").append("Reproducer: \n"); - - String replay = DataFusionUtil.getReplay(state.getDatabaseName()); + oracle.check(); + } - String errorLog = errorMessage.toString() + replay + "\n"; - String indentedErrorLog = errorLog.replaceAll("(?m)^", " "); - state.dfLogger.appendToLog(ERROR, errorLog); + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } - throw new AssertionError("\n\n" + indentedErrorLog); - } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From 97095e89b8a4f44c894744eb12036fe7d2266102 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 4 Sep 2024 18:29:39 +0800 Subject: [PATCH 0943/1284] [DataFusion] Update tests --- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestDataFusion.java | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 1845aa422..9a3b4c63d 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -6,6 +6,7 @@ public class TestConfig { public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; + public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestDataFusion.java b/test/sqlancer/dbms/TestDataFusion.java index ae568e40e..b2b5e2a1a 100644 --- a/test/sqlancer/dbms/TestDataFusion.java +++ b/test/sqlancer/dbms/TestDataFusion.java @@ -10,9 +10,7 @@ public class TestDataFusion { @Test public void testDataFusion() { - String datafusionAvailable = System.getenv("DATAFUSION_AVAILABLE"); - boolean datafusionIsAvailable = datafusionAvailable != null && datafusionAvailable.equalsIgnoreCase("true"); - assumeTrue(datafusionIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATAFUSION_ENV)); assertEquals(0, Main.executeMain("--random-seed", "0", "--num-threads", "1", // TODO(datafusion) update when // multithread is supported From 22d4bb48da62a1e4443348a925f4e53ce2807431 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 30 Aug 2024 12:53:53 +0800 Subject: [PATCH 0944/1284] [ClickHouse] Implement NoREC expression generator interface --- .../clickhouse/ClickHouseToStringVisitor.java | 6 +- .../clickhouse/ast/ClickHouseExpression.java | 14 ++- .../clickhouse/ast/ClickHouseSelect.java | 45 +++++++-- .../gen/ClickHouseExpressionGenerator.java | 91 ++++++++++++++++++- 4 files changed, 142 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index cebce8fc2..29bcddcb7 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -74,10 +74,10 @@ public void visit(ClickHouseSelect select, boolean inner) { } visit(select.getFetchColumns()); - ClickHouseExpression fromClause = select.getFromClause(); - if (fromClause != null) { + List fromList = select.getFromList(); + if (fromList != null) { sb.append(" FROM "); - visit(fromClause); + visit(fromList); } List joins = select.getJoinClauses(); if (!joins.isEmpty()) { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index f5cd2ed1e..eff88d012 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -1,9 +1,13 @@ package sqlancer.clickhouse.ast; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; import sqlancer.common.visitor.BinaryOperation; import sqlancer.common.visitor.UnaryOperation; -public abstract class ClickHouseExpression { +public abstract class ClickHouseExpression implements Expression { public ClickHouseConstant getExpectedValue() { return null; @@ -61,7 +65,8 @@ public String getOperatorRepresentation() { } } - public static class ClickHouseJoin extends ClickHouseExpression { + public static class ClickHouseJoin extends ClickHouseExpression + implements Join { // TODO: support ANY, ALL, ASOF modifiers // LEFT_SEMI, RIGHT_SEMI are not deterministic as ClickHouse allows to read columns from // whitelist table as well @@ -71,7 +76,7 @@ public enum JoinType { private final ClickHouseTableReference leftTable; private final ClickHouseTableReference rightTable; - private ClickHouseJoinOnClause onClause; + private ClickHouseExpression onClause; private final ClickHouseJoin.JoinType type; public ClickHouseJoin(ClickHouseTableReference leftTable, ClickHouseTableReference rightTable, @@ -109,7 +114,8 @@ public ClickHouseJoin.JoinType getType() { return type; } - public void setOnClause(ClickHouseJoinOnClause onClause) { + @Override + public void setOnClause(ClickHouseExpression onClause) { this.onClause = onClause; } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java index 3904b767b..61aeffa4d 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java @@ -3,10 +3,16 @@ import java.util.Collections; import java.util.List; -public class ClickHouseSelect extends ClickHouseExpression { +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ClickHouseToStringVisitor; +import sqlancer.common.ast.newast.Select; + +public class ClickHouseSelect extends ClickHouseExpression implements + Select { private ClickHouseSelect.SelectType fromOptions = ClickHouseSelect.SelectType.ALL; - private ClickHouseExpression fromClause; + private List fromClauses; private ClickHouseExpression whereClause; private List groupByClause = Collections.emptyList(); private ClickHouseExpression limitClause; @@ -24,12 +30,13 @@ public void setSelectType(ClickHouseSelect.SelectType fromOptions) { this.setFromOptions(fromOptions); } - public void setFromClause(ClickHouseExpression fromClause) { - this.fromClause = fromClause; + public void setFromClause(ClickHouseExpression fromList) { + this.fromClauses = List.of(fromList); } - public ClickHouseExpression getFromClause() { - return fromClause; + @Override + public List getFromList() { + return fromClauses; } public ClickHouseSelect.SelectType getFromOptions() { @@ -40,68 +47,94 @@ public void setFromOptions(ClickHouseSelect.SelectType fromOptions) { this.fromOptions = fromOptions; } + @Override public ClickHouseExpression getWhereClause() { return whereClause; } + @Override public void setWhereClause(ClickHouseExpression whereClause) { this.whereClause = whereClause; } + @Override public void setGroupByClause(List groupByClause) { this.groupByClause = groupByClause; } + @Override public List getGroupByClause() { return groupByClause; } + @Override public void setLimitClause(ClickHouseExpression limitClause) { this.limitClause = limitClause; } + @Override public ClickHouseExpression getLimitClause() { return limitClause; } + @Override public List getOrderByClauses() { return orderByClause; } + @Override public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } + @Override public void setOffsetClause(ClickHouseExpression offsetClause) { this.offsetClause = offsetClause; } + @Override public ClickHouseExpression getOffsetClause() { return offsetClause; } + @Override public void setFetchColumns(List fetchColumns) { this.fetchColumns = fetchColumns; } + @Override public List getFetchColumns() { return fetchColumns; } + @Override public void setJoinClauses(List joinStatements) { this.joinStatements = joinStatements; } + @Override public List getJoinClauses() { return joinStatements; } + @Override public void setHavingClause(ClickHouseExpression havingClause) { this.havingClause = havingClause; } + @Override public ClickHouseExpression getHavingClause() { assert orderByClause != null; return havingClause; } + + @Override + public String asString() { + return ClickHouseToStringVisitor.asString(this); + } + + @Override + public void setFromList(List fromList) { + this.fromClauses = fromList; + } } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 03750fee0..b523703a3 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -11,13 +11,18 @@ import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAggregate.ClickHouseAggregateFunction; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryArithmeticOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; +import sqlancer.clickhouse.ast.ClickHouseSelect; import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; @@ -25,14 +30,18 @@ import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; public class ClickHouseExpressionGenerator - extends TypedExpressionGenerator { + extends TypedExpressionGenerator implements + NoRECGenerator { private final ClickHouseGlobalState globalState; public boolean allowAggregateFunctions; + private List tables; private final List columnRefs; public ClickHouseExpressionGenerator(ClickHouseGlobalState globalState) { @@ -345,4 +354,84 @@ public ClickHouseExpression negatePredicate(ClickHouseExpression predicate) { public ClickHouseExpression isNull(ClickHouseExpression expr) { return new ClickHouseUnaryPostfixOperation(expr, ClickHouseUnaryPostfixOperator.IS_NULL, false); } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.tables = tables.getTables(); + this.columns = tables.getColumns(); + return this; + } + + @Override + public ClickHouseExpression generateBooleanExpression() { + List columnRefs = columns.stream() + .map(c -> c.asColumnReference(c.getTable().getName())).collect(Collectors.toList()); + return generateExpressionWithColumns(columnRefs, 5); + } + + @Override + public ClickHouseSelect generateSelect() { + return new ClickHouseSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + if (globalState.getClickHouseOptions().testJoins && Randomly.getBoolean()) { + return joinStatements; + } + List leftTables = new ArrayList<>(); + leftTables.add(new ClickHouseTableReference(tables.get(0), null)); + if (Randomly.getBoolean() && !tables.isEmpty()) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + for (int i = 0; i < nrJoinClauses; i++) { + ClickHouseTableReference leftTable = leftTables + .get((int) Randomly.getNotCachedInteger(0, leftTables.size() - 1)); + ClickHouseTableReference rightTable = new ClickHouseTableReference(Randomly.fromList(tables), + "right_" + i); + ClickHouseExpression.ClickHouseJoinOnClause joinClause = generateJoinClause(leftTable, rightTable); + ClickHouseExpression.ClickHouseJoin.JoinType options = Randomly + .fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); + ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(leftTable, rightTable, + options, joinClause); + joinStatements.add(j); + leftTables.add(rightTable); + } + } + return joinStatements; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new ClickHouseTableReference(t, null)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(ClickHouseSelect select, ClickHouseExpression whereCondition, + boolean shouldUseAggregate) { + List filteredColumns = Randomly.extractNrRandomColumns(columns, + (int) Randomly.getNotCachedInteger(1, columns.size())); + if (shouldUseAggregate) { + ClickHouseAggregate aggr = new ClickHouseAggregate( + new ClickHouseColumnReference(ClickHouseColumn.createDummy("*", null), null, null), + ClickHouseAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + select.setFetchColumns(filteredColumns.stream().map(c -> c.asColumnReference(c.getTable().getName())) + .collect(Collectors.toList())); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(ClickHouseSelect select, ClickHouseExpression whereCondition) { + ClickHouseExpression inner = new ClickHouseAliasOperation(whereCondition, "check"); + + select.setFetchColumns(List.of(inner)); + select.setWhereClause(null); + return "SELECT SUM(check <> 0) FROM (" + select.asString() + ") as res"; + } } From 2c4d09a70e98ddae5cc692add54c6965ef0d0287 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 30 Aug 2024 12:54:28 +0800 Subject: [PATCH 0945/1284] [ClickHouse] Use common NoREC oracle --- .../oracle/norec/ClickHouseNoRECOracle.java | 119 +++--------------- 1 file changed, 18 insertions(+), 101 deletions(-) diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java index 9818d198b..d2acc24d6 100644 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -1,128 +1,45 @@ package sqlancer.clickhouse.oracle.norec; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ClickHouseToStringVisitor; -import sqlancer.clickhouse.ast.ClickHouseAliasOperation; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; -public class ClickHouseNoRECOracle extends NoRECBase - implements TestOracle { +public class ClickHouseNoRECOracle implements TestOracle { - private final ClickHouseSchema schema; + NoRECOracle oracle; public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { - super(globalState); - this.schema = globalState.getSchema(); - ClickHouseErrors.addExpectedExpressionErrors(errors); + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state); - List tables = schema.getRandomTableNonEmptyTables().getTables(); - ClickHouseTableReference table = new ClickHouseTableReference( - tables.get((int) Randomly.getNotCachedInteger(0, tables.size() - 1)), "left"); - List columns = table.getColumnReferences(); - - List joinStatements = new ArrayList<>(); - if (state.getClickHouseOptions().testJoins && Randomly.getBoolean()) { - joinStatements = gen.getRandomJoinClauses(table, tables); - columns.addAll(joinStatements.stream().flatMap(j -> j.getRightTable().getColumnReferences().stream()) - .collect(Collectors.toList())); - } - gen.addColumns(columns); - - ClickHouseExpression randomWhereCondition = gen.generateExpressionWithColumns(columns, 5); - int secondCount = getSecondQuery(table, randomWhereCondition, joinStatements); - int firstCount = getFirstQueryCount(table, columns, randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getSecondQuery(ClickHouseExpression table, ClickHouseExpression whereClause, - List joins) throws SQLException { - ClickHouseSelect select = new ClickHouseSelect(); - - ClickHouseExpression inner = new ClickHouseAliasOperation(whereClause, "check"); - - select.setFetchColumns(Arrays.asList(inner)); - select.setFromClause(table); - select.setJoinClauses(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(check <> 0) FROM (" + ClickHouseToStringVisitor.asString(select) - + ") as res"; - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getFirstQueryCount(ClickHouseExpression tableList, List columns, - ClickHouseExpression randomWhereCondition, List joins) - throws SQLException { - ClickHouseSelect select = new ClickHouseSelect(); - List filteredColumns = Randomly.extractNrRandomColumns(columns, - (int) Randomly.getNotCachedInteger(1, columns.size())); - select.setFetchColumns( - filteredColumns.stream().map(c -> (ClickHouseExpression) c).collect(Collectors.toList())); - select.setFromClause(tableList); - select.setWhereClause(randomWhereCondition); - select.setJoinClauses(joins); - int firstCount = 0; - optimizedQueryString = ClickHouseToStringVisitor.asString(select); - SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - SQLancerResultSet rs; - - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(optimizedQueryString, e); - } - if (rs == null) { - return -1; - } - - while (rs.next()) { - firstCount++; - } - rs.close(); - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From f43858d50654065b17d4951d68719d1a0d2f5c06 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 30 Aug 2024 13:48:19 +0800 Subject: [PATCH 0946/1284] [ClickHouse] Refactor environment check in tests --- test/sqlancer/dbms/TestClickHouse.java | 56 +++++++------------------- test/sqlancer/dbms/TestConfig.java | 1 + 2 files changed, 15 insertions(+), 42 deletions(-) diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index ae861dfd9..87d7713fe 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -11,9 +11,7 @@ public class TestClickHouse { @Test public void testClickHouseTLPWhereGroupBy() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "--database-prefix", "T1_", "clickhouse", @@ -22,9 +20,7 @@ public void testClickHouseTLPWhereGroupBy() { @Test public void testClickHouseTLPWhere() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "--database-prefix", "T2_", "clickhouse", @@ -33,9 +29,7 @@ public void testClickHouseTLPWhere() { @Test public void testClickHouseTLPHaving() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", @@ -44,9 +38,7 @@ public void testClickHouseTLPHaving() { @Test public void testClickHouseTLPGroupBy() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -55,9 +47,7 @@ public void testClickHouseTLPGroupBy() { @Test public void testClickHouseTLPDistinct() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -66,9 +56,7 @@ public void testClickHouseTLPDistinct() { @Test public void testClickHouseTLPAggregate() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -77,9 +65,7 @@ public void testClickHouseTLPAggregate() { @Test public void testClickHouseNoREC() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", @@ -88,9 +74,7 @@ public void testClickHouseNoREC() { @Test public void testClickHouseTLPWhereGroupByWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "--database-prefix", "T8_", "clickhouse", @@ -99,9 +83,7 @@ public void testClickHouseTLPWhereGroupByWithJoins() { @Test public void testClickHouseTLPWhereWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "--database-prefix", "T9_", "clickhouse", @@ -110,9 +92,7 @@ public void testClickHouseTLPWhereWithJoins() { @Test public void testClickHouseTLPHavingWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", @@ -121,9 +101,7 @@ public void testClickHouseTLPHavingWithJoins() { @Test public void testClickHouseTLPGroupByWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -132,9 +110,7 @@ public void testClickHouseTLPGroupByWithJoins() { @Test public void testClickHouseTLPDistinctWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -143,9 +119,7 @@ public void testClickHouseTLPDistinctWithJoins() { @Test public void testClickHouseTLPAggregateWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -154,9 +128,7 @@ public void testClickHouseTLPAggregateWithJoins() { @Test public void testClickHouseNoRECWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index d70c0ae2f..2f2c18e83 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -4,6 +4,7 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + public static final String CLICKHOUSE_ENV = "CLICKHOUSE_AVAILABLE"; public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; From bfb9b237fd1dc5fa0fed7bc5d9019c20d604d4fa Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 9 Sep 2024 13:49:01 +0800 Subject: [PATCH 0947/1284] [HSQLDB] Add CI & tests --- .github/workflows/main.yml | 20 ++++++++++++++++++++ test/sqlancer/dbms/TestHSQLDBNoREC.java | 15 +++++++++++++++ test/sqlancer/dbms/TestHSQLDBTLP.java | 15 +++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 test/sqlancer/dbms/TestHSQLDBNoREC.java create mode 100644 test/sqlancer/dbms/TestHSQLDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68b02c443..dfa1b1238 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -268,6 +268,26 @@ jobs: - name: Run Tests run: H2_AVAILABLE=true mvn -Dtest=TestH2 test + hsqldb: + name: DBMS Tests (HSQLB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + mvn -Dtest=TestHSQLDBNoREC test + mvn -Dtest=TestHSQLDBTLP test + mariadb: name: DBMS Tests (MariaDB) runs-on: ubuntu-latest diff --git a/test/sqlancer/dbms/TestHSQLDBNoREC.java b/test/sqlancer/dbms/TestHSQLDBNoREC.java new file mode 100644 index 000000000..721fbb126 --- /dev/null +++ b/test/sqlancer/dbms/TestHSQLDBNoREC.java @@ -0,0 +1,15 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestHSQLDBNoREC { + @Test + public void testHSQLDBNoREC() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "hsqldb", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestHSQLDBTLP.java b/test/sqlancer/dbms/TestHSQLDBTLP.java new file mode 100644 index 000000000..a0b9c18d9 --- /dev/null +++ b/test/sqlancer/dbms/TestHSQLDBTLP.java @@ -0,0 +1,15 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestHSQLDBTLP { + @Test + public void testHSQLDBTLP() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "hsqldb", "--oracle", "WHERE" })); + } +} From 40c9c113f2c245479700e41a36afb487199b52a7 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 9 Sep 2024 14:47:38 +0800 Subject: [PATCH 0948/1284] [OceanBase] Add CI & update expected errors --- .github/workflows/main.yml | 25 +++++++++++++++++++ src/sqlancer/oceanbase/OceanBaseErrors.java | 1 + .../oceanbase/gen/OceanBaseAlterTable.java | 2 +- .../gen/OceanBaseDeleteGenerator.java | 2 +- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestOceanBaseNoREC.java | 5 +--- test/sqlancer/dbms/TestOceanBasePQS.java | 5 +--- test/sqlancer/dbms/TestOceanBaseTLP.java | 5 +--- 8 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aaafeb8cd..f38173d96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -384,6 +384,31 @@ jobs: MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP + oceanbase: + name: DBMS Tests (OceanBase) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up OceanBase + run: | + docker run -p 2881:2881 --name oceanbase-ce -e MODE=mini -d oceanbase/oceanbase-ce:4.2.1-lts + sleep 120 + mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" + - name: Run Tests + run: | + OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseNoREC + OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBasePQS + OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseTLP postgres: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index 822c5497d..5ad9dd3f3 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -20,6 +20,7 @@ public static List getExpressionErrors() { errors.add("Truncated incorrect DOUBLE value"); errors.add("Invalid numeric"); errors.add("Data truncated for argument"); + errors.add("Data truncated for column"); return errors; } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java index e67256448..7e1a69dfc 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java @@ -56,7 +56,7 @@ private SQLQueryAdapter create() { case COMPRESSION: sb.append("COMPRESSION "); sb.append("'"); - sb.append(Randomly.fromOptions("ZLIB_1.0", "LZ4_1.0", "NONE")); + sb.append(Randomly.fromOptions("LZ4_1.0", "NONE")); sb.append("'"); break; default: diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index f2a6b2731..ea1cb36e8 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -44,7 +44,7 @@ private SQLQueryAdapter generate() { errors.addAll(Arrays.asList("doesn't have this option", "Truncated incorrect DOUBLE value", "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index", "Incorrect value", "Out of range value for column", - "Data truncation: %s value is out of range in '%s'")); + "Data truncation:")); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 2f2c18e83..b51309dfe 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -11,6 +11,7 @@ public class TestConfig { public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; + public static final String OCEANBASE_ENV = "OCEANBASE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestOceanBaseNoREC.java b/test/sqlancer/dbms/TestOceanBaseNoREC.java index 9c650265c..c687590de 100644 --- a/test/sqlancer/dbms/TestOceanBaseNoREC.java +++ b/test/sqlancer/dbms/TestOceanBaseNoREC.java @@ -9,12 +9,9 @@ public class TestOceanBaseNoREC { - String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); - boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); - @Test public void testNoREC() { - assumeTrue(oceanBaseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.OCEANBASE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--database-prefix", "norecdb", "--num-queries", TestConfig.NUM_QUERIES, diff --git a/test/sqlancer/dbms/TestOceanBasePQS.java b/test/sqlancer/dbms/TestOceanBasePQS.java index adb2007d5..404977398 100644 --- a/test/sqlancer/dbms/TestOceanBasePQS.java +++ b/test/sqlancer/dbms/TestOceanBasePQS.java @@ -9,12 +9,9 @@ public class TestOceanBasePQS { - String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); - boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); - @Test public void testPQS() { - assumeTrue(oceanBaseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.OCEANBASE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", diff --git a/test/sqlancer/dbms/TestOceanBaseTLP.java b/test/sqlancer/dbms/TestOceanBaseTLP.java index fe84118e6..cf86c2078 100644 --- a/test/sqlancer/dbms/TestOceanBaseTLP.java +++ b/test/sqlancer/dbms/TestOceanBaseTLP.java @@ -9,12 +9,9 @@ public class TestOceanBaseTLP { - String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); - boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); - @Test public void testTLP() { - assumeTrue(oceanBaseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.OCEANBASE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--database-prefix", "tlpdb", "--num-queries", TestConfig.NUM_QUERIES, From b13132f5d4d3a6f4c1b3d04d6db6ff157f0db288 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 9 Sep 2024 13:09:07 -0400 Subject: [PATCH 0949/1284] Update CockroachDB to v24.2.0 --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9773919fa..459db4425 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -138,11 +138,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.1.0-rc.2.linux-amd64.tgz | tar xvz - cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: | COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBNoREC test @@ -165,11 +165,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.1.0-rc.2.linux-amd64.tgz | tar xvz - cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test From 47e7ab73aa36ec2c957668dd8300b3528c685bba Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 10 Sep 2024 15:58:26 +0800 Subject: [PATCH 0950/1284] [OceanBase] Implement expression generator interfaces --- .../oceanbase/ast/OceanBaseExpression.java | 5 +- src/sqlancer/oceanbase/ast/OceanBaseJoin.java | 9 +- .../oceanbase/ast/OceanBaseSelect.java | 21 ++- .../gen/OceanBaseExpressionGenerator.java | 153 +++++++++++++++++- 4 files changed, 184 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/oceanbase/ast/OceanBaseExpression.java b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java index 195a05967..b721d711f 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseExpression.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java @@ -1,6 +1,9 @@ package sqlancer.oceanbase.ast; -public interface OceanBaseExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; + +public interface OceanBaseExpression extends Expression { default OceanBaseConstant getExpectedValue() { throw new AssertionError("PQS not supported for this operator"); diff --git a/src/sqlancer/oceanbase/ast/OceanBaseJoin.java b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java index 9e8271bbf..4855c0808 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseJoin.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java @@ -1,10 +1,17 @@ package sqlancer.oceanbase.ast; -public class OceanBaseJoin implements OceanBaseExpression { +import sqlancer.common.ast.newast.Join; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseJoin implements OceanBaseExpression, Join { @Override public OceanBaseConstant getExpectedValue() { throw new UnsupportedOperationException(); } + @Override + public void setOnClause(OceanBaseExpression onClause) { + } } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java index a673492c2..c4bbc1b87 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java @@ -5,8 +5,13 @@ import java.util.List; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; -public class OceanBaseSelect extends SelectBase implements OceanBaseExpression { +public class OceanBaseSelect extends SelectBase + implements OceanBaseExpression, Select { private SelectType fromOptions = SelectType.ALL; private List modifiers = Collections.emptyList(); @@ -60,4 +65,18 @@ public OceanBaseStringExpression getHint() { return hint; } + @Override + public void setJoinClauses(List joinStatements) { + } + + @Override + public List getJoinClauses() { + return List.of(); + } + + @Override + public String asString() { + return OceanBaseVisitor.asString(this); + } + } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java index f4f573698..3c17e5084 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java @@ -3,13 +3,20 @@ import java.sql.Connection; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseRowValue; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseAggregate.OceanBaseAggregateFunction; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation.BinaryComparisonOperator; import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; @@ -23,15 +30,22 @@ import sqlancer.oceanbase.ast.OceanBaseExists; import sqlancer.oceanbase.ast.OceanBaseExpression; import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseJoin; +import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseText; import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; -public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator { +public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator + implements + NoRECGenerator { private OceanBaseGlobalState state; private OceanBaseRowValue rowVal; + private List tables; public OceanBaseExpressionGenerator(OceanBaseGlobalState state) { this.state = state; @@ -209,4 +223,141 @@ public OceanBaseExpression isNull(OceanBaseExpression expr) { return new OceanBaseUnaryPostfixOperation(expr, OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public OceanBaseExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public OceanBaseSelect generateSelect() { + return new OceanBaseSelect(); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(OceanBaseSelect select, OceanBaseExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + OceanBaseExpression aggr = new OceanBaseAggregate( + new OceanBaseColumnReference(new OceanBaseColumn("*", OceanBaseDataType.INT, false, 0, false), + null), + OceanBaseAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + List allColumns = columns.stream().map((c) -> new OceanBaseColumnReference(c, null)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(OceanBaseSelect select, OceanBaseExpression whereCondition) { + OceanBaseExpression expr = getTrueExpr(whereCondition); + + OceanBaseText asText = new OceanBaseText(expr, " as count", false); + select.setFetchColumns(List.of(asText)); + select.setSelectType(OceanBaseSelect.SelectType.ALL); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as asdf"; + } + + private enum Option { + TRUE, FALSE_NULL, NOT_NOT_TRUE, NOT_FALSE_NOT_NULL, IF, IFNULL, COALESCE + }; + + private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition) { + // we can treat "is true" as combinations of "is flase" and "not","is not true" and "not",etc. + OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE, false); + + OceanBaseUnaryPostfixOperation isFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, false); + + OceanBaseUnaryPostfixOperation isNotFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, true); + + OceanBaseUnaryPostfixOperation isNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + + OceanBaseUnaryPostfixOperation isNotNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, true); + + OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); + Option a = Randomly.fromOptions(Option.values()); + switch (a) { + case TRUE: + expr = isTrue; + break; + case FALSE_NULL: + // not((is false) or (is null)) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseBinaryLogicalOperation(isFalse, isNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_NOT_TRUE: + // not(not(is true))) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseUnaryPrefixOperation(isTrue, + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_FALSE_NOT_NULL: + // (is not false) and (is not null) + expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); + break; + case IF: + // if(1, xx is true, 0) + OceanBaseExpression[] args = new OceanBaseExpression[3]; + args[0] = OceanBaseConstant.createIntConstant(1); + args[1] = isTrue; + args[2] = OceanBaseConstant.createIntConstant(0); + expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); + break; + case IFNULL: + // ifnull(null, xx is true) + OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; + ifArgs[0] = OceanBaseConstant.createNullConstant(); + ifArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); + break; + case COALESCE: + // coalesce(null, xx is true) + OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; + coalesceArgs[0] = OceanBaseConstant.createNullConstant(); + coalesceArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); + break; + default: + expr = isTrue; + break; + } + return expr; + } } From ca6cb967e5da984645df3b4503e25c1b2d806300 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 10 Sep 2024 15:58:50 +0800 Subject: [PATCH 0951/1284] [OceanBase] Use common NoREC oracle --- src/sqlancer/oceanbase/OceanBaseErrors.java | 8 + .../oracle/OceanBaseNoRECOracle.java | 225 ++---------------- 2 files changed, 33 insertions(+), 200 deletions(-) diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index 5ad9dd3f3..c539ab129 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; import sqlancer.common.query.ExpectedErrors; @@ -25,6 +26,13 @@ public static List getExpressionErrors() { return errors; } + public static List getExpressionErrorsRegex() { + ArrayList errors = new ArrayList<>(); + errors.add(Pattern.compile("Unknown column '.+' in 'order clause'")); + + return errors; + } + public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); } diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java index cc98dcf0a..0ab0e6d2f 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java @@ -1,224 +1,49 @@ package sqlancer.oceanbase.oracle; import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.Randomly; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.OceanBaseVisitor; -import sqlancer.oceanbase.ast.OceanBaseAggregate; -import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; -import sqlancer.oceanbase.ast.OceanBaseColumnName; -import sqlancer.oceanbase.ast.OceanBaseComputableFunction; -import sqlancer.oceanbase.ast.OceanBaseComputableFunction.OceanBaseFunction; -import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseJoin; import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.ast.OceanBaseText; -import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; -import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { +public class OceanBaseNoRECOracle implements TestOracle { - // SELECT COUNT(*) FROM t0 WHERE ; - // SELECT SUM(count) FROM (SELECT IS TRUE as count FROM t0); - // SELECT (SELECT COUNT(*) FROM t0 WHERE c0 IS NOT 0) = (SELECT COUNT(*) FROM - // (SELECT c0 is NOT 0 FROM t0)); - private final OceanBaseSchema s; - private String firstQueryString; - private static final int NOT_FOUND = -1; - - private enum Option { - TRUE, FALSE_NULL, NOT_NOT_TRUE, NOT_FALSE_NOT_NULL, IF, IFNULL, COALESCE - }; + NoRECOracle oracle; public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - errors.add("is out of range"); - // regex - errors.add("unmatched parentheses"); - errors.add("nothing to repeat at offset"); - errors.add("missing )"); - errors.add("missing terminating ]"); - errors.add("range out of order in character class"); - errors.add("unrecognized character after "); - errors.add("Got error '(*VERB) not recognized or malformed"); - errors.add("must be followed by"); - errors.add("malformed number or name after"); - errors.add("digit expected after"); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()) + .with("canceling statement due to statement timeout").with("unmatched parentheses") + .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") + .with("range out of order in character class").with("unrecognized character after ") + .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") + .with("malformed number or name after").with("digit expected after").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - OceanBaseSchema.OceanBaseTable randomTable = s.getRandomTable(); - List columns = randomTable.getColumns(); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state).setColumns(columns); - OceanBaseExpression randomWhereCondition = gen.generateExpression(); - List groupBys = Collections.emptyList(); // getRandomExpressions(columns); - List tableList = Arrays.asList(randomTable).stream() - .map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList()); - int firstCount = getFirstQueryCount(tableList, randomWhereCondition, groupBys); - int secondCount = getSecondQuery(tableList, randomWhereCondition, groupBys); - if (firstCount != secondCount && firstCount != NOT_FOUND && secondCount != NOT_FOUND) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } + oracle.check(); } - private int getSecondQuery(List tableList, OceanBaseExpression randomWhereCondition, - List groupBys) throws SQLException { - OceanBaseSelect select = new OceanBaseSelect(); - select.setGroupByClause(groupBys); - OceanBaseExpression expr = getTrueExpr(randomWhereCondition); - - OceanBaseText asText = new OceanBaseText(expr, " as count", false); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(tableList); - select.setSelectType(OceanBaseSelect.SelectType.ALL); - int secondCount = 0; - - unoptimizedQueryString = "SELECT SUM(count) FROM (" + OceanBaseVisitor.asString(select) + ") as asdf"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(optimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getFirstQueryCount(List tableList, OceanBaseExpression randomWhereCondition, - List groupBys) throws SQLException { - OceanBaseSelect select = new OceanBaseSelect(); - select.setGroupByClause(groupBys); - // SELECT COUNT(t1.c3) FROM t1 WHERE (- (t1.c2)); - // SELECT SUM(count) FROM (SELECT ((- (t1.c2)) IS TRUE) as count FROM t1);; - OceanBaseAggregate aggr = new OceanBaseAggregate(new OceanBaseColumnName( - new OceanBaseSchema.OceanBaseColumn("*", OceanBaseSchema.OceanBaseDataType.INT, false, 0, false)), - OceanBaseAggregate.OceanBaseAggregateFunction.COUNT); - select.setFetchColumns(Arrays.asList(aggr)); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - select.setSelectType(OceanBaseSelect.SelectType.ALL); - int firstCount = 0; - optimizedQueryString = OceanBaseVisitor.asString(select); - SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - SQLancerResultSet rs; - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(firstQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - firstCount += rs.getLong(1); - } - rs.close(); - return firstCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition) { - // we can treat "is true" as combinations of "is flase" and "not","is not true" and "not",etc. - OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE, false); - - OceanBaseUnaryPostfixOperation isFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, false); - - OceanBaseUnaryPostfixOperation isNotFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, true); - - OceanBaseUnaryPostfixOperation isNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); - - OceanBaseUnaryPostfixOperation isNotNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, true); - - OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); - Option a = Randomly.fromOptions(Option.values()); - switch (a) { - case TRUE: - expr = isTrue; - break; - case FALSE_NULL: - // not((is false) or (is null)) - expr = new OceanBaseUnaryPrefixOperation( - new OceanBaseBinaryLogicalOperation(isFalse, isNULL, - OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), - OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case NOT_NOT_TRUE: - // not(not(is true))) - expr = new OceanBaseUnaryPrefixOperation( - new OceanBaseUnaryPrefixOperation(isTrue, - OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), - OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case NOT_FALSE_NOT_NULL: - // (is not false) and (is not null) - expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, - OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); - break; - case IF: - // if(1, xx is true, 0) - OceanBaseExpression[] args = new OceanBaseExpression[3]; - args[0] = OceanBaseConstant.createIntConstant(1); - args[1] = isTrue; - args[2] = OceanBaseConstant.createIntConstant(0); - expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); - break; - case IFNULL: - // ifnull(null, xx is true) - OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; - ifArgs[0] = OceanBaseConstant.createNullConstant(); - ifArgs[1] = isTrue; - expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); - break; - case COALESCE: - // coalesce(null, xx is true) - OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; - coalesceArgs[0] = OceanBaseConstant.createNullConstant(); - coalesceArgs[1] = isTrue; - expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); - break; - default: - expr = isTrue; - break; - } - return expr; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From fbbc86e6019321baaf36f6ebf0adb302c79e5a8b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 6 Sep 2024 14:46:57 +0800 Subject: [PATCH 0952/1284] CnosDB] Add CI & tests --- .github/workflows/main.yml | 26 ++++++++++++++++++++ src/sqlancer/cnosdb/CnosDBExpectedError.java | 5 ++++ src/sqlancer/cnosdb/CnosDBSchema.java | 2 +- test/sqlancer/dbms/TestCnosDBNoREC.java | 22 +++++++++++++++++ test/sqlancer/dbms/TestCnosDBTLP.java | 22 +++++++++++++++++ test/sqlancer/dbms/TestConfig.java | 1 + 6 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 test/sqlancer/dbms/TestCnosDBNoREC.java create mode 100644 test/sqlancer/dbms/TestCnosDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aaafeb8cd..552123501 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,6 +92,32 @@ jobs: - name: Run Tests run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test + cnosdb: + name: DBMS Tests (CnosDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up ClickHouse + run: | + docker pull cnosdb/cnosdb:community-latest + docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest + sleep 5 + - name: Run Tests + run: | + CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test + sleep 20 + CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBTLP test + clickhouse: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest diff --git a/src/sqlancer/cnosdb/CnosDBExpectedError.java b/src/sqlancer/cnosdb/CnosDBExpectedError.java index 5932bfdac..61dba101b 100644 --- a/src/sqlancer/cnosdb/CnosDBExpectedError.java +++ b/src/sqlancer/cnosdb/CnosDBExpectedError.java @@ -70,6 +70,11 @@ public static List getExpectedErrors() { errors.add( "Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed."); errors.add("Coercion from"); + + errors.add("Error parsing timestamp"); + errors.add("lpad requested length"); + errors.add("rpad requested length"); + errors.add("No function matches the given name and argument types"); return errors; } diff --git a/src/sqlancer/cnosdb/CnosDBSchema.java b/src/sqlancer/cnosdb/CnosDBSchema.java index 35d9d8887..022969ce5 100644 --- a/src/sqlancer/cnosdb/CnosDBSchema.java +++ b/src/sqlancer/cnosdb/CnosDBSchema.java @@ -38,7 +38,7 @@ public static CnosDBDataType getColumnType(String typeString) { case "bigint unsigned": case "unsigned": return CnosDBDataType.UINT; - case "timestamp": + case "timestamp(nanosecond)": return CnosDBDataType.TIMESTAMP; default: throw new AssertionError(typeString); diff --git a/test/sqlancer/dbms/TestCnosDBNoREC.java b/test/sqlancer/dbms/TestCnosDBNoREC.java new file mode 100644 index 000000000..1a89a972a --- /dev/null +++ b/test/sqlancer/dbms/TestCnosDBNoREC.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCnosDBNoREC { + + @Test + public void testCnosDBNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); + // Run with 0 queries as current implementation is resulting in database crashes + assertEquals(0, + Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", + "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", + "--oracle", "NOREC" })); + } + +} diff --git a/test/sqlancer/dbms/TestCnosDBTLP.java b/test/sqlancer/dbms/TestCnosDBTLP.java new file mode 100644 index 000000000..4b12aa409 --- /dev/null +++ b/test/sqlancer/dbms/TestCnosDBTLP.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCnosDBTLP { + + @Test + public void testCnosDBTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); + // Run with 0 queries as current implementation is resulting in database crashes + assertEquals(0, + Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", + "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", + "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 2f2c18e83..a5127900b 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -5,6 +5,7 @@ public class TestConfig { public static final String SECONDS = "300"; public static final String CLICKHOUSE_ENV = "CLICKHOUSE_AVAILABLE"; + public static final String CNOSDB_ENV = "CNOSDB_AVAILABLE"; public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; From fca4b1af286326a4c97ffa9d8e60bf55efe9e0e1 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 6 Sep 2024 13:20:50 +0800 Subject: [PATCH 0953/1284] [Presto] Add CI & tests --- .github/workflows/main.yml | 26 ++++++++++++++++++ src/sqlancer/presto/PrestoErrors.java | 27 ++++++++++++++----- src/sqlancer/presto/PrestoSchema.java | 4 ++- .../presto/gen/PrestoInsertGenerator.java | 2 +- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestPrestoNoREC.java | 17 ++++++++++++ test/sqlancer/dbms/TestPrestoTLP.java | 19 +++++++++++++ 7 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 test/sqlancer/dbms/TestPrestoNoREC.java create mode 100644 test/sqlancer/dbms/TestPrestoTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6150b8431..a34f31a7b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -457,6 +457,32 @@ jobs: POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresTLP test POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresNoREC test + presto: + name: DBMS Tests (Presto) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Set up Presto + run: | + docker pull prestodb/presto:latest + echo "connector.name=memory" >> memory.properties + docker run -p 8080:8080 -d -v ./memory.properties:/opt/presto-server/etc/catalog/memory.properties --name presto prestodb/presto:latest + sleep 30 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoNoREC test + docker restart presto && sleep 30 + PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoTLP test sqlite: name: DBMS Tests (SQLite) runs-on: ubuntu-latest diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 68a99ef9f..87eba768f 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -13,6 +13,8 @@ private PrestoErrors() { public static List getExpressionErrors() { ArrayList errors = new ArrayList<>(); + errors.addAll(getFunctionErrors()); + // Presto errors errors.add("cannot be applied to"); errors.add("LIKE expression must evaluate to a varchar"); @@ -25,6 +27,7 @@ public static List getExpressionErrors() { // -179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0000) // AS BIGINT)as count FROM t0) as res errors.add("Decimal overflow"); + errors.add("long overflow"); errors.add("multiplication overflow"); errors.add("addition overflow"); errors.add("subtraction overflow"); @@ -39,6 +42,7 @@ public static List getExpressionErrors() { // TODO: check errors.add("io.airlift.slice.Slice cannot be cast to java.lang.Number"); errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); + errors.add("Cannot cast java.lang.String to java.util.List"); errors.add("Unexpected subquery expression in logical plan"); // 9223372036854775808 @@ -51,10 +55,6 @@ public static List getExpressionErrors() { errors.add("Cannot add hour, minutes or seconds to a date"); errors.add("DECIMAL scale must be in range"); - errors.add("multiplication overflow"); - errors.add("addition overflow"); - errors.add("subtraction overflow"); - errors.add("Decimal overflow"); errors.add("IN value and list items must be the same type"); errors.add("is not a valid timestamp literal"); errors.add("Unknown time-zone ID"); @@ -63,6 +63,16 @@ public static List getExpressionErrors() { // ARRAY errors.add("Unknown type: ARRAY"); + // SELECT + errors.add("WHERE clause must evaluate to a boolean"); + errors.add("HAVING clause must evaluate to a boolean"); + errors.add("not yet implemented"); + + errors.add("Value expression and result of subquery must be of the same type for quantified comparison"); + errors.add("All IN list values must be the same type"); + errors.add("All CASE results must be the same type"); + errors.add("Mismatched types"); + return errors; } @@ -107,7 +117,12 @@ private static List getFunctionErrors() { errors.add("Could not choose a best candidate function for the function call"); // monthname errors.add("expected a numeric precision field"); // ROUND errors.add("with non-constant precision is not supported"); // ROUND - + errors.add("Unexpected parameters"); + errors.add("not registered"); + errors.add("Expected: least(E) E:orderable"); + errors.add("Expected: greatest(E) E:orderable"); + errors.add("Expected: max_by(V, K) K:orderable, V, max_by(V, K, bigint) V, K:orderable"); + errors.add("Expected: min_by(V, K) K:orderable, V, min_by(V, K, bigint) V, K:orderable"); return errors; } @@ -116,7 +131,7 @@ public static List getInsertErrors() { ArrayList errors = new ArrayList<>(); errors.addAll(getRegexErrors()); - errors.addAll(getFunctionErrors()); + errors.addAll(getExpressionErrors()); errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java index 70323a18b..112da7ef2 100644 --- a/src/sqlancer/presto/PrestoSchema.java +++ b/src/sqlancer/presto/PrestoSchema.java @@ -151,8 +151,10 @@ private static PrestoCompositeDataType getColumnType(String typeString) { primitiveType = PrestoDataType.JSON; break; case "ARRAY": + int bracesEnd = typeString.length() - 1; primitiveType = PrestoDataType.ARRAY; - break; + PrestoCompositeDataType elementType = getColumnType(typeString.substring(bracesStart + 1, bracesEnd)); + return new PrestoCompositeDataType(primitiveType, elementType); case "NULL": primitiveType = PrestoDataType.NULL; break; diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index 81e50e1d1..072a22ae0 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -16,7 +16,6 @@ public class PrestoInsertGenerator extends AbstractInsertGenerator { private final PrestoGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public PrestoInsertGenerator(PrestoGlobalState globalState) { this.globalState = globalState; @@ -36,6 +35,7 @@ private SQLQueryAdapter generate() { sb.append(")"); sb.append(" VALUES "); insertColumns(columns); + ExpectedErrors errors = new ExpectedErrors(); PrestoErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, false, false); } diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index b51309dfe..135142694 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -13,6 +13,7 @@ public class TestConfig { public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; public static final String OCEANBASE_ENV = "OCEANBASE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; + public static final String PRESTO_ENV = "PRESTO_AVAILABLE"; public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { diff --git a/test/sqlancer/dbms/TestPrestoNoREC.java b/test/sqlancer/dbms/TestPrestoNoREC.java new file mode 100644 index 000000000..e38e90483 --- /dev/null +++ b/test/sqlancer/dbms/TestPrestoNoREC.java @@ -0,0 +1,17 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPrestoNoREC { + @Test + public void testPrestoNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.PRESTO_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "presto", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestPrestoTLP.java b/test/sqlancer/dbms/TestPrestoTLP.java new file mode 100644 index 000000000..6ffefc333 --- /dev/null +++ b/test/sqlancer/dbms/TestPrestoTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPrestoTLP { + @Test + public void testPrestoTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.PRESTO_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--validate-result-size-only", + "true", "--canonicalize-sql-strings", "false", "presto", "--oracle", "QUERY_PARTITIONING" })); + } +} From d9f7cd823435539eaba604aeee2252d5f13e7445 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 9 Sep 2024 15:41:44 +0800 Subject: [PATCH 0954/1284] [Presto] Check expected error in NoREC result set --- src/sqlancer/presto/test/PrestoNoRECOracle.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java index 606183828..e9919e4d0 100644 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -96,8 +96,16 @@ private int getSecondQuery(List tableList, PrestoExpression ra if (rs == null) { return -1; } - if (rs.next()) { - secondCount += rs.getLong(1); + try { + if (rs.next()) { + secondCount += rs.getLong(1); + } + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw e; + } } rs.close(); return secondCount; From 61b80eaf429957084f92508433a8cafdc566a8cd Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 10 Sep 2024 18:20:26 +0800 Subject: [PATCH 0955/1284] [Presto] Add bug 23613 --- src/sqlancer/presto/PrestoBugs.java | 11 +++++++++++ src/sqlancer/presto/PrestoErrors.java | 4 ++++ 2 files changed, 15 insertions(+) create mode 100644 src/sqlancer/presto/PrestoBugs.java diff --git a/src/sqlancer/presto/PrestoBugs.java b/src/sqlancer/presto/PrestoBugs.java new file mode 100644 index 000000000..5e347c6f5 --- /dev/null +++ b/src/sqlancer/presto/PrestoBugs.java @@ -0,0 +1,11 @@ +package sqlancer.presto; + +public final class PrestoBugs { + + // https://github.com/prestodb/presto/issues/23613 + public static boolean bug23613 = true; + + private PrestoBugs() { + } + +} diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 87eba768f..4fcb749c3 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -73,6 +73,10 @@ public static List getExpressionErrors() { errors.add("All CASE results must be the same type"); errors.add("Mismatched types"); + if (PrestoBugs.bug23613) { + errors.add("at index 1"); + } + return errors; } From e087048f53ab749ac280c8944a124bcb6bcede5e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 10:51:47 +0800 Subject: [PATCH 0956/1284] [Presto] Add bug 23324 --- src/sqlancer/presto/PrestoBugs.java | 3 +++ src/sqlancer/presto/PrestoErrors.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/presto/PrestoBugs.java b/src/sqlancer/presto/PrestoBugs.java index 5e347c6f5..f5e888df1 100644 --- a/src/sqlancer/presto/PrestoBugs.java +++ b/src/sqlancer/presto/PrestoBugs.java @@ -2,6 +2,9 @@ public final class PrestoBugs { + // https://github.com/prestodb/presto/issues/23324 + public static boolean bug23324 = true; + // https://github.com/prestodb/presto/issues/23613 public static boolean bug23613 = true; diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 4fcb749c3..0e000119e 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -41,7 +41,9 @@ public static List getExpressionErrors() { // TODO: check errors.add("io.airlift.slice.Slice cannot be cast to java.lang.Number"); - errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); + if (PrestoBugs.bug23324) { + errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); + } errors.add("Cannot cast java.lang.String to java.util.List"); errors.add("Unexpected subquery expression in logical plan"); From c71f88e324418fa78a608836c6cdc36355cbac84 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 11:07:01 +0800 Subject: [PATCH 0957/1284] [Presto] Remove invalid timezone (America/Hermosilo) --- src/sqlancer/presto/ast/PrestoConstant.java | 44 ++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/presto/ast/PrestoConstant.java b/src/sqlancer/presto/ast/PrestoConstant.java index b1d30a1ab..a469b983c 100644 --- a/src/sqlancer/presto/ast/PrestoConstant.java +++ b/src/sqlancer/presto/ast/PrestoConstant.java @@ -39,28 +39,28 @@ public abstract class PrestoConstant implements PrestoExpression { "America/Dominica", "America/Edmonton", "America/Eirunepe", "America/El_Salvador", "America/Ensenada", "America/Fort_Nelson", "America/Fort_Wayne", "America/Fortaleza", "America/Glace_Bay", "America/Godthab", "America/Goose_Bay", "America/Grand_Turk", "America/Grenada", "America/Guadeloupe", "America/Guatemala", - "America/Guayaquil", "America/Guyana", "America/Halifax", "America/Havana", "America/Hermosillo", - "America/Indiana/Indianapolis", "America/Indiana/Knox", "America/Indiana/Marengo", - "America/Indiana/Petersburg", "America/Indiana/Tell_City", "America/Indiana/Vevay", - "America/Indiana/Vincennes", "America/Indiana/Winamac", "America/Indianapolis", "America/Inuvik", - "America/Iqaluit", "America/Jamaica", "America/Jujuy", "America/Juneau", "America/Kentucky/Louisville", - "America/Kentucky/Monticello", "America/Knox_IN", "America/Kralendijk", "America/La_Paz", "America/Lima", - "America/Los_Angeles", "America/Louisville", "America/Lower_Princes", "America/Maceio", "America/Managua", - "America/Manaus", "America/Marigot", "America/Martinique", "America/Matamoros", "America/Mendoza", - "America/Menominee", "America/Merida", "America/Metlakatla", "America/Mexico_City", "America/Miquelon", - "America/Moncton", "America/Monterrey", "America/Montevideo", "America/Montreal", "America/Montserrat", - "America/Nassau", "America/New_York", "America/Nipigon", "America/Nome", "America/Noronha", - "America/North_Dakota/Beulah", "America/North_Dakota/Center", "America/North_Dakota/New_Salem", - "America/Nuuk", "America/Ojinaga", "America/Panama", "America/Pangnirtung", "America/Paramaribo", - "America/Phoenix", "America/Port-au-Prince", "America/Port_of_Spain", "America/Porto_Acre", - "America/Porto_Velho", "America/Puerto_Rico", "America/Punta_Arenas", "America/Rainy_River", - "America/Rankin_Inlet", "America/Recife", "America/Regina", "America/Resolute", "America/Rio_Branco", - "America/Rosario", "America/Santa_Isabel", "America/Santarem", "America/Santiago", "America/Santo_Domingo", - "America/Sao_Paulo", "America/Scoresbysund", "America/Shiprock", "America/Sitka", "America/St_Barthelemy", - "America/St_Johns", "America/St_Kitts", "America/St_Lucia", "America/St_Thomas", "America/St_Vincent", - "America/Swift_Current", "America/Tegucigalpa", "America/Thule", "America/Thunder_Bay", "America/Tijuana", - "America/Toronto", "America/Tortola", "America/Vancouver", "America/Virgin", "America/Whitehorse", - "America/Winnipeg", "America/Yakutat", "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis", + "America/Guayaquil", "America/Guyana", "America/Halifax", "America/Havana", "America/Indiana/Indianapolis", + "America/Indiana/Knox", "America/Indiana/Marengo", "America/Indiana/Petersburg", + "America/Indiana/Tell_City", "America/Indiana/Vevay", "America/Indiana/Vincennes", + "America/Indiana/Winamac", "America/Indianapolis", "America/Inuvik", "America/Iqaluit", "America/Jamaica", + "America/Jujuy", "America/Juneau", "America/Kentucky/Louisville", "America/Kentucky/Monticello", + "America/Knox_IN", "America/Kralendijk", "America/La_Paz", "America/Lima", "America/Los_Angeles", + "America/Louisville", "America/Lower_Princes", "America/Maceio", "America/Managua", "America/Manaus", + "America/Marigot", "America/Martinique", "America/Matamoros", "America/Mendoza", "America/Menominee", + "America/Merida", "America/Metlakatla", "America/Mexico_City", "America/Miquelon", "America/Moncton", + "America/Monterrey", "America/Montevideo", "America/Montreal", "America/Montserrat", "America/Nassau", + "America/New_York", "America/Nipigon", "America/Nome", "America/Noronha", "America/North_Dakota/Beulah", + "America/North_Dakota/Center", "America/North_Dakota/New_Salem", "America/Nuuk", "America/Ojinaga", + "America/Panama", "America/Pangnirtung", "America/Paramaribo", "America/Phoenix", "America/Port-au-Prince", + "America/Port_of_Spain", "America/Porto_Acre", "America/Porto_Velho", "America/Puerto_Rico", + "America/Punta_Arenas", "America/Rainy_River", "America/Rankin_Inlet", "America/Recife", "America/Regina", + "America/Resolute", "America/Rio_Branco", "America/Rosario", "America/Santa_Isabel", "America/Santarem", + "America/Santiago", "America/Santo_Domingo", "America/Sao_Paulo", "America/Scoresbysund", + "America/Shiprock", "America/Sitka", "America/St_Barthelemy", "America/St_Johns", "America/St_Kitts", + "America/St_Lucia", "America/St_Thomas", "America/St_Vincent", "America/Swift_Current", + "America/Tegucigalpa", "America/Thule", "America/Thunder_Bay", "America/Tijuana", "America/Toronto", + "America/Tortola", "America/Vancouver", "America/Virgin", "America/Whitehorse", "America/Winnipeg", + "America/Yakutat", "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis", "Antarctica/DumontDUrville", "Antarctica/Macquarie", "Antarctica/Mawson", "Antarctica/McMurdo", "Antarctica/Palmer", "Antarctica/Rothera", "Antarctica/South_Pole", "Antarctica/Syowa", "Antarctica/Troll", "Antarctica/Vostok", "Arctic/Longyearbyen", "Asia/Aden", "Asia/Almaty", "Asia/Amman", "Asia/Anadyr", From 423c01ac0130fae96c3ef0bee9f4475eec0a1f5d Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 13:33:08 +0800 Subject: [PATCH 0958/1284] [HSQLDB] Implement expression generator interfaces --- src/sqlancer/hsqldb/ast/HSQLDBExpression.java | 5 +- src/sqlancer/hsqldb/ast/HSQLDBJoin.java | 51 +++------- src/sqlancer/hsqldb/ast/HSQLDBSelect.java | 26 ++++- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 98 ++++++++++++++++++- 4 files changed, 137 insertions(+), 43 deletions(-) diff --git a/src/sqlancer/hsqldb/ast/HSQLDBExpression.java b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java index b5359c11c..d066b4359 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBExpression.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java @@ -1,4 +1,7 @@ package sqlancer.hsqldb.ast; -public interface HSQLDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; + +public interface HSQLDBExpression extends Expression { } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java index 595384edd..a895e1b20 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java @@ -1,19 +1,16 @@ package sqlancer.hsqldb.ast; -import java.util.ArrayList; -import java.util.List; - import sqlancer.Randomly; -import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; -import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; +import sqlancer.common.ast.newast.Join; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; -public class HSQLDBJoin implements HSQLDBExpression { +public class HSQLDBJoin implements HSQLDBExpression, Join { private final HSQLDBTableReference leftTable; private final HSQLDBTableReference rightTable; private final JoinType joinType; - private final HSQLDBExpression onCondition; + private HSQLDBExpression onCondition; private OuterType outerType; public enum JoinType { @@ -64,37 +61,6 @@ public OuterType getOuterType() { return outerType; } - public static List getJoins(List tableList, HSQLDBGlobalState globalState) { - List joinExpressions = new ArrayList<>(); - while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - HSQLDBTableReference leftTable = tableList.remove(0); - HSQLDBTableReference rightTable = tableList.remove(0); - List columns = new ArrayList<>(leftTable.getTable().getColumns()); - columns.addAll(rightTable.getTable().getColumns()); - HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(globalState).setColumns(columns); - switch (HSQLDBJoin.JoinType.getRandom()) { - case INNER: - joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTable, rightTable, - joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - case NATURAL: - joinExpressions.add(HSQLDBJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); - break; - case LEFT: - joinExpressions.add(HSQLDBJoin.createLeftOuterJoin(leftTable, rightTable, - joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - case RIGHT: - joinExpressions.add(HSQLDBJoin.createRightOuterJoin(leftTable, rightTable, - joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - default: - throw new AssertionError(); - } - } - return joinExpressions; - } - public static HSQLDBJoin createRightOuterJoin(HSQLDBTableReference left, HSQLDBTableReference right, HSQLDBExpression predicate) { return new HSQLDBJoin(left, right, JoinType.RIGHT, predicate); @@ -110,11 +76,16 @@ public static HSQLDBJoin createInnerJoin(HSQLDBTableReference left, HSQLDBTableR return new HSQLDBJoin(left, right, JoinType.INNER, predicate); } - public static HSQLDBExpression createNaturalJoin(HSQLDBTableReference left, HSQLDBTableReference right, + public static HSQLDBJoin createNaturalJoin(HSQLDBTableReference left, HSQLDBTableReference right, OuterType naturalJoinType) { HSQLDBJoin join = new HSQLDBJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; } + @Override + public void setOnClause(HSQLDBExpression onClause) { + onCondition = onClause; + } + } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java index c75b0b88a..d58275d1e 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.hsqldb.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.HSQLDBToStringVisitor; -public class HSQLDBSelect extends SelectBase implements HSQLDBExpression { +public class HSQLDBSelect extends SelectBase + implements HSQLDBExpression, Select { private boolean isDistinct; @@ -14,4 +22,20 @@ public boolean isDistinct() { return isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (HSQLDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (HSQLDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return HSQLDBToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index b5304c890..4426c0c6c 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -7,18 +7,28 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; import sqlancer.hsqldb.ast.HSQLDBBinaryOperation; import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.ast.HSQLDBTableReference; import sqlancer.hsqldb.ast.HSQLDBUnaryPostfixOperation; import sqlancer.hsqldb.ast.HSQLDBUnaryPrefixOperation; public final class HSQLDBExpressionGenerator extends - TypedExpressionGenerator { + TypedExpressionGenerator + implements NoRECGenerator { + + List tables; private enum Expression { BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC; @@ -193,4 +203,90 @@ public List generateOrderBys() { } return expressions; } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public HSQLDBExpression generateBooleanExpression() { + return generatePredicate(); + } + + @Override + public HSQLDBSelect generateSelect() { + return new HSQLDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinExpressions = new ArrayList<>(); + while (tables.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + HSQLDBTable leftTable = tables.remove(0); + HSQLDBTable rightTable = tables.remove(0); + List columns = new ArrayList<>(leftTable.getColumns()); + columns.addAll(rightTable.getColumns()); + HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(hsqldbGlobalState).setColumns(columns); + HSQLDBTableReference leftTableRef = new HSQLDBTableReference(leftTable); + HSQLDBTableReference rightTableRef = new HSQLDBTableReference(rightTable); + switch (HSQLDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTableRef, rightTableRef, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case NATURAL: + joinExpressions.add( + HSQLDBJoin.createNaturalJoin(leftTableRef, rightTableRef, HSQLDBJoin.OuterType.getRandom())); + break; + case LEFT: + joinExpressions.add(HSQLDBJoin.createLeftOuterJoin(leftTableRef, rightTableRef, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case RIGHT: + joinExpressions.add(HSQLDBJoin.createRightOuterJoin(leftTableRef, rightTableRef, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new HSQLDBTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(HSQLDBSelect select, HSQLDBExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + HSQLDBColumn aggr = new HSQLDBColumn("COUNT(*)", null, null); + select.setFetchColumns(List.of(new HSQLDBColumnReference(aggr))); + } else { + List allColumns = columns.stream().map((c) -> new HSQLDBColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(HSQLDBSelect select, HSQLDBExpression whereCondition) { + HSQLDBColumn c = new HSQLDBColumn("COUNT(*) as count", null, null); + select.setFetchColumns(List.of(new HSQLDBColumnReference(c))); + select.setWhereClause(null); + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } From 5e8cd5d51f08cca6a0f15b965651178dbc1bfb7f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 13:34:34 +0800 Subject: [PATCH 0959/1284] [HSQLDB] Use common NoREC oracle --- .../hsqldb/test/HSQLDBNoRECOracle.java | 108 +++--------------- 1 file changed, 15 insertions(+), 93 deletions(-) diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java index e5cc786c3..0b1a3b8b8 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -1,121 +1,43 @@ package sqlancer.hsqldb.test; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.hsqldb.HSQLDBErrors; import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBCompositeDataType; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBDataType; import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; -import sqlancer.hsqldb.HSQLDBToStringVisitor; -import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBExpression; import sqlancer.hsqldb.ast.HSQLDBJoin; import sqlancer.hsqldb.ast.HSQLDBSelect; -import sqlancer.hsqldb.ast.HSQLDBTableReference; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { +public class HSQLDBNoRECOracle implements TestOracle { - private final HSQLDBSchema s; + NoRECOracle oracle; public HSQLDBNoRECOracle(HSQLDBGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - HSQLDBErrors.addExpressionErrors(errors); + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - List tables = s.getDatabaseTablesRandomSubsetNotEmpty(); - List columns = tables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList()); - HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state).setColumns(columns); - - HSQLDBExpression randomWhereCondition = gen - .generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)); - - List tableList = tables.stream().map(t -> new HSQLDBTableReference(t)) - .collect(Collectors.toList()); - List joins = HSQLDBJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); // 禁用优化 - int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getSecondQuery(List tableList, HSQLDBExpression randomWhereCondition, - List joins) throws SQLException { - HSQLDBSelect select = new HSQLDBSelect(); - HSQLDBColumn c = new HSQLDBColumn("COUNT(*)", null, null); - select.setFetchColumns(List.of(new HSQLDBColumnReference(c))); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + HSQLDBToStringVisitor.asString(select) + ") as res"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getFirstQueryCount(SQLConnection con, List tableList, List columns, - HSQLDBExpression randomWhereCondition, List joins) throws SQLException { - HSQLDBSelect select = new HSQLDBSelect(); - List allColumns = columns.stream().map((c) -> new HSQLDBColumnReference(c)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new HSQLDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = HSQLDBToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From 9a4ce4b5639c264cf226e572613e0d8a0e69b05e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 13:36:58 +0800 Subject: [PATCH 0960/1284] [HSQLDB] Update expected errors --- src/sqlancer/hsqldb/HSQLDBErrors.java | 29 +++++++++------------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/hsqldb/HSQLDBErrors.java b/src/sqlancer/hsqldb/HSQLDBErrors.java index c02f8f2b9..33995a0e1 100644 --- a/src/sqlancer/hsqldb/HSQLDBErrors.java +++ b/src/sqlancer/hsqldb/HSQLDBErrors.java @@ -12,35 +12,26 @@ private HSQLDBErrors() { public static List getExpressionErrors() { ArrayList errors = new ArrayList<>(); - errors.add(""); - return errors; - } - - public static void addExpressionErrors(ExpectedErrors errors) { - errors.addAll(getExpressionErrors()); - } - - private static List getRegexErrors() { - ArrayList errors = new ArrayList<>(); - errors.add(""); + errors.add("invalid datetime format"); + errors.add("invalid character value for cast"); + errors.add("invalid ORDER BY expression"); + errors.add("data type of expression is not boolean"); + errors.add("numeric value out of range"); + errors.add("incompatible data types in combination"); + errors.add("string data, right truncation"); return errors; } - private static List getFunctionErrors() { - ArrayList errors = new ArrayList<>(); - - errors.add(""); - - return errors; + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); } public static List getInsertErrors() { ArrayList errors = new ArrayList<>(); - errors.addAll(getRegexErrors()); - errors.addAll(getFunctionErrors()); + errors.addAll(getExpressionErrors()); return errors; } From 252cb961350ac89cd60f9f190db7f9baf2f58638 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 15:14:20 +0800 Subject: [PATCH 0961/1284] [H2] Implement TLP where expression generator interfaces --- src/sqlancer/h2/H2ExpressionGenerator.java | 54 ++++++++++++++++++- src/sqlancer/h2/H2RandomQuerySynthesizer.java | 2 +- src/sqlancer/h2/ast/H2Expression.java | 5 +- src/sqlancer/h2/ast/H2Join.java | 17 ++++-- src/sqlancer/h2/ast/H2Select.java | 26 ++++++++- 5 files changed, 95 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index d4a2240e6..3d816292c 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -1,12 +1,18 @@ package sqlancer.h2; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2CompositeDataType; import sqlancer.h2.H2Schema.H2DataType; +import sqlancer.h2.H2Schema.H2Table; import sqlancer.h2.ast.H2BetweenOperation; import sqlancer.h2.ast.H2BinaryOperation; import sqlancer.h2.ast.H2CaseOperation; @@ -15,12 +21,17 @@ import sqlancer.h2.ast.H2Constant; import sqlancer.h2.ast.H2Expression; import sqlancer.h2.ast.H2InOperation; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; +import sqlancer.h2.ast.H2TableReference; import sqlancer.h2.ast.H2UnaryPostfixOperation; import sqlancer.h2.ast.H2UnaryPrefixOperation; -public class H2ExpressionGenerator extends UntypedExpressionGenerator { +public class H2ExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { private final H2GlobalState globalState; + private List tables; public H2ExpressionGenerator(H2GlobalState globalState) { this.globalState = globalState; @@ -336,4 +347,45 @@ public H2Expression isNull(H2Expression expr) { return new H2UnaryPostfixOperation(expr, H2UnaryPostfixOperator.IS_NULL); } + @Override + public TLPWhereGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public H2Expression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public H2Select generateSelect() { + return new H2Select(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new H2TableReference(t)) + .collect(Collectors.toList()); + List joins = H2Join.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new H2TableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBoolean()) { + return List.of(new H2ColumnReference(new H2Column("*", null))); + } + return Randomly.nonEmptySubset(this.columns).stream().map(c -> new H2ColumnReference(c)) + .collect(Collectors.toList()); + } } diff --git a/src/sqlancer/h2/H2RandomQuerySynthesizer.java b/src/sqlancer/h2/H2RandomQuerySynthesizer.java index ba89bfd0b..7390b87e7 100644 --- a/src/sqlancer/h2/H2RandomQuerySynthesizer.java +++ b/src/sqlancer/h2/H2RandomQuerySynthesizer.java @@ -32,7 +32,7 @@ public static H2Select generateSelect(H2GlobalState globalState, int nrColumns) List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new H2TableReference(t)) .collect(Collectors.toList()); - List joins = H2Join.getJoins(tableList, globalState); + List joins = H2Join.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/h2/ast/H2Expression.java b/src/sqlancer/h2/ast/H2Expression.java index ed41cdfab..afd114c91 100644 --- a/src/sqlancer/h2/ast/H2Expression.java +++ b/src/sqlancer/h2/ast/H2Expression.java @@ -1,5 +1,8 @@ package sqlancer.h2.ast; -public interface H2Expression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.h2.H2Schema.H2Column; + +public interface H2Expression extends Expression { } diff --git a/src/sqlancer/h2/ast/H2Join.java b/src/sqlancer/h2/ast/H2Join.java index e777e4fc9..6aafd1de9 100644 --- a/src/sqlancer/h2/ast/H2Join.java +++ b/src/sqlancer/h2/ast/H2Join.java @@ -4,16 +4,18 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.h2.H2ExpressionGenerator; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; -public class H2Join implements H2Expression { +public class H2Join implements H2Expression, Join { private final H2TableReference leftTable; private final H2TableReference rightTable; private final JoinType joinType; - private final H2Expression onCondition; + private H2Expression onCondition; public enum JoinType { INNER, CROSS, NATURAL, LEFT, RIGHT; @@ -47,8 +49,8 @@ public H2Expression getOnCondition() { return onCondition; } - public static List getJoins(List tableList, H2GlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, H2GlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { H2TableReference leftTable = tableList.remove(0); H2TableReference rightTable = tableList.remove(0); @@ -91,8 +93,13 @@ public static H2Join createInnerJoin(H2TableReference left, H2TableReference rig return new H2Join(left, right, JoinType.INNER, predicate); } - public static H2Expression createNaturalJoin(H2TableReference left, H2TableReference right) { + public static H2Join createNaturalJoin(H2TableReference left, H2TableReference right) { return new H2Join(left, right, JoinType.NATURAL, null); } + @Override + public void setOnClause(H2Expression onClause) { + onCondition = onClause; + } + } diff --git a/src/sqlancer/h2/ast/H2Select.java b/src/sqlancer/h2/ast/H2Select.java index 9658ebca0..e2f6f8519 100644 --- a/src/sqlancer/h2/ast/H2Select.java +++ b/src/sqlancer/h2/ast/H2Select.java @@ -1,7 +1,31 @@ package sqlancer.h2.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; +import sqlancer.h2.H2ToStringVisitor; + +public class H2Select extends SelectBase + implements H2Expression, Select { + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (H2Expression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } -public class H2Select extends SelectBase implements H2Expression { + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (H2Join) e).collect(Collectors.toList()); + } + @Override + public String asString() { + return H2ToStringVisitor.asString(this); + } } From 05fb96ebdbd1875d851c0e181274993ebdb9d0b8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 15:14:48 +0800 Subject: [PATCH 0962/1284] [H2] Use common TLP where oracle --- src/sqlancer/h2/H2QueryPartitioningBase.java | 62 ------------------- .../h2/H2QueryPartitioningWhereTester.java | 47 ++++++-------- 2 files changed, 20 insertions(+), 89 deletions(-) delete mode 100644 src/sqlancer/h2/H2QueryPartitioningBase.java diff --git a/src/sqlancer/h2/H2QueryPartitioningBase.java b/src/sqlancer/h2/H2QueryPartitioningBase.java deleted file mode 100644 index 34351f58f..000000000 --- a/src/sqlancer/h2/H2QueryPartitioningBase.java +++ /dev/null @@ -1,62 +0,0 @@ -package sqlancer.h2; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.h2.H2Provider.H2GlobalState; -import sqlancer.h2.H2Schema.H2Column; -import sqlancer.h2.H2Schema.H2Table; -import sqlancer.h2.H2Schema.H2Tables; -import sqlancer.h2.ast.H2ColumnReference; -import sqlancer.h2.ast.H2Expression; -import sqlancer.h2.ast.H2Join; -import sqlancer.h2.ast.H2Select; -import sqlancer.h2.ast.H2TableReference; - -public class H2QueryPartitioningBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - H2Schema s; - H2Tables targetTables; - H2ExpressionGenerator gen; - H2Select select; - - public H2QueryPartitioningBase(H2GlobalState state) { - super(state); - H2Errors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new H2ExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new H2Select(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List tableList = tables.stream().map(t -> new H2TableReference(t)) - .collect(Collectors.toList()); - List joins = H2Join.getJoins(tableList, state); - select.setJoinList(joins.stream().collect(Collectors.toList())); - select.setFromList(tableList.stream().collect(Collectors.toList())); - select.setWhereClause(null); - } - - List generateFetchColumns() { - List columns = new ArrayList<>(); - columns.add(new H2ColumnReference(new H2Column("*", null))); - return columns; - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java index cc09dea7a..f1bb73dfa 100644 --- a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java +++ b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java @@ -1,42 +1,35 @@ package sqlancer.h2; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; -public class H2QueryPartitioningWhereTester extends H2QueryPartitioningBase { +public class H2QueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public H2QueryPartitioningWhereTester(H2GlobalState state) { - super(state); + H2ExpressionGenerator gen = new H2ExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(H2Errors.getExpressionErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = H2ToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = H2ToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = H2ToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = H2ToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } } From 07b28ca7e22017443b8a1268805b52d492dfbfed Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 15:15:29 +0800 Subject: [PATCH 0963/1284] [H2] Remove environment check in test --- .github/workflows/main.yml | 2 +- test/sqlancer/dbms/TestH2.java | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6150b8431..9434cb8e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -268,7 +268,7 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: H2_AVAILABLE=true mvn -Dtest=TestH2 test + run: mvn -Dtest=TestH2 test hsqldb: name: DBMS Tests (HSQLB) diff --git a/test/sqlancer/dbms/TestH2.java b/test/sqlancer/dbms/TestH2.java index 701f6d50b..ec6962101 100644 --- a/test/sqlancer/dbms/TestH2.java +++ b/test/sqlancer/dbms/TestH2.java @@ -1,7 +1,6 @@ package sqlancer.dbms; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; import org.junit.jupiter.api.Test; @@ -11,9 +10,6 @@ public class TestH2 { @Test public void testH2DB() { - String h2Available = System.getenv("H2_AVAILABLE"); - boolean h2DBIsAvailable = h2Available != null && h2Available.equalsIgnoreCase("true"); - assumeTrue(h2DBIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "h2" })); From 52cee37ce27fc754087869daad2df3df190aa1aa Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 16:53:34 +0800 Subject: [PATCH 0964/1284] [TiDB] Implement TLP where expression gen interface --- .../tidb/TiDBExpressionGenerator.java | 54 ++++++++++++++++++- src/sqlancer/tidb/ast/TiDBExpression.java | 5 +- src/sqlancer/tidb/ast/TiDBJoin.java | 12 +++-- src/sqlancer/tidb/ast/TiDBSelect.java | 26 ++++++++- src/sqlancer/tidb/oracle/TiDBDQPOracle.java | 2 +- 5 files changed, 92 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index a1db6fe2a..0db285284 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -3,13 +3,18 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; import sqlancer.tidb.TiDBSchema.TiDBDataType; +import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.ast.TiDBAggregate; import sqlancer.tidb.ast.TiDBAggregate.TiDBAggregateFunction; import sqlancer.tidb.ast.TiDBBinaryBitOperation; @@ -25,17 +30,22 @@ import sqlancer.tidb.ast.TiDBExpression; import sqlancer.tidb.ast.TiDBFunctionCall; import sqlancer.tidb.ast.TiDBFunctionCall.TiDBFunction; +import sqlancer.tidb.ast.TiDBJoin; import sqlancer.tidb.ast.TiDBOrderingTerm; import sqlancer.tidb.ast.TiDBRegexOperation; import sqlancer.tidb.ast.TiDBRegexOperation.TiDBRegexOperator; +import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableReference; import sqlancer.tidb.ast.TiDBUnaryPostfixOperation; import sqlancer.tidb.ast.TiDBUnaryPostfixOperation.TiDBUnaryPostfixOperator; import sqlancer.tidb.ast.TiDBUnaryPrefixOperation; import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; -public class TiDBExpressionGenerator extends UntypedExpressionGenerator { +public class TiDBExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { private final TiDBGlobalState globalState; + private List tables; public TiDBExpressionGenerator(TiDBGlobalState globalState) { this.globalState = globalState; @@ -188,4 +198,46 @@ public TiDBExpression generateConstant(TiDBDataType type) { } } + @Override + public TLPWhereGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public TiDBExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public TiDBSelect generateSelect() { + return new TiDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new TiDBTableReference(t)) + .collect(Collectors.toList()); + List joins = TiDBJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> ((TiDBTableReference) t).getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new TiDBTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBoolean()) { + return List.of(new TiDBColumnReference( + new TiDBColumn("*", new TiDBCompositeDataType(TiDBDataType.INT), false, false, false))); + } + return Randomly.nonEmptySubset(this.columns).stream().map(c -> new TiDBColumnReference(c)) + .collect(Collectors.toList()); + } } diff --git a/src/sqlancer/tidb/ast/TiDBExpression.java b/src/sqlancer/tidb/ast/TiDBExpression.java index 49ac9fb70..1f4921836 100644 --- a/src/sqlancer/tidb/ast/TiDBExpression.java +++ b/src/sqlancer/tidb/ast/TiDBExpression.java @@ -1,5 +1,8 @@ package sqlancer.tidb.ast; -public interface TiDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.tidb.TiDBSchema.TiDBColumn; + +public interface TiDBExpression extends Expression { } diff --git a/src/sqlancer/tidb/ast/TiDBJoin.java b/src/sqlancer/tidb/ast/TiDBJoin.java index f4496a736..1e9a30b63 100644 --- a/src/sqlancer/tidb/ast/TiDBJoin.java +++ b/src/sqlancer/tidb/ast/TiDBJoin.java @@ -5,11 +5,13 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBTable; -public class TiDBJoin implements TiDBExpression { +public class TiDBJoin implements TiDBExpression, Join { private final TiDBExpression leftTable; private final TiDBExpression rightTable; @@ -101,8 +103,8 @@ public NaturalJoinType getNaturalJoinType() { return outerType; } - public static List getJoins(List tableList, TiDBGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, TiDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBoolean()) { TiDBTableReference leftTable = (TiDBTableReference) tableList.remove(0); TiDBTableReference rightTable = (TiDBTableReference) tableList.remove(0); @@ -172,4 +174,8 @@ public void setOnCondition(TiDBExpression generateExpression) { this.onCondition = generateExpression; } + @Override + public void setOnClause(TiDBExpression onClause) { + onCondition = onClause; + } } diff --git a/src/sqlancer/tidb/ast/TiDBSelect.java b/src/sqlancer/tidb/ast/TiDBSelect.java index f812ee8e0..7da05da4c 100644 --- a/src/sqlancer/tidb/ast/TiDBSelect.java +++ b/src/sqlancer/tidb/ast/TiDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.tidb.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBTable; +import sqlancer.tidb.visitor.TiDBVisitor; -public class TiDBSelect extends SelectBase implements TiDBExpression { +public class TiDBSelect extends SelectBase + implements TiDBExpression, Select { private TiDBExpression hint; @@ -14,4 +22,20 @@ public TiDBExpression getHint() { return hint; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (TiDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (TiDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return TiDBVisitor.asString(this); + } } diff --git a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java index 17b288b7d..57ee4409f 100644 --- a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java @@ -47,7 +47,7 @@ public void check() throws SQLException { List tableList = tables.getTables().stream().map(t -> new TiDBTableReference(t)) .collect(Collectors.toList()); - List joins = TiDBJoin.getJoins(tableList, state); + List joins = TiDBJoin.getJoins(tableList, state).stream().collect(Collectors.toList()); select.setJoinList(joins); select.setFromList(tableList); if (Randomly.getBoolean()) { From ec567350c880e36660926631b3081e51dec6ba48 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 16:54:19 +0800 Subject: [PATCH 0965/1284] Implement reproducer for TLP-Where oracle --- .../common/oracle/TLPWhereOracle.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 5f7b3feb6..14834a62f 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -6,6 +6,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.SQLGlobalState; import sqlancer.common.ast.newast.Expression; import sqlancer.common.ast.newast.Join; @@ -25,8 +26,43 @@ public class TLPWhereOracle, J extends Join gen; private final ExpectedErrors errors; + private Reproducer reproducer; private String generatedQueryString; + private class TLPWhereReproducer implements Reproducer { + final String firstQueryString; + final String secondQueryString; + final String thirdQueryString; + final String originalQueryString; + final List resultSet; + final boolean orderBy; + + TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, + String originalQueryString, List resultSet, boolean orderBy) { + this.firstQueryString = firstQueryString; + this.secondQueryString = secondQueryString; + this.thirdQueryString = thirdQueryString; + this.originalQueryString = originalQueryString; + this.resultSet = resultSet; + this.orderBy = orderBy; + } + + @Override + public boolean bugStillTriggers(G globalState) { + try { + List combinedString1 = new ArrayList<>(); + List secondResultSet1 = ComparatorHelper.getCombinedResultSet(firstQueryString, + secondQueryString, thirdQueryString, combinedString1, !orderBy, globalState, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet1, originalQueryString, + combinedString1, globalState); + } catch (AssertionError triggeredError) { + return true; + } catch (SQLException ignored) { + } + return false; + } + } + public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -38,6 +74,7 @@ public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErr @Override public void check() throws SQLException { + reproducer = null; S s = state.getSchema(); AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(s); gen = gen.setTablesAndColumns(targetTables); @@ -75,6 +112,14 @@ public void check() throws SQLException { ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); + + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, + firstResultSet, orderBy); + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; } @Override From 7e558b54378ff0c01e6466f5512541440f60a680 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 13 Sep 2024 17:40:31 +0800 Subject: [PATCH 0966/1284] Add expected errors for NoREC oracle --- src/sqlancer/duckdb/DuckDBErrors.java | 17 +++++++++++++---- src/sqlancer/duckdb/test/DuckDBNoRECOracle.java | 3 ++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 1cc7bd96b..6e02a58f6 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; import sqlancer.common.query.ExpectedErrors; @@ -58,12 +59,20 @@ public static List getExpressionErrors() { return errors; } + public static List getExpressionErrorsRegex() { + ArrayList errors = new ArrayList<>(); + + errors.add(Pattern.compile("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause")); + errors.add(Pattern.compile("Binder Error: Cannot mix values of type .* and .* in CASE expression")); + errors.add(Pattern.compile("Cannot mix values of type .* and .* in COALESCE operator")); + errors.add(Pattern.compile("Cannot compare values of type .* and type .*")); + + return errors; + } + public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); - errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); - errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in CASE expression"); - errors.addRegexString("Cannot mix values of type .* and .* in COALESCE operator"); - errors.addRegexString("Cannot compare values of type .* and type .*"); + errors.addAllRegexes(getExpressionErrorsRegex()); } private static List getRegexErrors() { diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 4c57c0920..4096714ac 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -23,7 +23,8 @@ public class DuckDBNoRECOracle implements TestOracle { public DuckDBNoRECOracle(DuckDBGlobalState globalState) { DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); ExpectedErrors errors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) - .with("canceling statement due to statement timeout").build(); + .withRegex(DuckDBErrors.getExpressionErrorsRegex()).with("canceling statement due to statement timeout") + .build(); this.oracle = new NoRECOracle<>(globalState, gen, errors); } From cf9492ddcb803790f9572dc783b1991987bfdb4e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 16:54:59 +0800 Subject: [PATCH 0967/1284] [TiDB] Use common NoREC oracle --- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 2 +- .../tidb/oracle/TiDBTLPWhereOracle.java | 88 ++++--------------- test/sqlancer/dbms/TestConfig.java | 1 + 3 files changed, 20 insertions(+), 71 deletions(-) diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index 9e86ffd12..d23e2c4c0 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -50,7 +50,7 @@ public void check() throws SQLException { List tableList = tables.stream().map(t -> new TiDBTableReference(t)) .collect(Collectors.toList()); - List joins = TiDBJoin.getJoins(tableList, state); + List joins = TiDBJoin.getJoins(tableList, state).stream().collect(Collectors.toList()); select.setJoinList(joins); select.setFromList(tableList); select.setWhereClause(null); diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java index e437b2856..61ec16788 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java @@ -1,96 +1,44 @@ package sqlancer.tidb.oracle; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.tidb.TiDBErrors; +import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.visitor.TiDBVisitor; +import sqlancer.tidb.TiDBSchema; +import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBTable; +import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBSelect; -public class TiDBTLPWhereOracle extends TiDBTLPBase { +public class TiDBTLPWhereOracle implements TestOracle { - private String generatedQueryString; - private Reproducer reproducer; + private final TLPWhereOracle oracle; public TiDBTLPWhereOracle(TiDBGlobalState state) { - super(state); - TiDBErrors.addExpressionErrors(errors); - } - - private class TiDBTLPWhereReproducer implements Reproducer { - final String firstQueryString; - final String secondQueryString; - final String thirdQueryString; - final String originalQueryString; - final List resultSet; - final boolean orderBy; - - TiDBTLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, - String originalQueryString, List resultSet, boolean orderBy) { - this.firstQueryString = firstQueryString; - this.secondQueryString = secondQueryString; - this.thirdQueryString = thirdQueryString; - this.originalQueryString = originalQueryString; - this.resultSet = resultSet; - this.orderBy = orderBy; - } + TiDBExpressionGenerator gen = new TiDBExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); - @Override - public boolean bugStillTriggers(TiDBGlobalState globalState) { - try { - List combinedString1 = new ArrayList<>(); - List secondResultSet1 = ComparatorHelper.getCombinedResultSet(firstQueryString, - secondQueryString, thirdQueryString, combinedString1, !orderBy, globalState, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet1, originalQueryString, - combinedString1, globalState); - } catch (AssertionError triggeredError) { - return true; - } catch (SQLException ignored) { - } - return false; - } + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - reproducer = null; - super.check(); - select.setWhereClause(null); - String originalQueryString = TiDBVisitor.asString(select); - generatedQueryString = originalQueryString; - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = TiDBVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = TiDBVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = TiDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - reproducer = new TiDBTLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, - originalQueryString, resultSet, orderBy); + oracle.check(); } @Override public String getLastQueryString() { - return generatedQueryString; + return oracle.getLastQueryString(); } @Override public Reproducer getLastReproducer() { - return reproducer; + return oracle.getLastReproducer(); } - } diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 6fbf12dc7..8759ff5d5 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -15,6 +15,7 @@ public class TestConfig { public static final String OCEANBASE_ENV = "OCEANBASE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static final String PRESTO_ENV = "PRESTO_AVAILABLE"; + public static final String TIDB_ENV = "TIDB_AVAILABLE"; public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { From 21689fc78c33781c21d5088748785fadeb3f9e84 Mon Sep 17 00:00:00 2001 From: John Keith Hohm Date: Fri, 13 Sep 2024 12:13:33 -0400 Subject: [PATCH 0968/1284] Bump SQLite3 to version 3.46.1.0, recognize new bad JSON path error * bump SQLite to 3.46.1.0 (from 3.44.0.0) * recognize `bad JSON path`, the new `JSON path error` since https://sqlite.org/src/info/7f0c79b94e8f55e5 --- pom.xml | 2 +- src/sqlancer/sqlite3/SQLite3Errors.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9bee83485..16ad8c59f 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ org.xerial sqlite-jdbc - 3.40.0.0 + 3.46.1.0 mysql diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 406729bfb..96e45c549 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -56,6 +56,7 @@ public static List getExpectedExpressionErrors() { errors.add("malformed JSON"); errors.add("JSON cannot hold BLOB values"); errors.add("JSON path error"); + errors.add("bad JSON path"); errors.add("json_insert() needs an odd number of arguments"); errors.add("json_object() labels must be TEXT"); errors.add("json_object() requires an even number of arguments"); From 5bcd4ce0e4de0b34e9c1f9765843cf8d8a90377c Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 10:16:26 +0800 Subject: [PATCH 0969/1284] Update DuckDB version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 129908302..49d8a2ddf 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 1.0.0 + 1.1.0 com.facebook.presto From a330e8e82a6d3de39562c3167e5949a4e2c36705 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 11:22:17 +0800 Subject: [PATCH 0970/1284] Update expected errors --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 6e02a58f6..5141ca0b1 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -51,6 +51,7 @@ public static List getExpressionErrors() { // removed column errors.add("Contents of view were altered: types don't match!"); errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + errors.add("ORDER BY non-integer literal has no effect"); // timestamp errors.add("Cannot subtract infinite timestamps"); From 41df4fbba0d413ef5e0ec1ffd4da401189087981 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 11:22:38 +0800 Subject: [PATCH 0971/1284] Add bug 13933 --- src/sqlancer/duckdb/DuckDBBugs.java | 11 +++++++++++ src/sqlancer/duckdb/DuckDBErrors.java | 6 ++++++ 2 files changed, 17 insertions(+) create mode 100644 src/sqlancer/duckdb/DuckDBBugs.java diff --git a/src/sqlancer/duckdb/DuckDBBugs.java b/src/sqlancer/duckdb/DuckDBBugs.java new file mode 100644 index 000000000..001deec80 --- /dev/null +++ b/src/sqlancer/duckdb/DuckDBBugs.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb; + +public final class DuckDBBugs { + + // https://github.com/duckdb/duckdb/issues/13933 + public static boolean bug13933 = true; + + private DuckDBBugs() { + } + +} diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 5141ca0b1..bea136670 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -57,6 +57,12 @@ public static List getExpressionErrors() { errors.add("Cannot subtract infinite timestamps"); errors.add("Timestamp difference is out of bounds"); + if (DuckDBBugs.bug13933) { + errors.add("INTERNAL Error: Unimplemented type for normalify"); + errors.add( + "FATAL Error: Failed: database has been invalidated because of a previous fatal error. The database must be restarted prior to being used again"); + } + return errors; } From 97b47768e73d56bb98fe89f8234b093fcec2f8e0 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 15:15:40 +0800 Subject: [PATCH 0972/1284] [Presto] Implement NoREC expression generator interfaces --- src/sqlancer/presto/ast/PrestoExpression.java | 5 +- src/sqlancer/presto/ast/PrestoJoin.java | 15 +++- src/sqlancer/presto/ast/PrestoSelect.java | 26 +++++- .../gen/PrestoRandomQuerySynthesizer.java | 3 +- .../gen/PrestoTypedExpressionGenerator.java | 87 ++++++++++++++++++- .../test/PrestoQueryPartitioningBase.java | 2 +- 6 files changed, 128 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/presto/ast/PrestoExpression.java b/src/sqlancer/presto/ast/PrestoExpression.java index 143eb6823..52e0ed784 100644 --- a/src/sqlancer/presto/ast/PrestoExpression.java +++ b/src/sqlancer/presto/ast/PrestoExpression.java @@ -1,5 +1,8 @@ package sqlancer.presto.ast; -public interface PrestoExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.presto.PrestoSchema.PrestoColumn; + +public interface PrestoExpression extends Expression { } diff --git a/src/sqlancer/presto/ast/PrestoJoin.java b/src/sqlancer/presto/ast/PrestoJoin.java index 0732e55be..3fe82fc37 100644 --- a/src/sqlancer/presto/ast/PrestoJoin.java +++ b/src/sqlancer/presto/ast/PrestoJoin.java @@ -4,17 +4,19 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoJoin implements PrestoExpression { +public class PrestoJoin implements PrestoExpression, Join { private final PrestoTableReference leftTable; private final PrestoTableReference rightTable; private final JoinType joinType; - private final PrestoExpression onCondition; + private PrestoExpression onCondition; private OuterType outerType; public PrestoJoin(PrestoTableReference leftTable, PrestoTableReference rightTable, JoinType joinType, @@ -25,8 +27,8 @@ public PrestoJoin(PrestoTableReference leftTable, PrestoTableReference rightTabl this.onCondition = whereCondition; } - public static List getJoins(List tableList, PrestoGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, PrestoGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { PrestoTableReference leftTable = tableList.remove(0); PrestoTableReference rightTable = tableList.remove(0); @@ -110,4 +112,9 @@ public static OuterType getRandom() { } } + @Override + public void setOnClause(PrestoExpression onClause) { + onCondition = onClause; + } + } diff --git a/src/sqlancer/presto/ast/PrestoSelect.java b/src/sqlancer/presto/ast/PrestoSelect.java index 328c154c4..f1eb50186 100644 --- a/src/sqlancer/presto/ast/PrestoSelect.java +++ b/src/sqlancer/presto/ast/PrestoSelect.java @@ -1,8 +1,16 @@ package sqlancer.presto.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; -public class PrestoSelect extends SelectBase implements PrestoExpression { +public class PrestoSelect extends SelectBase + implements PrestoExpression, Select { private boolean isDistinct; @@ -14,4 +22,20 @@ public void setDistinct(boolean isDistinct) { this.isDistinct = isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (PrestoExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (PrestoJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return PrestoToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java index 27a54d926..5c3c0db82 100644 --- a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java +++ b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java @@ -42,7 +42,8 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new PrestoTableReference(t)) .collect(Collectors.toList()); - List joins = PrestoJoin.getJoins(tableList, globalState); + List joins = PrestoJoin.getJoins(tableList, globalState).stream() + .collect(Collectors.toList()); select.setJoinList(new ArrayList<>(joins)); select.setFromList(new ArrayList<>(tableList)); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java index 712ebdcc3..42e3b8691 100644 --- a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -9,10 +9,17 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; import sqlancer.presto.ast.PrestoAggregateFunction; import sqlancer.presto.ast.PrestoAtTimeZoneOperator; import sqlancer.presto.ast.PrestoBetweenOperation; @@ -29,6 +36,7 @@ import sqlancer.presto.ast.PrestoMultiValuedComparison; import sqlancer.presto.ast.PrestoMultiValuedComparisonOperator; import sqlancer.presto.ast.PrestoMultiValuedComparisonType; +import sqlancer.presto.ast.PrestoPostfixText; import sqlancer.presto.ast.PrestoQuantifiedComparison; import sqlancer.presto.ast.PrestoSelect; import sqlancer.presto.ast.PrestoTableReference; @@ -37,11 +45,13 @@ import sqlancer.presto.ast.PrestoUnaryPrefixOperation; public final class PrestoTypedExpressionGenerator extends - TypedExpressionGenerator { + TypedExpressionGenerator + implements NoRECGenerator { private final Randomly randomly; private final PrestoGlobalState globalState; private final int maxDepth; + private List tables; public PrestoTypedExpressionGenerator(PrestoGlobalState globalState) { this.globalState = globalState; @@ -338,7 +348,8 @@ private PrestoSelect generateSubquery(List columns) { if (Randomly.getBooleanWithSmallProbability()) { select.setOrderByClauses(typedExpressionGenerator.generateOrderBys()); } - List joins = PrestoJoin.getJoins(tableList, globalState); + List joins = PrestoJoin.getJoins(tableList, globalState).stream() + .collect(Collectors.toList()); select.setJoinList(joins); return select; } @@ -791,4 +802,76 @@ private enum Expression { BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public PrestoExpression generateBooleanExpression() { + return generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), + randomly.getInteger(0, maxDepth)); + } + + @Override + public PrestoSelect generateSelect() { + return new PrestoSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new PrestoTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(PrestoSelect select, PrestoExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + PrestoFunctionNode aggr = new PrestoFunctionNode<>( + List.of(new PrestoColumnReference(new PrestoColumn("*", + new PrestoCompositeDataType(PrestoDataType.INT, 0, 0), false, false))), + PrestoAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + + } else { + List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(PrestoSelect select, PrestoExpression whereCondition) { + PrestoExpression asText = new PrestoPostfixText( + + new PrestoCastFunction( + new PrestoPostfixText(whereCondition, + " IS NOT NULL AND " + PrestoToStringVisitor.asString(whereCondition)), + new PrestoCompositeDataType(PrestoDataType.INT, 8, 0)), + "as count"); + + select.setFetchColumns(List.of(asText)); + select.setWhereClause(null); + return "SELECT SUM(count) FROM (" + PrestoToStringVisitor.asString(select) + ") as res"; + } } diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java index 9ac6ef9f4..0e80d5c41 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java @@ -63,7 +63,7 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new PrestoTableReference(t)) .collect(Collectors.toList()); - List joins = PrestoJoin.getJoins(tableList, state); + List joins = PrestoJoin.getJoins(tableList, state).stream().collect(Collectors.toList()); select.setJoinList(new ArrayList<>(joins)); select.setFromList(new ArrayList<>(tableList)); select.setWhereClause(null); From 8dbe1f02da1167c7b6c791613bc832d0c58bf3b2 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 15:16:12 +0800 Subject: [PATCH 0973/1284] [Presto] Use common NoREC oracle --- .../presto/test/PrestoNoRECOracle.java | 131 +++--------------- 1 file changed, 16 insertions(+), 115 deletions(-) diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java index e9919e4d0..6ef5e68ce 100644 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -1,143 +1,44 @@ package sqlancer.presto.test; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoColumn; -import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; -import sqlancer.presto.PrestoSchema.PrestoDataType; import sqlancer.presto.PrestoSchema.PrestoTable; -import sqlancer.presto.PrestoSchema.PrestoTables; -import sqlancer.presto.PrestoToStringVisitor; -import sqlancer.presto.ast.PrestoCastFunction; -import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoExpression; import sqlancer.presto.ast.PrestoJoin; -import sqlancer.presto.ast.PrestoPostfixText; import sqlancer.presto.ast.PrestoSelect; -import sqlancer.presto.ast.PrestoTableReference; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoNoRECOracle extends NoRECBase implements TestOracle { +public class PrestoNoRECOracle implements TestOracle { - private final PrestoSchema s; + NoRECOracle oracle; public PrestoNoRECOracle(PrestoGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - PrestoErrors.addExpressionErrors(errors); + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - PrestoTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - - List tables = randomTables.getTables(); - - List tableList = tables.stream().map(t -> new PrestoTableReference(t)) - .collect(Collectors.toList()); - List joins = PrestoJoin.getJoins(tableList, state); - - PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(state).setColumns(columns); - PrestoExpression randomWhereCondition = gen.generatePredicate(); - int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); - - int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getSecondQuery(List tableList, PrestoExpression randomWhereCondition, - List joins) throws SQLException { - PrestoSelect select = new PrestoSelect(); - - PrestoExpression asText = new PrestoPostfixText( - - new PrestoCastFunction( - new PrestoPostfixText(randomWhereCondition, - " IS NOT NULL AND " + PrestoToStringVisitor.asString(randomWhereCondition)), - new PrestoCompositeDataType(PrestoDataType.INT, 8, 0)), - "as count"); - - select.setFetchColumns(List.of(asText)); - select.setFromList(tableList); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + PrestoToStringVisitor.asString(select) + ") as res"; - - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors, false, false); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - try { - if (rs.next()) { - secondCount += rs.getLong(1); - } - } catch (SQLException e) { - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } else { - throw e; - } - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getFirstQueryCount(SQLConnection con, List tableList, List columns, - PrestoExpression randomWhereCondition, List joins) { - PrestoSelect select = new PrestoSelect(); - List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new PrestoTypedExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = PrestoToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From b5c5d710ce1698ccaa734fe72302ecc1f87dd7dc Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 15:16:22 +0800 Subject: [PATCH 0974/1284] Update SQL adapter options for NoREC oracle --- src/sqlancer/common/oracle/NoRECOracle.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 530488296..caf3dff87 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -121,7 +121,7 @@ public Reproducer getLastReproducer() { } private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { @@ -146,7 +146,7 @@ private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState< } private int extractCounts(String queryString, ExpectedErrors errors, SQLGlobalState state) { - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { From fab83b92b14dba5896015a1266c1565beff064b7 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 17:13:03 +0800 Subject: [PATCH 0975/1284] [HSQLDB] Implement TLP-where generator interface --- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index 4426c0c6c..be73d1cb1 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -8,6 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.hsqldb.HSQLDBProvider; @@ -26,7 +27,8 @@ public final class HSQLDBExpressionGenerator extends TypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { List tables; @@ -205,8 +207,7 @@ public List generateOrderBys() { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public HSQLDBExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -289,4 +290,13 @@ public String generateUnoptimizedQueryString(HSQLDBSelect select, HSQLDBExpressi select.setWhereClause(null); return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new HSQLDBColumnReference(new HSQLDBSchema.HSQLDBColumn("*", null, null))); + } + return Randomly + .nonEmptySubset(columns.stream().map(c -> new HSQLDBColumnReference(c)).collect(Collectors.toList())); + } } From f41e801bb76217ce2f12253fae1e875d33c4d79f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 17:13:55 +0800 Subject: [PATCH 0976/1284] [HSQLDB] Use common TLP-where oracle --- .../test/HSQLDBQueryPartitioningBase.java | 72 ------------------- .../HSQLDBQueryPartitioningWhereTester.java | 53 +++++++------- 2 files changed, 25 insertions(+), 100 deletions(-) delete mode 100644 src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java deleted file mode 100644 index b428fd02f..000000000 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ /dev/null @@ -1,72 +0,0 @@ -package sqlancer.hsqldb.test; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.hsqldb.HSQLDBProvider; -import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.HSQLDBToStringVisitor; -import sqlancer.hsqldb.ast.HSQLDBColumnReference; -import sqlancer.hsqldb.ast.HSQLDBExpression; -//import sqlancer.hsqldb.ast.HSQLDBJoin; -import sqlancer.hsqldb.ast.HSQLDBSelect; -import sqlancer.hsqldb.ast.HSQLDBTableReference; -import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; - -public class HSQLDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - HSQLDBSelect select; - HSQLDBExpressionGenerator expressionGenerator; - - HSQLDBSchema schema; - - List targetTables; - - public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { - super(state); - } - - @Override - protected ExpressionGenerator getGen() { - return expressionGenerator; - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getDatabaseTablesRandomSubsetNotEmpty(); - expressionGenerator = new HSQLDBExpressionGenerator(state) - .setColumns(targetTables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList())); - initializeTernaryPredicateVariants(); - select = new HSQLDBSelect(); - select.setFetchColumns(generateFetchColumns()); - List tableList = targetTables.stream().map(t -> new HSQLDBTableReference(t)) - .collect(Collectors.toList()); - // List> joins = HSQLDBJoin.getJoins(tableList, state); - // select.setJoinList(joins.stream().collect(Collectors.toList())); - select.setFromList(tableList.stream().collect(Collectors.toList())); - select.setWhereClause(null); - - ComparatorHelper.getResultSetFirstColumnAsString(HSQLDBToStringVisitor.asString(select), errors, state); - - } - - List generateFetchColumns() { - List columns = new ArrayList<>(); - if (Randomly.getBoolean()) { - columns.add(new HSQLDBColumnReference(new HSQLDBSchema.HSQLDBColumn("*", null, null))); - } else { - columns = Randomly.nonEmptySubset(targetTables.stream().flatMap(t -> t.getColumns().stream()) - .map(c -> new HSQLDBColumnReference(c)).collect(Collectors.toList())); - } - return columns; - } -} diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java index 2b793383e..795c6656c 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java @@ -1,41 +1,38 @@ package sqlancer.hsqldb.test; -import java.util.ArrayList; -import java.util.List; +import java.sql.SQLException; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.hsqldb.HSQLDBErrors; import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; -import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -public class HSQLDBQueryPartitioningWhereTester extends HSQLDBQueryPartitioningBase { +public class HSQLDBQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public HSQLDBQueryPartitioningWhereTester(HSQLDBGlobalState state) { - super(state); + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override - public void check() throws Exception { - super.check(); - String originalQueryString = HSQLDBToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(expressionGenerator.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = HSQLDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = HSQLDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = HSQLDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + public void check() throws SQLException { + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } } From 393c531124ca5a041838ad6a5bbddffb120481aa Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 16:47:01 +0800 Subject: [PATCH 0977/1284] [MySQL] Implement TLP-where expression generator interfaces --- src/sqlancer/mysql/ast/MySQLExpression.java | 5 ++- src/sqlancer/mysql/ast/MySQLJoin.java | 4 +- src/sqlancer/mysql/ast/MySQLSelect.java | 24 +++++++++- .../mysql/gen/MySQLExpressionGenerator.java | 44 ++++++++++++++++++- 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/ast/MySQLExpression.java b/src/sqlancer/mysql/ast/MySQLExpression.java index 61a3b8aeb..1f3ae5bcb 100644 --- a/src/sqlancer/mysql/ast/MySQLExpression.java +++ b/src/sqlancer/mysql/ast/MySQLExpression.java @@ -1,6 +1,9 @@ package sqlancer.mysql.ast; -public interface MySQLExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.mysql.MySQLSchema.MySQLColumn; + +public interface MySQLExpression extends Expression { default MySQLConstant getExpectedValue() { throw new AssertionError("PQS not supported for this operator"); diff --git a/src/sqlancer/mysql/ast/MySQLJoin.java b/src/sqlancer/mysql/ast/MySQLJoin.java index 13c21b7a0..c063b4f1b 100644 --- a/src/sqlancer/mysql/ast/MySQLJoin.java +++ b/src/sqlancer/mysql/ast/MySQLJoin.java @@ -5,12 +5,13 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLExpressionGenerator; -public class MySQLJoin implements MySQLExpression { +public class MySQLJoin implements MySQLExpression, Join { public enum JoinType { NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; @@ -44,6 +45,7 @@ public JoinType getType() { return type; } + @Override public void setOnClause(MySQLExpression onClause) { this.onClause = onClause; } diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index a50007417..7b9243c20 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -2,10 +2,16 @@ import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; -public class MySQLSelect extends SelectBase implements MySQLExpression { +public class MySQLSelect extends SelectBase + implements MySQLExpression, Select { private SelectType fromOptions = SelectType.ALL; private List modifiers = Collections.emptyList(); @@ -48,4 +54,20 @@ public MySQLText getHint() { return hint; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (MySQLExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (MySQLJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return MySQLVisitor.asString(this); + } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index c3d754f58..bddf5946f 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -2,14 +2,18 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; +import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; @@ -26,17 +30,22 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; +import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; +import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; -public class MySQLExpressionGenerator extends UntypedExpressionGenerator { +public class MySQLExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; + private List tables; public MySQLExpressionGenerator(MySQLGlobalState state) { this.state = state; @@ -198,4 +207,37 @@ public List generateOrderBys() { return newOrderBys; } + @Override + public TLPWhereGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public MySQLExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public MySQLSelect generateSelect() { + return new MySQLSelect(); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new MySQLTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + return columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); + } } From 1e41016068b0b17883adfcb0490fc32f3370e60e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 16:47:23 +0800 Subject: [PATCH 0978/1284] [MySQL] Use common TLP-where oracle --- src/sqlancer/mysql/MySQLErrors.java | 2 + .../oracle/MySQLQueryPartitioningBase.java | 61 ------------------- .../mysql/oracle/MySQLTLPWhereOracle.java | 59 +++++++++--------- 3 files changed, 32 insertions(+), 90 deletions(-) delete mode 100644 src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 9dcc0a5c3..13159e49e 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -32,6 +32,8 @@ public static List getExpressionRegexErrors() { // "00000000000000000000-0" } + errors.add(Pattern.compile("Unknown column '.*' in 'order clause'")); + return errors; } diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java deleted file mode 100644 index 7b7ba8b34..000000000 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ /dev/null @@ -1,61 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mysql.MySQLErrors; -import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLSchema; -import sqlancer.mysql.MySQLSchema.MySQLTable; -import sqlancer.mysql.MySQLSchema.MySQLTables; -import sqlancer.mysql.ast.MySQLColumnReference; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLSelect; -import sqlancer.mysql.ast.MySQLTableReference; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -public abstract class MySQLQueryPartitioningBase extends - TernaryLogicPartitioningOracleBase implements TestOracle { - - MySQLSchema s; - MySQLTables targetTables; - MySQLExpressionGenerator gen; - MySQLSelect select; - - public MySQLQueryPartitioningBase(MySQLGlobalState state) { - super(state); - MySQLErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new MySQLExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new MySQLSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List tableList = tables.stream().map(t -> new MySQLTableReference(t)) - .collect(Collectors.toList()); - // List joins = MySQLJoin.getJoins(tableList, state); - select.setFromList(tableList); - select.setWhereClause(null); - // select.setJoins(joins); - } - - List generateFetchColumns() { - return Arrays.asList(MySQLColumnReference.create(targetTables.getColumns().get(0), null)); - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java index 3d4de3873..6d0fd1178 100644 --- a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java @@ -1,44 +1,45 @@ package sqlancer.mysql.oracle; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.MySQLSchema; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLJoin; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.gen.MySQLExpressionGenerator; -public class MySQLTLPWhereOracle extends MySQLQueryPartitioningBase { +public class MySQLTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public MySQLTLPWhereOracle(MySQLGlobalState state) { - super(state); + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = MySQLVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - if (Randomly.getBoolean()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = MySQLVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = MySQLVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = MySQLVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From 1fef4a3b4c313548b52cd573f4a42d4d31ff0be8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 17:48:46 +0800 Subject: [PATCH 0979/1284] [MySQL] Refactor environment check in test --- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestMySQLPQS.java | 5 +---- test/sqlancer/dbms/TestMySQLTLP.java | 5 +---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 8759ff5d5..92de68fe9 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -12,6 +12,7 @@ public class TestConfig { public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; + public static final String MYSQL_ENV = "MYSQL_AVAILABLE"; public static final String OCEANBASE_ENV = "OCEANBASE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static final String PRESTO_ENV = "PRESTO_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestMySQLPQS.java b/test/sqlancer/dbms/TestMySQLPQS.java index ba6a9c501..6f1b0786f 100644 --- a/test/sqlancer/dbms/TestMySQLPQS.java +++ b/test/sqlancer/dbms/TestMySQLPQS.java @@ -9,12 +9,9 @@ public class TestMySQLPQS { - String mysqlAvailable = System.getenv("MYSQL_AVAILABLE"); - boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); - @Test public void testPQS() { - assumeTrue(mysqlIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC", "--database-prefix", diff --git a/test/sqlancer/dbms/TestMySQLTLP.java b/test/sqlancer/dbms/TestMySQLTLP.java index 3d3e7d107..6254c58f3 100644 --- a/test/sqlancer/dbms/TestMySQLTLP.java +++ b/test/sqlancer/dbms/TestMySQLTLP.java @@ -9,12 +9,9 @@ public class TestMySQLTLP { - String mysqlAvailable = System.getenv("MYSQL_AVAILABLE"); - boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); - @Test public void testMySQL() { - assumeTrue(mysqlIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, From 045117e4a80ccbc9e0777b3c870a6dd2f614b5ad Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 18:07:40 +0800 Subject: [PATCH 0980/1284] [DataFusion] Implement TLP-where expression generator interface --- .../gen/DataFusionExpressionGenerator.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java index d56493fa2..520043fa1 100644 --- a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java @@ -14,6 +14,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; @@ -35,7 +36,8 @@ public final class DataFusionExpressionGenerator extends TypedExpressionGenerator implements - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private List tables; private final DataFusionGlobalState globalState; @@ -234,8 +236,7 @@ public String getTextRepresentation() { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public DataFusionExpressionGenerator setTablesAndColumns(AbstractTables tables) { List randomTables = Randomly.nonEmptySubset(tables.getTables()); int maxSize = Randomly.fromOptions(1, 2, 3, 4); if (randomTables.size() > maxSize) { @@ -295,4 +296,10 @@ public String generateUnoptimizedQueryString(DataFusionSelect select, DataFusion return select.asString(); } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + List randomColumns = DataFusionTable.getRandomColumns(tables); + return randomColumns.stream().map((c) -> new DataFusionColumnReference(c)).collect(Collectors.toList()); + } } From 848f96c1bea253b5cbb0c5bceb5d9e130e4da5c9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 18:07:58 +0800 Subject: [PATCH 0981/1284] [DataFusion] Use common TLP-where oracle --- .../datafusion/ast/DataFusionSelect.java | 39 ---------- .../test/DataFusionQueryPartitioningBase.java | 37 ---------- ...ataFusionQueryPartitioningWhereTester.java | 74 ++++++------------- 3 files changed, 23 insertions(+), 127 deletions(-) delete mode 100644 src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java diff --git a/src/sqlancer/datafusion/ast/DataFusionSelect.java b/src/sqlancer/datafusion/ast/DataFusionSelect.java index 10cf4859f..75bbb26ae 100644 --- a/src/sqlancer/datafusion/ast/DataFusionSelect.java +++ b/src/sqlancer/datafusion/ast/DataFusionSelect.java @@ -4,54 +4,15 @@ import java.util.Optional; import java.util.stream.Collectors; -import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; import sqlancer.common.ast.newast.Select; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionSchema; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionTable; import sqlancer.datafusion.DataFusionToStringVisitor; -import sqlancer.datafusion.gen.DataFusionExpressionGenerator; public class DataFusionSelect extends SelectBase implements DataFusionExpression, Select { public Optional fetchColumnsString = Optional.empty(); // When available, override `fetchColumns` in base - // class's `Node` representation (for display) - public DataFusionExpressionGenerator exprGen; - - // Construct a `DataFusionSelect` with random SELECT, FROM, WHERE - public static DataFusionSelect getRandomSelect(DataFusionGlobalState state) { - DataFusionSelect randomSelect = new DataFusionSelect(); - - // Randomly pick up to 4 tables to select from - DataFusionSchema schema = state.getSchema(); // schema of all tables - List allTables = schema.getDatabaseTables(); - List randomTables = Randomly.nonEmptySubset(allTables); - int maxSize = Randomly.fromOptions(1, 2, 3, 4); - if (randomTables.size() > maxSize) { - randomTables = randomTables.subList(0, maxSize); - } - - // Randomly choose some columns from `randomTables` - // And generate a random expression which might contain those columns - List randomColumns = DataFusionTable.getRandomColumns(randomTables); - randomSelect.exprGen = new DataFusionExpressionGenerator(state).setColumns(randomColumns); - DataFusionExpression whereExpr = randomSelect.exprGen - .generateExpression(DataFusionSchema.DataFusionDataType.BOOLEAN); - - // Constructing result - List randomTableNodes = randomTables.stream().map(t -> new DataFusionTableReference(t)) - .collect(Collectors.toList()); - List randomColumnNodes = randomColumns.stream() - .map((c) -> new DataFusionColumnReference(c)).collect(Collectors.toList()); - - randomSelect.setFetchColumns(randomColumnNodes); - randomSelect.setFromList(randomTableNodes); - randomSelect.setWhereClause(whereExpr); - - return randomSelect; - } /* * If set fetch columns with string It will override `fetchColumns` in base class when diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java deleted file mode 100644 index 067b07677..000000000 --- a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java +++ /dev/null @@ -1,37 +0,0 @@ -package sqlancer.datafusion.test; - -import java.sql.SQLException; - -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.ast.DataFusionExpression; -import sqlancer.datafusion.ast.DataFusionSelect; -import sqlancer.datafusion.gen.DataFusionExpressionGenerator; - -public class DataFusionQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase - implements TestOracle { - DataFusionGlobalState state; - DataFusionExpressionGenerator gen; - DataFusionSelect select; - - public DataFusionQueryPartitioningBase(DataFusionGlobalState state) { - super(state); - this.state = state; - } - - @Override - public void check() throws SQLException { - select = DataFusionSelect.getRandomSelect(state); - gen = select.exprGen; - initializeTernaryPredicateVariants(); - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java index 3235c3d69..e0335a18f 100644 --- a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java +++ b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java @@ -1,67 +1,39 @@ package sqlancer.datafusion.test; -import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.ERROR; - import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.datafusion.DataFusionErrors; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionToStringVisitor; -import sqlancer.datafusion.DataFusionUtil; +import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionJoin; import sqlancer.datafusion.ast.DataFusionSelect; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +public class DataFusionQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; -public class DataFusionQueryPartitioningWhereTester extends DataFusionQueryPartitioningBase { public DataFusionQueryPartitioningWhereTester(DataFusionGlobalState state) { - super(state); - DataFusionErrors.registerExpectedExecutionErrors(errors); + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) + .build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } - /* - * Query Partitioning - Where q: SELECT [expr1] FROM [expr2] qp1: SELECT [expr1] FROM [expr2] WHERE [expr3] qp2: - * SELECT [expr1] FROM [expr2] WHERE NOT [expr3] qp3: SELECT [expr1] FROM [expr2] WHERE [expr3] IS NULL - * - * Oracle check: q's result equals to union(qp1, qp2, qp3) - */ @Override public void check() throws SQLException { - // generate a random 'SELECT [expr1] FROM [expr2] WHERE [expr3] - super.check(); - DataFusionSelect randomSelect = select; - randomSelect.setWhereClause(null); - - // Construct q - String qString = DataFusionToStringVisitor.asString(randomSelect); - // Construct qp1, qp2, qp3 - randomSelect.setWhereClause(predicate); - String qp1String = DataFusionToStringVisitor.asString(randomSelect); - randomSelect.setWhereClause(negatedPredicate); - String qp2String = DataFusionToStringVisitor.asString(randomSelect); - randomSelect.setWhereClause(isNullPredicate); - String qp3String = DataFusionToStringVisitor.asString(randomSelect); - - try { - /* - * Run all queires - */ - List qResultSet = ComparatorHelper.getResultSetFirstColumnAsString(qString, errors, state); - List combinedString = new ArrayList<>(); - List qpResultSet = ComparatorHelper.getCombinedResultSet(qp1String, qp2String, qp3String, - combinedString, true, state, errors); - /* - * Query Partitioning-Where check - */ - ComparatorHelper.assumeResultSetsAreEqual(qResultSet, qpResultSet, qString, combinedString, state, - ComparatorHelper::canonicalizeResultValue); - } catch (AssertionError e) { - // Append more error message - String replay = DataFusionUtil.getReplay(state.getDatabaseName()); - String newMessage = e.getMessage() + "\n" + e.getCause() + "\n" + replay + "\n"; - state.dfLogger.appendToLog(ERROR, newMessage); + oracle.check(); + } - throw new AssertionError(newMessage); - } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From df714f1df7a119c2a479261aeb0ef37eee8122e3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:09:11 +0800 Subject: [PATCH 0982/1284] [OceanBase] Implement TLP-where expression generator interface --- .../oceanbase/gen/OceanBaseExpressionGenerator.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java index 3c17e5084..42b144c93 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java @@ -7,6 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.oceanbase.OceanBaseGlobalState; @@ -40,8 +41,8 @@ import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator - implements - NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private OceanBaseGlobalState state; private OceanBaseRowValue rowVal; @@ -225,8 +226,7 @@ public OceanBaseExpression isNull(OceanBaseExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public OceanBaseExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -360,4 +360,9 @@ private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition } return expr; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + return columns.stream().map(c -> new OceanBaseColumnReference(c, null)).collect(Collectors.toList()); + } } From 6709c8a8b30507630a4772c053a484ced79f865d Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:09:37 +0800 Subject: [PATCH 0983/1284] [OceanBase] Use common TLP-where oracle --- .../oceanbase/oracle/OceanBaseTLPBase.java | 63 ------------------- .../oracle/OceanBaseTLPWhereOracle.java | 59 ++++++++--------- 2 files changed, 30 insertions(+), 92 deletions(-) delete mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java deleted file mode 100644 index 73ff19bfb..000000000 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java +++ /dev/null @@ -1,63 +0,0 @@ -package sqlancer.oceanbase.oracle; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.oceanbase.OceanBaseErrors; -import sqlancer.oceanbase.OceanBaseGlobalState; -import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTables; -import sqlancer.oceanbase.ast.OceanBaseColumnReference; -import sqlancer.oceanbase.ast.OceanBaseExpression; -import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -import sqlancer.oceanbase.gen.OceanBaseHintGenerator; - -public abstract class OceanBaseTLPBase - extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - OceanBaseSchema s; - OceanBaseTables targetTables; - OceanBaseExpressionGenerator gen; - OceanBaseSelect select; - - public OceanBaseTLPBase(OceanBaseGlobalState state) { - super(state); - OceanBaseErrors.addExpressionErrors(errors); - errors.add("value is out of range"); - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new OceanBaseExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new OceanBaseSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - OceanBaseHintGenerator.generateHints(select, tables); - List tableList = tables.stream().map(t -> new OceanBaseTableReference(t)) - .collect(Collectors.toList()); - select.setFromList(tableList); - select.setWhereClause(null); - } - - List generateFetchColumns() { - return Arrays.asList(OceanBaseColumnReference.create(targetTables.getColumns().get(0), null)); - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java index 7c8c64e3c..08f5fef93 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java @@ -1,44 +1,45 @@ package sqlancer.oceanbase.oracle; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; -import sqlancer.oceanbase.OceanBaseVisitor; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseJoin; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -public class OceanBaseTLPWhereOracle extends OceanBaseTLPBase { +public class OceanBaseTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public OceanBaseTLPWhereOracle(OceanBaseGlobalState state) { - super(state); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).with("value is out of range").build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = OceanBaseVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - if (Randomly.getBoolean()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = OceanBaseVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = OceanBaseVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = OceanBaseVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From 276c2b7a1818ffe0f58c935125ca33031ec1ddb4 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:33:28 +0800 Subject: [PATCH 0984/1284] [ClickHouse] Implemenet TLP-where expression generator interface --- .../gen/ClickHouseExpressionGenerator.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index b523703a3..4e04e91f4 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.IntStream; import com.clickhouse.client.ClickHouseDataType; @@ -31,12 +32,14 @@ import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; public class ClickHouseExpressionGenerator extends TypedExpressionGenerator implements - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private final ClickHouseGlobalState globalState; public boolean allowAggregateFunctions; @@ -356,8 +359,7 @@ public ClickHouseExpression isNull(ClickHouseExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public ClickHouseExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.tables = tables.getTables(); this.columns = tables.getColumns(); return this; @@ -434,4 +436,15 @@ public String generateUnoptimizedQueryString(ClickHouseSelect select, ClickHouse select.setWhereClause(null); return "SELECT SUM(check <> 0) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new ClickHouseColumnReference(ClickHouseColumn.createDummy("*", null), null, null)); + } + List columnReferences = columns.stream() + .map(c -> c.asColumnReference(c.getTable().getName())).collect(Collectors.toList()); + return IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> generateExpressionWithColumns(columnReferences, 5)).collect(Collectors.toList()); + } } From 33707784baed2cc113c29b56e306fb51f0716a11 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:33:45 +0800 Subject: [PATCH 0985/1284] [ClickHouse] Use common TLP-where oracle --- .../oracle/tlp/ClickHouseTLPWhereOracle.java | 69 +++++++++---------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java index be2bd71fc..564db22f7 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java @@ -1,50 +1,45 @@ package sqlancer.clickhouse.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.clickhouse.ClickHouseErrors; -import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; +import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; -public class ClickHouseTLPWhereOracle extends ClickHouseTLPBase { +public class ClickHouseTLPWhereOracle implements TestOracle { - public ClickHouseTLPWhereOracle(ClickHouseProvider.ClickHouseGlobalState state) { - super(state); - ClickHouseErrors.addExpectedExpressionErrors(errors); + private final TLPWhereOracle oracle; + + public ClickHouseTLPWhereOracle(ClickHouseGlobalState state) { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) + .build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) - .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); - } - String originalQueryString = ClickHouseVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) - .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); - } - - select.setWhereClause(predicate); - String firstQueryString = ClickHouseVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = ClickHouseVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = ClickHouseVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From 84278c8f17fad4390f0f2d999ca84b674b03aefc Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:48:33 +0800 Subject: [PATCH 0986/1284] [CockroachDB] Implement TLP-where expression generator interface --- .../gen/CockroachDBExpressionGenerator.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index d36e7673c..189be028d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -45,12 +45,14 @@ import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; public class CockroachDBExpressionGenerator extends TypedExpressionGenerator implements - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private List tables; private final CockroachDBGlobalState globalState; @@ -369,7 +371,7 @@ public CockroachDBExpression isNull(CockroachDBExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( + public CockroachDBExpressionGenerator setTablesAndColumns( AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -447,4 +449,13 @@ public String generateUnoptimizedQueryString(CockroachDBSelect select, Cockroach + " IS TRUE AS INT) as count FROM " + fromString + " " + joinList.stream().map(j -> CockroachDBVisitor.asString(j)).collect(Collectors.joining(", ")) + ")"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy || columns.size() == 0) { + return List.of(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new CockroachDBColumnReference(c)) + .collect(Collectors.toList()); + } } From e6d31efbd120f156a8c3055a108f2a65da98af62 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:48:58 +0800 Subject: [PATCH 0987/1284] [CockroachdB] Use common TLP-where oracle --- .../oracle/tlp/CockroachDBTLPWhereOracle.java | 60 ++++++++----------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java index 0fe935d18..5fc14c492 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java @@ -1,55 +1,45 @@ package sqlancer.cockroachdb.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.cockroachdb.CockroachDBSchema; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBNotOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; +import sqlancer.cockroachdb.ast.CockroachDBJoin; +import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; -public class CockroachDBTLPWhereOracle extends CockroachDBTLPBase { +public class CockroachDBTLPWhereOracle implements TestOracle { - private String generatedQueryString; + private final TLPWhereOracle oracle; public CockroachDBTLPWhereOracle(CockroachDBGlobalState state) { - super(state); - errors.add("GROUP BY term out of range"); + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with("GROUP BY term out of range").build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - String originalQueryString = CockroachDBVisitor.asString(select); - generatedQueryString = originalQueryString; - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean allowOrderBy = Randomly.getBoolean(); - if (allowOrderBy) { - select.setOrderByClauses(gen.getOrderingTerms()); - } - CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); - select.setWhereClause(predicate); - String firstQueryString = CockroachDBVisitor.asString(select); - select.setWhereClause(new CockroachDBNotOperation(predicate)); - String secondQueryString = CockroachDBVisitor.asString(select); - select.setWhereClause(new CockroachDBUnaryPostfixOperation(predicate, CockroachDBUnaryPostfixOperator.IS_NULL)); - String thirdQueryString = CockroachDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !allowOrderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } @Override public String getLastQueryString() { - return generatedQueryString; + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From 6f0a07ed46a2302ee1a43b859c9d976a9e66d116 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:15:57 +0800 Subject: [PATCH 0988/1284] [Databend] Implement TLP-where expression generator interface --- .../gen/DatabendNewExpressionGenerator.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index a3b7e996e..20901bc1b 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -10,6 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.databend.DatabendBugs; @@ -48,7 +49,8 @@ public class DatabendNewExpressionGenerator extends TypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private final DatabendGlobalState globalState; private List tables; @@ -355,8 +357,7 @@ public DatabendExpression generateHavingClause() { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public DatabendNewExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -421,4 +422,11 @@ public String generateUnoptimizedQueryString(DatabendSelect select, DatabendExpr return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new DatabendColumnReference(new DatabendColumn("*", null, false, false))); + } + return columns.stream().map(c -> new DatabendColumnReference(c)).collect(Collectors.toList()); + } } From 98d40769b4932e9c9284ee48afa0307271e336b0 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:16:18 +0800 Subject: [PATCH 0989/1284] [Databend] Use common TLP-where oracle --- .../DatabendQueryPartitioningWhereTester.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index 6ea6f5e57..b6520e7f2 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -1,45 +1,45 @@ package sqlancer.databend.test.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendQueryPartitioningWhereTester extends DatabendQueryPartitioningBase { +public class DatabendQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public DatabendQueryPartitioningWhereTester(DatabendGlobalState state) { - super(state); - DatabendErrors.addGroupByErrors(errors); + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with(DatabendErrors.getGroupByErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = DatabendToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - // boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - boolean orderBy = false; - // if (orderBy) { //TODO 待开启 - // select.setOrderByClauses(gen.generateOrderBys()); - // } - select.setWhereClause(predicate); - String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = DatabendToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From eadad7b6784f5ac5aa66ef17c36ef91629c4fcf9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:25:14 +0800 Subject: [PATCH 0990/1284] [Doris] Implement TLP-where expression generator interfaces --- .../doris/gen/DorisNewExpressionGenerator.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java index 7cea29f31..bddc6cc5a 100644 --- a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -11,6 +11,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.doris.DorisBugs; @@ -50,7 +51,8 @@ import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisNewExpressionGenerator extends TypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private final DorisGlobalState globalState; private List tables; @@ -463,8 +465,7 @@ public void setAllowAggregateFunctions(boolean allowAggregateFunctions) { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public DorisNewExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -532,4 +533,13 @@ public String generateUnoptimizedQueryString(DorisSelect select, DorisExpression return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new DorisColumnReference(new DorisColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new DorisColumnReference(c)) + .collect(Collectors.toList()); + } } From 37c4963d340a58df013d87504ee0f3d2f44daace Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:25:35 +0800 Subject: [PATCH 0991/1284] [Doris] Use common TLP-where oracle --- .../DorisQueryPartitioningWhereTester.java | 62 +++++++++---------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java index 5a593a573..32b171af0 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -1,51 +1,45 @@ package sqlancer.doris.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisToStringVisitor; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.gen.DorisNewExpressionGenerator; -public class DorisQueryPartitioningWhereTester extends DorisQueryPartitioningBase { +public class DorisQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public DorisQueryPartitioningWhereTester(DorisGlobalState state) { - super(state); - DorisErrors.addExpressionErrors(errors); - DorisErrors.addInsertErrors(errors); + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with(DorisErrors.getExpressionErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = DorisToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - List constants = new ArrayList<>(); - constants.add( - new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByClauses(constants); - } - select.setWhereClause(predicate); - String firstQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = DorisToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From 8a32ad7ed2116113953fcbf8caebd338183a4c6f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:35:20 +0800 Subject: [PATCH 0992/1284] [DuckDB] Implement TLP-where expression generator interface --- .../duckdb/gen/DuckDBExpressionGenerator.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 3f6abc37f..278685a46 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -11,6 +11,7 @@ import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -35,7 +36,8 @@ import sqlancer.duckdb.ast.DuckDBTernary; public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private final DuckDBGlobalState globalState; private List tables; @@ -449,8 +451,7 @@ public DuckDBExpression isNull(DuckDBExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public DuckDBExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -513,4 +514,13 @@ public String generateUnoptimizedQueryString(DuckDBSelect select, DuckDBExpressi return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (Randomly.getBoolean()) { + return List.of(new DuckDBColumnReference(new DuckDBColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); + } } From 85281e2b1ff840446d89571c17ce0ef3186450da Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:36:03 +0800 Subject: [PATCH 0993/1284] [DuckDB] Use common TLP-where oracle --- .../DuckDBQueryPartitioningWhereTester.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java index 7e20e605c..d4ffa74c8 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java @@ -1,45 +1,45 @@ package sqlancer.duckdb.test; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.DuckDBSchema; +import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; +import sqlancer.duckdb.DuckDBSchema.DuckDBTable; +import sqlancer.duckdb.ast.DuckDBExpression; +import sqlancer.duckdb.ast.DuckDBJoin; +import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -public class DuckDBQueryPartitioningWhereTester extends DuckDBQueryPartitioningBase { +public class DuckDBQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public DuckDBQueryPartitioningWhereTester(DuckDBGlobalState state) { - super(state); - DuckDBErrors.addGroupByErrors(errors); + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .with(DuckDBErrors.getGroupByErrors()).withRegex(DuckDBErrors.getExpressionErrorsRegex()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = DuckDBToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = DuckDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = DuckDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = DuckDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From e486f85fbc6a7131d1eca35f2df3c69a8b8da869 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:51:20 +0800 Subject: [PATCH 0994/1284] [Materialize] Implement TLP-where expression generator interface --- .../gen/MaterializeExpressionGenerator.java | 24 +++++++++++++++---- .../gen/MaterializeRandomQueryGenerator.java | 2 +- ...aterializePivotedQuerySynthesisOracle.java | 2 +- .../tlp/MaterializeTLPAggregateOracle.java | 2 +- .../oracle/tlp/MaterializeTLPBase.java | 2 +- .../tlp/MaterializeTLPHavingOracle.java | 2 +- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java index f3af50038..d61e452f8 100644 --- a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -12,6 +12,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.materialize.MaterializeCompoundDataType; import sqlancer.materialize.MaterializeGlobalState; @@ -59,7 +60,8 @@ import sqlancer.materialize.oracle.tlp.MaterializeTLPBase; public class MaterializeExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private final int maxDepth; @@ -103,7 +105,8 @@ public MaterializeExpression generateExpression(int depth) { return generateExpression(depth, MaterializeDataType.getRandomType()); } - public List generateOrderBy() { + @Override + public List generateOrderBys() { List orderBys = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber(); i++) { orderBys.add(new MaterializeOrderByTerm(MaterializeColumnValue.create(Randomly.fromList(columns), null), @@ -530,7 +533,7 @@ public MaterializeExpression isNull(MaterializeExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( + public MaterializeExpressionGenerator setTablesAndColumns( AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -593,7 +596,7 @@ public String generateOptimizedQueryString(MaterializeSelect select, Materialize MaterializeColumnValue allColumns = new MaterializeColumnValue(Randomly.fromList(columns), null); select.setFetchColumns(List.of(allColumns)); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(generateOrderBy()); + select.setOrderByClauses(generateOrderBys()); } select.setSelectType(SelectType.ALL); } @@ -612,4 +615,17 @@ public String generateUnoptimizedQueryString(MaterializeSelect select, Materiali return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new MaterializeColumnValue(MaterializeColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(columns); + for (MaterializeColumn c : targetColumns) { + fetchColumns.add(new MaterializeColumnValue(c, null)); + } + return fetchColumns; + } } diff --git a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java index c8b1441fc..05d908e05 100644 --- a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java @@ -46,7 +46,7 @@ public static MaterializeSelect createRandomQuery(int nrColumns, MaterializeGlob } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java index f4b2a1cc6..50dff2b49 100644 --- a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java +++ b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java @@ -63,7 +63,7 @@ public SQLQueryAdapter getRectifiedQuery() throws SQLException { selectStatement.setOffsetClause(offsetClause); } List orderBy = new MaterializeExpressionGenerator(globalState).setColumns(columns) - .generateOrderBy(); + .generateOrderBys(); selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(MaterializeVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java index e8154bb9a..6bbd7d795 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java @@ -63,7 +63,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = MaterializeVisitor.asString(select); generatedQueryString = originalQuery; diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java index 36711997a..faf136c21 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java @@ -127,7 +127,7 @@ public static MaterializeSubquery createSubquery(MaterializeGlobalState globalSt select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java index e4328ce40..deefb4fb9 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java @@ -49,7 +49,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setHavingClause(predicate); String firstQueryString = MaterializeVisitor.asString(select); From a736f3233a4969c530378f4530433040078dee96 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:51:47 +0800 Subject: [PATCH 0995/1284] [Materialize] Use common TLP-where oracle --- .../oracle/tlp/MaterializeTLPWhereOracle.java | 59 ++++++++----------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java index 6cec373a9..cd14bfb75 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java @@ -1,52 +1,45 @@ package sqlancer.materialize.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.materialize.MaterializeGlobalState; -import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; -public class MaterializeTLPWhereOracle extends MaterializeTLPBase { - private String generatedQueryString; +public class MaterializeTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public MaterializeTLPWhereOracle(MaterializeGlobalState state) { - super(state); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) + .with(MaterializeCommon.getCommonFetchErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - whereCheck(); + oracle.check(); } - protected void whereCheck() throws SQLException { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - String originalQueryString = MaterializeVisitor.asString(select); - generatedQueryString = originalQueryString; - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = MaterializeVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = MaterializeVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = MaterializeVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } @Override - public String getLastQueryString() { - return generatedQueryString; + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From 2dd43a25e6c600401bee04a0469f6434a5724c0f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:58:47 +0800 Subject: [PATCH 0996/1284] [Presto] Implement TLP-where expression generator interfaces --- .../gen/PrestoTypedExpressionGenerator.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java index 42e3b8691..4a788f01b 100644 --- a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -10,6 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; @@ -46,7 +47,8 @@ public final class PrestoTypedExpressionGenerator extends TypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private final Randomly randomly; private final PrestoGlobalState globalState; @@ -803,8 +805,7 @@ private enum Expression { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public PrestoTypedExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -874,4 +875,13 @@ public String generateUnoptimizedQueryString(PrestoSelect select, PrestoExpressi select.setWhereClause(null); return "SELECT SUM(count) FROM (" + PrestoToStringVisitor.asString(select) + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (Randomly.getBoolean()) { + return List.of(new PrestoColumnReference(new PrestoColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); + } } From 3d3bc2c128b29e7fd0031ecb32d906a4f96f61d3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:59:04 +0800 Subject: [PATCH 0997/1284] [Presto] Use common TLP-where oracle --- .../PrestoQueryPartitioningWhereTester.java | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java index cfff823c5..1fb2b7018 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java @@ -1,46 +1,45 @@ package sqlancer.presto.test; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; -import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoQueryPartitioningWhereTester extends PrestoQueryPartitioningBase { +public class PrestoQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public PrestoQueryPartitioningWhereTester(PrestoGlobalState state) { - super(state); - PrestoErrors.addGroupByErrors(errors); - PrestoErrors.addExpressionErrors(errors); + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) + .with(PrestoErrors.getGroupByErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = PrestoToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = PrestoToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = PrestoToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = PrestoToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, PrestoQueryPartitioningBase::canonicalizeResultValue); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From 9b509a744bebf227d38e9ae35580d7e671630a58 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 14:23:15 +0800 Subject: [PATCH 0998/1284] [Citus] Use common TLP Where oracle --- src/sqlancer/citus/gen/CitusCommon.java | 2 +- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 66 +++++++------------ 2 files changed, 24 insertions(+), 44 deletions(-) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 1ca2af138..317dfc0f1 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -54,6 +54,7 @@ public static List getCitusErrors() { errors.add("unlogged columnar tables are not supported"); errors.add("UPDATE and CTID scans not supported for ColumnarScan"); errors.add("indexes not supported for columnar tables"); + errors.add("invalid byte sequence for encoding \"UTF8\": 0x00"); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { @@ -78,7 +79,6 @@ public static List getCitusErrors() { if (CitusBugs.bug4079) { errors.add("aggregate function calls cannot be nested"); } - return errors; } diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java index dbfc26e4b..9d1b13f41 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java @@ -1,62 +1,42 @@ package sqlancer.citus.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresVisitor; -import sqlancer.postgres.oracle.tlp.PostgresTLPBase; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class CitusTLPWhereOracle extends PostgresTLPBase { +public class CitusTLPWhereOracle implements TestOracle { - private final CitusTLPBase citusTLPBase; + private final TLPWhereOracle oracle; public CitusTLPWhereOracle(CitusGlobalState state) { - super(state); - CitusCommon.addCitusErrors(errors); - citusTLPBase = new CitusTLPBase(state); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .with(CitusCommon.getCitusErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - state.setAllowedFunctionTypes(Arrays.asList(PostgresGlobalState.IMMUTABLE)); - citusTLPBase.check(); - s = citusTLPBase.getSchema(); - targetTables = citusTLPBase.getTargetTables(); - gen = citusTLPBase.getGenerator(); - select = citusTLPBase.getSelect(); - predicate = citusTLPBase.getPredicate(); - negatedPredicate = citusTLPBase.getNegatedPredicate(); - isNullPredicate = citusTLPBase.getIsNullPredicate(); - whereCheck(); - state.setDefaultAllowedFunctionTypes(); + oracle.check(); } - void whereCheck() throws SQLException { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - String originalQueryString = PostgresVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = PostgresVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = PostgresVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = PostgresVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From c234b11a15d9048348e31b7951c81217560e3258 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 21 Sep 2024 16:09:32 +0800 Subject: [PATCH 0999/1284] [Citus] Update expected errors --- src/sqlancer/citus/gen/CitusCommon.java | 5 +++++ src/sqlancer/postgres/gen/PostgresTruncateGenerator.java | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 317dfc0f1..7a9f9c659 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -55,6 +55,11 @@ public static List getCitusErrors() { errors.add("UPDATE and CTID scans not supported for ColumnarScan"); errors.add("indexes not supported for columnar tables"); errors.add("invalid byte sequence for encoding \"UTF8\": 0x00"); + errors.add("columnar_tuple_insert_speculative not implemented"); + errors.add("row field count is 1, expected 2"); + errors.add("incorrect binary data format"); + errors.add("invalid sign in external \"numeric\" value"); + errors.add("Foreign keys and AFTER ROW triggers are not supported for columnar tables"); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { diff --git a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java index 197668ac5..0745d1cce 100644 --- a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java @@ -33,8 +33,9 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } - return new SQLQueryAdapter(sb.toString(), ExpectedErrors - .from("cannot truncate a table referenced in a foreign key constraint", "is not a table")); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot truncate a table referenced in a foreign key constraint", "is not a table", + "is not distributed")); } } From 3131e3f34d2a27f1a63b5f9213c73b98cbbc40b3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 21 Sep 2024 16:05:27 +0800 Subject: [PATCH 1000/1284] [Presto] Update expected errors --- src/sqlancer/presto/PrestoErrors.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 0e000119e..2296223d6 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -41,6 +41,7 @@ public static List getExpressionErrors() { // TODO: check errors.add("io.airlift.slice.Slice cannot be cast to java.lang.Number"); + errors.add("class io.airlift.slice.Slice cannot be cast to class java.lang.Number"); if (PrestoBugs.bug23324) { errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); } @@ -74,6 +75,11 @@ public static List getExpressionErrors() { errors.add("All IN list values must be the same type"); errors.add("All CASE results must be the same type"); errors.add("Mismatched types"); + errors.add("CASE operand type does not match WHEN clause operand type"); + errors.add("Subquery result type must be orderable"); + errors.add("Escape character must be followed by '%', '_' or the escape character itself"); + errors.add("Types are not comparable with NULLIF"); + errors.add("not of the same type"); if (PrestoBugs.bug23613) { errors.add("at index 1"); From 0e514c14e5c8da12667fb318f45faa57c971a511 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 25 Sep 2024 11:47:17 +0800 Subject: [PATCH 1001/1284] Refactor test oracle factory --- src/sqlancer/citus/CitusOptions.java | 48 -------- src/sqlancer/citus/CitusOracleFactory.java | 70 +++++++++++ .../citus/oracle/CitusNoRECOracle.java | 46 -------- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 42 ------- .../clickhouse/ClickHouseOptions.java | 51 -------- .../clickhouse/ClickHouseOracleFactory.java | 62 ++++++++++ .../oracle/norec/ClickHouseNoRECOracle.java | 45 -------- .../oracle/tlp/ClickHouseTLPWhereOracle.java | 45 -------- src/sqlancer/cnosdb/CnosDBOptions.java | 36 ------ src/sqlancer/cnosdb/CnosDBOracleFactory.java | 39 +++++++ .../cockroachdb/CockroachDBOptions.java | 87 -------------- .../cockroachdb/CockroachDBOracleFactory.java | 109 ++++++++++++++++++ .../cockroachdb/CockroachDBProvider.java | 1 - .../CockroachDBRandomQuerySynthesizer.java | 4 +- .../oracle/CockroachDBNoRECOracle.java | 69 ----------- .../tlp/CockroachDBTLPAggregateOracle.java | 3 +- .../oracle/tlp/CockroachDBTLPBase.java | 21 +++- .../oracle/tlp/CockroachDBTLPWhereOracle.java | 45 -------- src/sqlancer/databend/DatabendOptions.java | 76 ------------ .../databend/DatabendOracleFactory.java | 93 +++++++++++++++ .../databend/test/DatabendNoRECOracle.java | 45 -------- .../DatabendQueryPartitioningWhereTester.java | 45 -------- .../datafusion/DataFusionOptions.java | 22 ---- .../datafusion/DataFusionOracleFactory.java | 34 ++++++ .../test/DataFusionNoRECOracle.java | 44 ------- ...ataFusionQueryPartitioningWhereTester.java | 39 ------- src/sqlancer/doris/DorisOptions.java | 88 -------------- src/sqlancer/doris/DorisOracleFactory.java | 108 +++++++++++++++++ .../doris/oracle/DorisNoRECOracle.java | 44 ------- .../DorisQueryPartitioningWhereTester.java | 45 -------- src/sqlancer/duckdb/DuckDBOptions.java | 69 ----------- src/sqlancer/duckdb/DuckDBOracleFactory.java | 86 ++++++++++++++ .../duckdb/test/DuckDBNoRECOracle.java | 45 -------- .../DuckDBQueryPartitioningWhereTester.java | 45 -------- src/sqlancer/h2/H2Options.java | 18 --- src/sqlancer/h2/H2OracleFactory.java | 23 ++++ .../h2/H2QueryPartitioningWhereTester.java | 35 ------ src/sqlancer/hsqldb/HSQLDBOptions.java | 27 +---- src/sqlancer/hsqldb/HSQLDBOracleFactory.java | 32 +++++ .../hsqldb/test/HSQLDBNoRECOracle.java | 43 ------- .../HSQLDBQueryPartitioningWhereTester.java | 38 ------ src/sqlancer/mariadb/MariaDBOptions.java | 25 ---- .../mariadb/MariaDBOracleFactory.java | 36 ++++++ .../mariadb/oracle/MariaDBNoRECOracle.java | 48 -------- .../materialize/MaterializeOptions.java | 50 -------- .../materialize/MaterializeOracleFactory.java | 70 +++++++++++ .../materialize/MaterializeProvider.java | 1 - .../oracle/MaterializeNoRECOracle.java | 45 -------- .../oracle/tlp/MaterializeTLPWhereOracle.java | 45 -------- src/sqlancer/mysql/MySQLGlobalState.java | 1 - src/sqlancer/mysql/MySQLOptions.java | 58 ---------- src/sqlancer/mysql/MySQLOracleFactory.java | 64 ++++++++++ src/sqlancer/mysql/MySQLProvider.java | 1 - .../mysql/oracle/MySQLTLPWhereOracle.java | 45 -------- .../oceanbase/OceanBaseGlobalState.java | 1 - src/sqlancer/oceanbase/OceanBaseOptions.java | 35 ------ .../oceanbase/OceanBaseOracleFactory.java | 50 ++++++++ .../oracle/OceanBaseNoRECOracle.java | 49 -------- .../oracle/OceanBaseTLPWhereOracle.java | 45 -------- src/sqlancer/postgres/PostgresOptions.java | 70 ----------- .../postgres/PostgresOracleFactory.java | 91 +++++++++++++++ src/sqlancer/postgres/PostgresProvider.java | 1 - .../postgres/oracle/PostgresNoRECOracle.java | 79 ------------- .../postgres/oracle/tlp/PostgresTLPBase.java | 25 +++- .../oracle/tlp/PostgresTLPWhereOracle.java | 40 ------- src/sqlancer/presto/PrestoOptions.java | 66 +---------- src/sqlancer/presto/PrestoOracleFactory.java | 73 ++++++++++++ .../presto/test/PrestoNoRECOracle.java | 44 ------- src/sqlancer/questdb/QuestDBOptions.java | 16 --- .../questdb/QuestDBOracleFactory.java | 18 +++ src/sqlancer/sqlite3/SQLite3Options.java | 88 -------------- .../sqlite3/SQLite3OracleFactory.java | 101 ++++++++++++++++ src/sqlancer/sqlite3/SQLite3Provider.java | 1 - .../sqlite3/gen/SQLite3ColumnBuilder.java | 2 +- .../gen/ddl/SQLite3TableGenerator.java | 2 +- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 2 +- .../sqlite3/oracle/SQLite3NoRECOracle.java | 48 -------- .../oracle/tlp/SQLite3TLPWhereOracle.java | 37 ------ src/sqlancer/tidb/TiDBOptions.java | 53 --------- src/sqlancer/tidb/TiDBOracleFactory.java | 64 ++++++++++ src/sqlancer/tidb/TiDBProvider.java | 1 - .../tidb/oracle/TiDBTLPWhereOracle.java | 44 ------- src/sqlancer/yugabyte/ycql/YCQLOptions.java | 16 --- .../yugabyte/ycql/YCQLOracleFactory.java | 18 +++ src/sqlancer/yugabyte/ysql/YSQLOptions.java | 63 ---------- .../yugabyte/ysql/YSQLOracleFactory.java | 72 ++++++++++++ .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 44 ------- 87 files changed, 1363 insertions(+), 2452 deletions(-) create mode 100644 src/sqlancer/citus/CitusOracleFactory.java delete mode 100644 src/sqlancer/citus/oracle/CitusNoRECOracle.java delete mode 100644 src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java create mode 100644 src/sqlancer/clickhouse/ClickHouseOracleFactory.java delete mode 100644 src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java delete mode 100644 src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java create mode 100644 src/sqlancer/cnosdb/CnosDBOracleFactory.java create mode 100644 src/sqlancer/cockroachdb/CockroachDBOracleFactory.java delete mode 100644 src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java delete mode 100644 src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java create mode 100644 src/sqlancer/databend/DatabendOracleFactory.java delete mode 100644 src/sqlancer/databend/test/DatabendNoRECOracle.java delete mode 100644 src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/datafusion/DataFusionOracleFactory.java delete mode 100644 src/sqlancer/datafusion/test/DataFusionNoRECOracle.java delete mode 100644 src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/doris/DorisOracleFactory.java delete mode 100644 src/sqlancer/doris/oracle/DorisNoRECOracle.java delete mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/duckdb/DuckDBOracleFactory.java delete mode 100644 src/sqlancer/duckdb/test/DuckDBNoRECOracle.java delete mode 100644 src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/h2/H2OracleFactory.java delete mode 100644 src/sqlancer/h2/H2QueryPartitioningWhereTester.java create mode 100644 src/sqlancer/hsqldb/HSQLDBOracleFactory.java delete mode 100644 src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java delete mode 100644 src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/mariadb/MariaDBOracleFactory.java delete mode 100644 src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java create mode 100644 src/sqlancer/materialize/MaterializeOracleFactory.java delete mode 100644 src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java delete mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java create mode 100644 src/sqlancer/mysql/MySQLOracleFactory.java delete mode 100644 src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java create mode 100644 src/sqlancer/oceanbase/OceanBaseOracleFactory.java delete mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java delete mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java create mode 100644 src/sqlancer/postgres/PostgresOracleFactory.java delete mode 100644 src/sqlancer/postgres/oracle/PostgresNoRECOracle.java delete mode 100644 src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java create mode 100644 src/sqlancer/presto/PrestoOracleFactory.java delete mode 100644 src/sqlancer/presto/test/PrestoNoRECOracle.java create mode 100644 src/sqlancer/questdb/QuestDBOracleFactory.java create mode 100644 src/sqlancer/sqlite3/SQLite3OracleFactory.java delete mode 100644 src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java delete mode 100644 src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java create mode 100644 src/sqlancer/tidb/TiDBOracleFactory.java delete mode 100644 src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java delete mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index 67b14c304..f2d1b0abc 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -1,22 +1,11 @@ package sqlancer.citus; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameter; -import sqlancer.OracleFactory; -import sqlancer.citus.oracle.CitusNoRECOracle; -import sqlancer.citus.oracle.tlp.CitusTLPAggregateOracle; -import sqlancer.citus.oracle.tlp.CitusTLPHavingOracle; -import sqlancer.citus.oracle.tlp.CitusTLPWhereOracle; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresOptions; -import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; public class CitusOptions extends PostgresOptions { @@ -26,41 +15,4 @@ public class CitusOptions extends PostgresOptions { @Parameter(names = "--citusoracle", description = "Specifies which test oracle should be used for Citus extension to PostgreSQL") public List citusOracle = Arrays.asList(CitusOracleFactory.QUERY_PARTITIONING); - public enum CitusOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; - return new CitusNoRECOracle(citusGlobalState); - } - }, - PQS { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresPivotedQuerySynthesisOracle(globalState); - } - }, - HAVING { - - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; - return new CitusTLPHavingOracle(citusGlobalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; - List> oracles = new ArrayList<>(); - oracles.add(new CitusTLPWhereOracle(citusGlobalState)); - oracles.add(new CitusTLPHavingOracle(citusGlobalState)); - oracles.add(new CitusTLPAggregateOracle(citusGlobalState)); - return new CompositeTestOracle(oracles, globalState); - } - }; - - } - } diff --git a/src/sqlancer/citus/CitusOracleFactory.java b/src/sqlancer/citus/CitusOracleFactory.java new file mode 100644 index 000000000..b6e0a7f3e --- /dev/null +++ b/src/sqlancer/citus/CitusOracleFactory.java @@ -0,0 +1,70 @@ +package sqlancer.citus; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.citus.oracle.tlp.CitusTLPAggregateOracle; +import sqlancer.citus.oracle.tlp.CitusTLPHavingOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; + +public enum CitusOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .with(CitusCommon.getCitusErrors().toArray(new String[0])).build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + }, + WHERE { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()).with(CitusCommon.getCitusErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + HAVING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; + return new CitusTLPHavingOracle(citusGlobalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(citusGlobalState)); + oracles.add(HAVING.create(citusGlobalState)); + oracles.add(new CitusTLPAggregateOracle(citusGlobalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/citus/oracle/CitusNoRECOracle.java b/src/sqlancer/citus/oracle/CitusNoRECOracle.java deleted file mode 100644 index ffc7c13fb..000000000 --- a/src/sqlancer/citus/oracle/CitusNoRECOracle.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.citus.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.citus.gen.CitusCommon; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; - -public class CitusNoRECOracle implements TestOracle { - private final NoRECOracle oracle; - - public CitusNoRECOracle(PostgresGlobalState globalState) { - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) - .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) - .with(CitusCommon.getCitusErrors().toArray(new String[0])).build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - -} diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java deleted file mode 100644 index 9d1b13f41..000000000 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.citus.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.citus.CitusGlobalState; -import sqlancer.citus.gen.CitusCommon; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; - -public class CitusTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public CitusTLPWhereOracle(CitusGlobalState state) { - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) - .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) - .with(CitusCommon.getCitusErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 94297ad20..fec8b62a7 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -1,6 +1,5 @@ package sqlancer.clickhouse; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,16 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.clickhouse.ClickHouseOptions.ClickHouseOracleFactory; -import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; -import sqlancer.clickhouse.oracle.norec.ClickHouseNoRECOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPHavingOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; -import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "ClickHouse (default port: " + ClickHouseOptions.DEFAULT_PORT + ", default host: " + ClickHouseOptions.DEFAULT_HOST + ")") @@ -34,46 +23,6 @@ public class ClickHouseOptions implements DBMSSpecificOptions { - TLPWhere { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPWhereOracle(globalState); - } - }, - TLPDistinct { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPDistinctOracle(globalState); - } - }, - TLPGroupBy { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPGroupByOracle(globalState); - } - }, - TLPAggregate { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPAggregateOracle(globalState); - } - }, - TLPHaving { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPHavingOracle(globalState); - } - }, - NoREC { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseNoRECOracle(globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return oracle; diff --git a/src/sqlancer/clickhouse/ClickHouseOracleFactory.java b/src/sqlancer/clickhouse/ClickHouseOracleFactory.java new file mode 100644 index 000000000..53b38a6f1 --- /dev/null +++ b/src/sqlancer/clickhouse/ClickHouseOracleFactory.java @@ -0,0 +1,62 @@ +package sqlancer.clickhouse; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPHavingOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; + +public enum ClickHouseOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(ClickHouseErrors.getExpectedExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + TLPDistinct { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPDistinctOracle(globalState); + } + }, + TLPGroupBy { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPGroupByOracle(globalState); + } + }, + TLPAggregate { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPAggregateOracle(globalState); + } + }, + TLPHaving { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPHavingOracle(globalState); + } + }, + NoREC { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + + return new NoRECOracle<>(globalState, gen, errors); + } + } +} diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java deleted file mode 100644 index d2acc24d6..000000000 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.clickhouse.oracle.norec; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.clickhouse.ClickHouseErrors; -import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; -import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ast.ClickHouseExpression; -import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; -import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class ClickHouseNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - -} diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java deleted file mode 100644 index 564db22f7..000000000 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.clickhouse.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.clickhouse.ClickHouseErrors; -import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; -import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ast.ClickHouseExpression; -import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; -import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class ClickHouseTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public ClickHouseTLPWhereOracle(ClickHouseGlobalState state) { - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) - .build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/cnosdb/CnosDBOptions.java b/src/sqlancer/cnosdb/CnosDBOptions.java index 9a6da00c4..f101c2d38 100644 --- a/src/sqlancer/cnosdb/CnosDBOptions.java +++ b/src/sqlancer/cnosdb/CnosDBOptions.java @@ -1,20 +1,11 @@ package sqlancer.cnosdb; -import java.util.ArrayList; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.cnosdb.CnosDBOptions.CnosDBOracleFactory; -import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPAggregateOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPHavingOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPWhereOracle; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "CnosDB (default port: " + CnosDBOptions.DEFAULT_PORT + ", default host: " + CnosDBOptions.DEFAULT_HOST + ")") @@ -34,31 +25,4 @@ public List getTestOracleFactory() { return oracle; } - public enum CnosDBOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBNoRECOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - List> oracles = new ArrayList<>(); - oracles.add(new CnosDBTLPWhereOracle(globalState)); - oracles.add(new CnosDBTLPHavingOracle(globalState)); - oracles.add(new CnosDBTLPAggregateOracle(globalState)); - return new CompositeTestOracle<>(oracles, globalState); - } - } - - } - } diff --git a/src/sqlancer/cnosdb/CnosDBOracleFactory.java b/src/sqlancer/cnosdb/CnosDBOracleFactory.java new file mode 100644 index 000000000..7cb9c4fc6 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBOracleFactory.java @@ -0,0 +1,39 @@ +package sqlancer.cnosdb; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPAggregateOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPHavingOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPWhereOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; + +public enum CnosDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + return new CnosDBNoRECOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + return new CnosDBTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + List> oracles = new ArrayList<>(); + oracles.add(new CnosDBTLPWhereOracle(globalState)); + oracles.add(new CnosDBTLPHavingOracle(globalState)); + oracles.add(new CnosDBTLPAggregateOracle(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index e50a30c51..ce8a207d6 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -1,7 +1,5 @@ package sqlancer.cockroachdb; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,19 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPWhereOracle; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "CockroachDB (default port: " + CockroachDBOptions.DEFAULT_PORT + " default host: " + CockroachDBOptions.DEFAULT_HOST + ")") @@ -32,78 +17,6 @@ public class CockroachDBOptions implements DBMSSpecificOptions { - NOREC { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBNoRECOracle(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPAggregateOracle(globalState); - } - - }, - GROUP_BY { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPGroupByOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPHavingOracle(globalState); - } - }, - WHERE { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPWhereOracle(globalState); - } - }, - DISTINCT { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPDistinctOracle(globalState); - } - }, - EXTENDED_WHERE { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPExtendedWhereOracle(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new CockroachDBTLPAggregateOracle(globalState)); - oracles.add(new CockroachDBTLPHavingOracle(globalState)); - oracles.add(new CockroachDBTLPWhereOracle(globalState)); - oracles.add(new CockroachDBTLPGroupByOracle(globalState)); - oracles.add(new CockroachDBTLPExtendedWhereOracle(globalState)); - oracles.add(new CockroachDBTLPDistinctOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - CERT { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBCERTOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }; - - } - @Parameter(names = { "--test-hash-indexes" }, description = "Test the USING HASH WITH BUCKET_COUNT=n_buckets option in CREATE INDEX") public boolean testHashIndexes = true; diff --git a/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java new file mode 100644 index 000000000..bbf261f16 --- /dev/null +++ b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java @@ -0,0 +1,109 @@ +package sqlancer.cockroachdb; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; + +public enum CockroachDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with(CockroachDBErrors.getTransactionErrors()).with("unable to vectorize execution plan") // SET + // vectorize=experimental_always; + .with(" mismatched physical types at index") // SET vectorize=experimental_always; + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + AGGREGATE { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPAggregateOracle(globalState); + } + + }, + GROUP_BY { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPGroupByOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPHavingOracle(globalState); + } + }, + WHERE { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with("GROUP BY term out of range").build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + DISTINCT { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPDistinctOracle(globalState); + } + }, + EXTENDED_WHERE { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPExtendedWhereOracle(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(WHERE.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + oracles.add(EXTENDED_WHERE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CERT { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }; + +} diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 494c63a5b..66e80fd59 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -19,7 +19,6 @@ import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; -import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.gen.CockroachDBCommentOnGenerator; diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java index 09f5b165d..880f33310 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java @@ -13,7 +13,7 @@ import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPBase; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBRandomQuerySynthesizer { @@ -49,7 +49,7 @@ public static CockroachDBSelect generateSelect(CockroachDBGlobalState globalStat .map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList()); List updatedTableList = CockroachDBCommon.getTableReferences(tableList); if (Randomly.getBoolean()) { - select.setJoinList(CockroachDBNoRECOracle.getJoins(updatedTableList, globalState)); + select.setJoinList(CockroachDBTLPBase.getJoins(updatedTableList, globalState)); } select.setFromList(updatedTableList); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java deleted file mode 100644 index 3d0422dfd..000000000 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ /dev/null @@ -1,69 +0,0 @@ -package sqlancer.cockroachdb.oracle; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.Reproducer; -import sqlancer.cockroachdb.CockroachDBErrors; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBSelect; -import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class CockroachDBNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public CockroachDBNoRECOracle(CockroachDBGlobalState globalState) { - CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) - .with(CockroachDBErrors.getTransactionErrors()).with("unable to vectorize execution plan") // SET - // vectorize=experimental_always; - .with(" mismatched physical types at index") // SET vectorize=experimental_always; - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - public static List getJoins(List tableList, - CockroachDBGlobalState globalState) throws AssertionError { - List joinExpressions = new ArrayList<>(); - while (tableList.size() >= 2 && Randomly.getBoolean()) { - CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); - CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); - List columns = new ArrayList<>(leftTable.getTable().getColumns()); - columns.addAll(rightTable.getTable().getColumns()); - CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) - .setColumns(columns); - joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), - joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); - } - return joinExpressions; - } - -} diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index c9034eb65..6dd4bd5d5 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -29,7 +29,6 @@ import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -72,7 +71,7 @@ public void check() throws SQLException { .map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList()); List from = CockroachDBCommon.getTableReferences(tableList); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setJoinList(CockroachDBNoRECOracle.getJoins(from, state)); + select.setJoinList(CockroachDBTLPBase.getJoins(from, state)); } select.setFromList(from); if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index 21914eae2..c4809d231 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -10,14 +10,15 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; import sqlancer.cockroachdb.ast.CockroachDBColumnReference; import sqlancer.cockroachdb.ast.CockroachDBExpression; +import sqlancer.cockroachdb.ast.CockroachDBJoin; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -47,7 +48,7 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new CockroachDBTableReference(t)) .collect(Collectors.toList()); - List joins = CockroachDBNoRECOracle.getJoins(tableList, state); + List joins = getJoins(tableList, state); select.setJoinList(joins); select.setFromList(tableList); select.setWhereClause(null); @@ -69,4 +70,20 @@ protected ExpressionGenerator getGen() { return gen; } + public static List getJoins(List tableList, + CockroachDBGlobalState globalState) throws AssertionError { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBoolean()) { + CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); + CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + return joinExpressions; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java deleted file mode 100644 index 5fc14c492..000000000 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.cockroachdb.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.cockroachdb.CockroachDBErrors; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBSelect; -import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class CockroachDBTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public CockroachDBTLPWhereOracle(CockroachDBGlobalState state) { - CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) - .with("GROUP BY term out of range").build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index ed597066b..d38ee0d59 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -1,7 +1,5 @@ package sqlancer.databend; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,18 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.databend.DatabendOptions.DatabendOracleFactory; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.test.DatabendNoRECOracle; -import sqlancer.databend.test.DatabendPivotedQuerySynthesisOracle; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningAggregateTester; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningDistinctTester; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningGroupByTester; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningHavingTester; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningWhereTester; @Parameters(commandDescription = "Databend") public class DatabendOptions implements DBMSSpecificOptions { @@ -96,68 +82,6 @@ public class DatabendOptions implements DBMSSpecificOptions oracles = Arrays.asList(DatabendOracleFactory.QUERY_PARTITIONING); - public enum DatabendOracleFactory implements OracleFactory { - NOREC { - - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendNoRECOracle(globalState); - } - - }, - HAVING { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningAggregateTester(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningDistinctTester(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new DatabendQueryPartitioningWhereTester(globalState)); - oracles.add(new DatabendQueryPartitioningHavingTester(globalState)); - oracles.add(new DatabendQueryPartitioningAggregateTester(globalState)); - oracles.add(new DatabendQueryPartitioningDistinctTester(globalState)); - oracles.add(new DatabendQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - PQS { - @Override - public TestOracle create(DatabendGlobalState globalState) throws Exception { - return new DatabendPivotedQuerySynthesisOracle(globalState); - } - } - - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/databend/DatabendOracleFactory.java b/src/sqlancer/databend/DatabendOracleFactory.java new file mode 100644 index 000000000..ea66bc353 --- /dev/null +++ b/src/sqlancer/databend/DatabendOracleFactory.java @@ -0,0 +1,93 @@ +package sqlancer.databend; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; +import sqlancer.databend.test.DatabendPivotedQuerySynthesisOracle; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningAggregateTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningDistinctTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningGroupByTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningHavingTester; + +public enum DatabendOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with(DatabendErrors.getGroupByErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + GROUP_BY { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + PQS { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws Exception { + return new DatabendPivotedQuerySynthesisOracle(globalState); + } + } + +} diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java deleted file mode 100644 index 4d0d29711..000000000 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.databend.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.ast.DatabendJoin; -import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.gen.DatabendNewExpressionGenerator; - -public class DatabendNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public DatabendNoRECOracle(DatabendGlobalState globalState) { - DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - -} diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java deleted file mode 100644 index b6520e7f2..000000000 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.databend.test.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.ast.DatabendJoin; -import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.gen.DatabendNewExpressionGenerator; - -public class DatabendQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public DatabendQueryPartitioningWhereTester(DatabendGlobalState state) { - DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) - .with(DatabendErrors.getGroupByErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/datafusion/DataFusionOptions.java b/src/sqlancer/datafusion/DataFusionOptions.java index fcb0221a6..582b2f658 100644 --- a/src/sqlancer/datafusion/DataFusionOptions.java +++ b/src/sqlancer/datafusion/DataFusionOptions.java @@ -1,6 +1,5 @@ package sqlancer.datafusion; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,12 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.datafusion.DataFusionOptions.DataFusionOracleFactory; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.test.DataFusionNoRECOracle; -import sqlancer.datafusion.test.DataFusionQueryPartitioningWhereTester; @Parameters(commandDescription = "DataFusion") public class DataFusionOptions implements DBMSSpecificOptions { @@ -25,19 +18,4 @@ public List getTestOracleFactory() { return Arrays.asList(DataFusionOracleFactory.NOREC, DataFusionOracleFactory.QUERY_PARTITIONING_WHERE); } - public enum DataFusionOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(DataFusionGlobalState globalState) throws SQLException { - return new DataFusionNoRECOracle(globalState); - } - }, - QUERY_PARTITIONING_WHERE { - @Override - public TestOracle create(DataFusionGlobalState globalState) throws SQLException { - return new DataFusionQueryPartitioningWhereTester(globalState); - } - } - } - } diff --git a/src/sqlancer/datafusion/DataFusionOracleFactory.java b/src/sqlancer/datafusion/DataFusionOracleFactory.java new file mode 100644 index 000000000..a7a3f21e8 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionOracleFactory.java @@ -0,0 +1,34 @@ +package sqlancer.datafusion; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +public enum DataFusionOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create( + DataFusionProvider.DataFusionGlobalState globalState) throws SQLException { + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + QUERY_PARTITIONING_WHERE { + @Override + public TestOracle create( + DataFusionProvider.DataFusionGlobalState globalState) throws SQLException { + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(DataFusionErrors.getExpectedExecutionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + } +} diff --git a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java deleted file mode 100644 index 36ab97854..000000000 --- a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.datafusion.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.datafusion.DataFusionErrors; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionSchema; -import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; -import sqlancer.datafusion.DataFusionSchema.DataFusionTable; -import sqlancer.datafusion.ast.DataFusionExpression; -import sqlancer.datafusion.ast.DataFusionJoin; -import sqlancer.datafusion.ast.DataFusionSelect; -import sqlancer.datafusion.gen.DataFusionExpressionGenerator; - -public class DataFusionNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public DataFusionNoRECOracle(DataFusionGlobalState globalState) { - DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java deleted file mode 100644 index e0335a18f..000000000 --- a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.datafusion.test; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.datafusion.DataFusionErrors; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionSchema; -import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; -import sqlancer.datafusion.DataFusionSchema.DataFusionTable; -import sqlancer.datafusion.ast.DataFusionExpression; -import sqlancer.datafusion.ast.DataFusionJoin; -import sqlancer.datafusion.ast.DataFusionSelect; -import sqlancer.datafusion.gen.DataFusionExpressionGenerator; - -public class DataFusionQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public DataFusionQueryPartitioningWhereTester(DataFusionGlobalState state) { - DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) - .build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/doris/DorisOptions.java b/src/sqlancer/doris/DorisOptions.java index a2a05b95e..7e5a2c7e3 100644 --- a/src/sqlancer/doris/DorisOptions.java +++ b/src/sqlancer/doris/DorisOptions.java @@ -1,7 +1,5 @@ package sqlancer.doris; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,18 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.doris.DorisOptions.DorisOracleFactory; -import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.oracle.DorisNoRECOracle; -import sqlancer.doris.oracle.DorisPivotedQuerySynthesisOracle; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningAggregateTester; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningDistinctTester; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningGroupByTester; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningHavingTester; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningWhereTester; @Parameters(commandDescription = "Apache Doris (default port: " + DorisOptions.DEFAULT_PORT + ", default host: " + DorisOptions.DEFAULT_HOST + ")") @@ -112,80 +98,6 @@ public class DorisOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracles = Arrays.asList(DorisOracleFactory.NOREC); - public enum DorisOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisNoRECOracle(globalState); - } - - }, - HAVING { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningAggregateTester(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningDistinctTester(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new DorisQueryPartitioningWhereTester(globalState)); - oracles.add(new DorisQueryPartitioningHavingTester(globalState)); - oracles.add(new DorisQueryPartitioningAggregateTester(globalState)); - oracles.add(new DorisQueryPartitioningDistinctTester(globalState)); - oracles.add(new DorisQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - PQS { - @Override - public TestOracle create(DorisGlobalState globalState) throws Exception { - return new DorisPivotedQuerySynthesisOracle(globalState); - } - }, - ALL { - @Override - public TestOracle create(DorisGlobalState globalState) throws Exception { - List> oracles = new ArrayList<>(); - oracles.add(new DorisNoRECOracle(globalState)); - oracles.add(new DorisQueryPartitioningWhereTester(globalState)); - oracles.add(new DorisQueryPartitioningHavingTester(globalState)); - oracles.add(new DorisQueryPartitioningAggregateTester(globalState)); - oracles.add(new DorisQueryPartitioningDistinctTester(globalState)); - oracles.add(new DorisQueryPartitioningGroupByTester(globalState)); - oracles.add(new DorisPivotedQuerySynthesisOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - } - - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/doris/DorisOracleFactory.java b/src/sqlancer/doris/DorisOracleFactory.java new file mode 100644 index 000000000..8f28f0f21 --- /dev/null +++ b/src/sqlancer/doris/DorisOracleFactory.java @@ -0,0 +1,108 @@ +package sqlancer.doris; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.oracle.DorisPivotedQuerySynthesisOracle; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningAggregateTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningDistinctTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningGroupByTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningHavingTester; + +public enum DorisOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with(DorisErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + GROUP_BY { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + PQS { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws Exception { + return new DorisPivotedQuerySynthesisOracle(globalState); + } + }, + ALL { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(NOREC.create(globalState)); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + oracles.add(new DorisPivotedQuerySynthesisOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java deleted file mode 100644 index 9ce94b952..000000000 --- a/src/sqlancer/doris/oracle/DorisNoRECOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.doris.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.doris.DorisErrors; -import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.DorisSchema; -import sqlancer.doris.DorisSchema.DorisColumn; -import sqlancer.doris.DorisSchema.DorisTable; -import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.ast.DorisJoin; -import sqlancer.doris.ast.DorisSelect; -import sqlancer.doris.gen.DorisNewExpressionGenerator; - -public class DorisNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public DorisNoRECOracle(DorisGlobalState globalState) { - DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java deleted file mode 100644 index 32b171af0..000000000 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.doris.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.doris.DorisErrors; -import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.DorisSchema; -import sqlancer.doris.DorisSchema.DorisColumn; -import sqlancer.doris.DorisSchema.DorisTable; -import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.ast.DorisJoin; -import sqlancer.doris.ast.DorisSelect; -import sqlancer.doris.gen.DorisNewExpressionGenerator; - -public class DorisQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public DorisQueryPartitioningWhereTester(DorisGlobalState state) { - DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) - .with(DorisErrors.getExpressionErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index f14379163..00f85eece 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -1,7 +1,5 @@ package sqlancer.duckdb; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,17 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.duckdb.DuckDBOptions.DuckDBOracleFactory; -import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.test.DuckDBNoRECOracle; -import sqlancer.duckdb.test.DuckDBQueryPartitioningAggregateTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningDistinctTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningGroupByTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningHavingTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningWhereTester; @Parameters(commandDescription = "DuckDB") public class DuckDBOptions implements DBMSSpecificOptions { @@ -93,62 +80,6 @@ public class DuckDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracles = Arrays.asList(DuckDBOracleFactory.QUERY_PARTITIONING); - public enum DuckDBOracleFactory implements OracleFactory { - NOREC { - - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBNoRECOracle(globalState); - } - - }, - HAVING { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningAggregateTester(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningDistinctTester(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new DuckDBQueryPartitioningWhereTester(globalState)); - oracles.add(new DuckDBQueryPartitioningHavingTester(globalState)); - oracles.add(new DuckDBQueryPartitioningAggregateTester(globalState)); - oracles.add(new DuckDBQueryPartitioningDistinctTester(globalState)); - oracles.add(new DuckDBQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/duckdb/DuckDBOracleFactory.java b/src/sqlancer/duckdb/DuckDBOracleFactory.java new file mode 100644 index 000000000..8fd0f96af --- /dev/null +++ b/src/sqlancer/duckdb/DuckDBOracleFactory.java @@ -0,0 +1,86 @@ +package sqlancer.duckdb; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator; +import sqlancer.duckdb.test.DuckDBQueryPartitioningAggregateTester; +import sqlancer.duckdb.test.DuckDBQueryPartitioningDistinctTester; +import sqlancer.duckdb.test.DuckDBQueryPartitioningGroupByTester; +import sqlancer.duckdb.test.DuckDBQueryPartitioningHavingTester; + +public enum DuckDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .withRegex(DuckDBErrors.getExpressionErrorsRegex()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .with(DuckDBErrors.getGroupByErrors()).withRegex(DuckDBErrors.getExpressionErrorsRegex()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + GROUP_BY { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java deleted file mode 100644 index 4096714ac..000000000 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.duckdb.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.duckdb.DuckDBErrors; -import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBSchema; -import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; -import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.ast.DuckDBExpression; -import sqlancer.duckdb.ast.DuckDBJoin; -import sqlancer.duckdb.ast.DuckDBSelect; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator; - -public class DuckDBNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public DuckDBNoRECOracle(DuckDBGlobalState globalState) { - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) - .withRegex(DuckDBErrors.getExpressionErrorsRegex()).with("canceling statement due to statement timeout") - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java deleted file mode 100644 index d4ffa74c8..000000000 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.duckdb.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.duckdb.DuckDBErrors; -import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBSchema; -import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; -import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.ast.DuckDBExpression; -import sqlancer.duckdb.ast.DuckDBJoin; -import sqlancer.duckdb.ast.DuckDBSelect; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator; - -public class DuckDBQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public DuckDBQueryPartitioningWhereTester(DuckDBGlobalState state) { - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) - .with(DuckDBErrors.getGroupByErrors()).withRegex(DuckDBErrors.getExpressionErrorsRegex()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/h2/H2Options.java b/src/sqlancer/h2/H2Options.java index e7141fcf2..985853e94 100644 --- a/src/sqlancer/h2/H2Options.java +++ b/src/sqlancer/h2/H2Options.java @@ -1,33 +1,15 @@ package sqlancer.h2; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.h2.H2Options.H2OracleFactory; -import sqlancer.h2.H2Provider.H2GlobalState; @Parameters(commandDescription = "H2") public class H2Options implements DBMSSpecificOptions { - public enum H2OracleFactory implements OracleFactory { - - TLP_WHERE { - - @Override - public TestOracle create(H2GlobalState globalState) throws SQLException { - return new H2QueryPartitioningWhereTester(globalState); - } - - }; - - } - @Override public List getTestOracleFactory() { return Arrays.asList(H2OracleFactory.TLP_WHERE); diff --git a/src/sqlancer/h2/H2OracleFactory.java b/src/sqlancer/h2/H2OracleFactory.java new file mode 100644 index 000000000..fbbaade05 --- /dev/null +++ b/src/sqlancer/h2/H2OracleFactory.java @@ -0,0 +1,23 @@ +package sqlancer.h2; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; + +public enum H2OracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(H2Provider.H2GlobalState globalState) throws SQLException { + H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(H2Errors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }; + +} diff --git a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java deleted file mode 100644 index f1bb73dfa..000000000 --- a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.h2; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.h2.H2Provider.H2GlobalState; -import sqlancer.h2.H2Schema.H2Column; -import sqlancer.h2.H2Schema.H2Table; -import sqlancer.h2.ast.H2Expression; -import sqlancer.h2.ast.H2Join; -import sqlancer.h2.ast.H2Select; - -public class H2QueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public H2QueryPartitioningWhereTester(H2GlobalState state) { - H2ExpressionGenerator gen = new H2ExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(H2Errors.getExpressionErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index 3e3dcc4dc..215f4f0fc 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -1,40 +1,17 @@ package sqlancer.hsqldb; -import java.sql.SQLException; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.hsqldb.test.HSQLDBNoRECOracle; -import sqlancer.hsqldb.test.HSQLDBQueryPartitioningWhereTester; @Parameters(commandDescription = "hsqldb") -public class HSQLDBOptions implements DBMSSpecificOptions { +public class HSQLDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") - public List oracle = List.of(HSQLDBOracleFactory.WHERE, - HSQLDBOracleFactory.NOREC); - - public enum HSQLDBOracleFactory implements OracleFactory { - WHERE { - @Override - public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) - throws SQLException { - return new HSQLDBQueryPartitioningWhereTester(globalState); - } - }, - NOREC { - @Override - public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) - throws Exception { - return new HSQLDBNoRECOracle(globalState); - } - } - } + public List oracle = List.of(HSQLDBOracleFactory.WHERE, HSQLDBOracleFactory.NOREC); @Override public List getTestOracleFactory() { diff --git a/src/sqlancer/hsqldb/HSQLDBOracleFactory.java b/src/sqlancer/hsqldb/HSQLDBOracleFactory.java new file mode 100644 index 000000000..ed4fdf79c --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBOracleFactory.java @@ -0,0 +1,32 @@ +package sqlancer.hsqldb; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; + +public enum HSQLDBOracleFactory implements OracleFactory { + WHERE { + @Override + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws SQLException { + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + NOREC { + @Override + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws Exception { + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + return new NoRECOracle<>(globalState, gen, errors); + } + } +} diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java deleted file mode 100644 index 0b1a3b8b8..000000000 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ /dev/null @@ -1,43 +0,0 @@ -package sqlancer.hsqldb.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.hsqldb.HSQLDBErrors; -import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; -import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; -import sqlancer.hsqldb.ast.HSQLDBExpression; -import sqlancer.hsqldb.ast.HSQLDBJoin; -import sqlancer.hsqldb.ast.HSQLDBSelect; -import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; - -public class HSQLDBNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public HSQLDBNoRECOracle(HSQLDBGlobalState globalState) { - HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java deleted file mode 100644 index 795c6656c..000000000 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,38 +0,0 @@ -package sqlancer.hsqldb.test; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.hsqldb.HSQLDBErrors; -import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; -import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; -import sqlancer.hsqldb.ast.HSQLDBExpression; -import sqlancer.hsqldb.ast.HSQLDBJoin; -import sqlancer.hsqldb.ast.HSQLDBSelect; -import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; - -public class HSQLDBQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public HSQLDBQueryPartitioningWhereTester(HSQLDBGlobalState state) { - HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index f601ab0a6..b1e7f807e 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -1,6 +1,5 @@ package sqlancer.mariadb; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,12 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mariadb.MariaDBOptions.MariaDBOracleFactory; -import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; -import sqlancer.mariadb.oracle.MariaDBDQPOracle; -import sqlancer.mariadb.oracle.MariaDBNoRECOracle; @Parameters(separators = "=", commandDescription = "MariaDB (default port: " + MariaDBOptions.DEFAULT_PORT + ", default host: " + MariaDBOptions.DEFAULT_HOST + ")") @@ -24,24 +17,6 @@ public class MariaDBOptions implements DBMSSpecificOptions @Parameter(names = "--oracle") public List oracles = Arrays.asList(MariaDBOracleFactory.NOREC); - public enum MariaDBOracleFactory implements OracleFactory { - - NOREC { - - @Override - public TestOracle create(MariaDBGlobalState globalState) throws SQLException { - return new MariaDBNoRECOracle(globalState); - } - - }, - DQP { - @Override - public TestOracle create(MariaDBGlobalState globalState) throws SQLException { - return new MariaDBDQPOracle(globalState); - } - } - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/mariadb/MariaDBOracleFactory.java b/src/sqlancer/mariadb/MariaDBOracleFactory.java new file mode 100644 index 000000000..549da27c8 --- /dev/null +++ b/src/sqlancer/mariadb/MariaDBOracleFactory.java @@ -0,0 +1,36 @@ +package sqlancer.mariadb; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.mariadb.gen.MariaDBExpressionGenerator; +import sqlancer.mariadb.oracle.MariaDBDQPOracle; + +public enum MariaDBOracleFactory implements OracleFactory { + + NOREC { + @Override + public TestOracle create(MariaDBProvider.MariaDBGlobalState globalState) + throws SQLException { + MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(globalState.getRandomly()); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MariaDBErrors.getCommonErrors()) + .with("is out of range").with("unmatched parentheses").with("nothing to repeat at offset") + .with("missing )").with("missing terminating ]").with("range out of order in character class") + .with("unrecognized character after ").with("Got error '(*VERB) not recognized or malformed") + .with("must be followed by").with("malformed number or name after").with("digit expected after") + .with("Could not create a join buffer").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + DQP { + @Override + public TestOracle create(MariaDBProvider.MariaDBGlobalState globalState) + throws SQLException { + return new MariaDBDQPOracle(globalState); + } + } +} diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java deleted file mode 100644 index 039abd85f..000000000 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ /dev/null @@ -1,48 +0,0 @@ -package sqlancer.mariadb.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mariadb.MariaDBErrors; -import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; -import sqlancer.mariadb.MariaDBSchema; -import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; -import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -import sqlancer.mariadb.ast.MariaDBExpression; -import sqlancer.mariadb.ast.MariaDBJoin; -import sqlancer.mariadb.ast.MariaDBSelectStatement; -import sqlancer.mariadb.gen.MariaDBExpressionGenerator; - -public class MariaDBNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public MariaDBNoRECOracle(MariaDBGlobalState globalState) { - MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(globalState.getRandomly()); - ExpectedErrors errors = ExpectedErrors.newErrors().with(MariaDBErrors.getCommonErrors()).with("is out of range") - .with("unmatched parentheses").with("nothing to repeat at offset").with("missing )") - .with("missing terminating ]").with("range out of order in character class") - .with("unrecognized character after ").with("Got error '(*VERB) not recognized or malformed") - .with("must be followed by").with("malformed number or name after").with("digit expected after") - .with("Could not create a join buffer").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/materialize/MaterializeOptions.java b/src/sqlancer/materialize/MaterializeOptions.java index 23191e3a9..104dc9b51 100644 --- a/src/sqlancer/materialize/MaterializeOptions.java +++ b/src/sqlancer/materialize/MaterializeOptions.java @@ -1,7 +1,5 @@ package sqlancer.materialize; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,15 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.materialize.MaterializeOptions.MaterializeOracleFactory; -import sqlancer.materialize.oracle.MaterializeNoRECOracle; -import sqlancer.materialize.oracle.MaterializePivotedQuerySynthesisOracle; -import sqlancer.materialize.oracle.tlp.MaterializeTLPAggregateOracle; -import sqlancer.materialize.oracle.tlp.MaterializeTLPHavingOracle; -import sqlancer.materialize.oracle.tlp.MaterializeTLPWhereOracle; @Parameters(separators = "=", commandDescription = "Materialize (default port: " + MaterializeOptions.DEFAULT_PORT + ", default host: " + MaterializeOptions.DEFAULT_HOST + ", default user: " + MaterializeOptions.DEFAULT_USER @@ -46,45 +35,6 @@ public class MaterializeOptions implements DBMSSpecificOptions { - NOREC { - @Override - public TestOracle create(MaterializeGlobalState globalState) throws SQLException { - return new MaterializeNoRECOracle(globalState); - } - }, - PQS { - @Override - public TestOracle create(MaterializeGlobalState globalState) throws SQLException { - return new MaterializePivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - HAVING { - - @Override - public TestOracle create(MaterializeGlobalState globalState) throws SQLException { - return new MaterializeTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(MaterializeGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new MaterializeTLPWhereOracle(globalState)); - oracles.add(new MaterializeTLPHavingOracle(globalState)); - oracles.add(new MaterializeTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return oracle; diff --git a/src/sqlancer/materialize/MaterializeOracleFactory.java b/src/sqlancer/materialize/MaterializeOracleFactory.java new file mode 100644 index 000000000..c652fd56a --- /dev/null +++ b/src/sqlancer/materialize/MaterializeOracleFactory.java @@ -0,0 +1,70 @@ +package sqlancer.materialize; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; +import sqlancer.materialize.oracle.MaterializePivotedQuerySynthesisOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPAggregateOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPHavingOracle; + +public enum MaterializeOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) + .with(MaterializeCommon.getCommonFetchErrors()).with("canceling statement due to statement timeout") + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + WHERE { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(MaterializeCommon.getCommonExpressionErrors()).with(MaterializeCommon.getCommonFetchErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + HAVING { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializeTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(new MaterializeTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java index f35e476f3..5b18b1dfb 100644 --- a/src/sqlancer/materialize/MaterializeProvider.java +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -25,7 +25,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.materialize.MaterializeOptions.MaterializeOracleFactory; import sqlancer.materialize.gen.MaterializeDeleteGenerator; import sqlancer.materialize.gen.MaterializeDropIndexGenerator; import sqlancer.materialize.gen.MaterializeIndexGenerator; diff --git a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java deleted file mode 100644 index e0f665d93..000000000 --- a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.materialize.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.materialize.MaterializeGlobalState; -import sqlancer.materialize.MaterializeSchema; -import sqlancer.materialize.MaterializeSchema.MaterializeColumn; -import sqlancer.materialize.MaterializeSchema.MaterializeTable; -import sqlancer.materialize.ast.MaterializeExpression; -import sqlancer.materialize.ast.MaterializeJoin; -import sqlancer.materialize.ast.MaterializeSelect; -import sqlancer.materialize.gen.MaterializeCommon; -import sqlancer.materialize.gen.MaterializeExpressionGenerator; - -public class MaterializeNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public MaterializeNoRECOracle(MaterializeGlobalState globalState) { - MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) - .with(MaterializeCommon.getCommonFetchErrors()).with("canceling statement due to statement timeout") - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java deleted file mode 100644 index cd14bfb75..000000000 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.materialize.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.materialize.MaterializeGlobalState; -import sqlancer.materialize.MaterializeSchema; -import sqlancer.materialize.MaterializeSchema.MaterializeColumn; -import sqlancer.materialize.MaterializeSchema.MaterializeTable; -import sqlancer.materialize.ast.MaterializeExpression; -import sqlancer.materialize.ast.MaterializeJoin; -import sqlancer.materialize.ast.MaterializeSelect; -import sqlancer.materialize.gen.MaterializeCommon; -import sqlancer.materialize.gen.MaterializeExpressionGenerator; - -public class MaterializeTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public MaterializeTLPWhereOracle(MaterializeGlobalState state) { - MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) - .with(MaterializeCommon.getCommonFetchErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 86cf6dd18..10132b57c 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import sqlancer.SQLGlobalState; -import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; public class MySQLGlobalState extends SQLGlobalState { diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 87408c39f..9219073d5 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -1,6 +1,5 @@ package sqlancer.mysql; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,14 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; -import sqlancer.mysql.oracle.MySQLCERTOracle; -import sqlancer.mysql.oracle.MySQLDQPOracle; -import sqlancer.mysql.oracle.MySQLFuzzer; -import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; -import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @Parameters(separators = "=", commandDescription = "MySQL (default port: " + MySQLOptions.DEFAULT_PORT + ", default host: " + MySQLOptions.DEFAULT_HOST + ")") @@ -26,55 +17,6 @@ public class MySQLOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracles = Arrays.asList(MySQLOracleFactory.TLP_WHERE); - public enum MySQLOracleFactory implements OracleFactory { - - TLP_WHERE { - - @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLTLPWhereOracle(globalState); - } - - }, - PQS { - - @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLPivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - - }, - CERT { - @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLCERTOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - FUZZER { - @Override - public TestOracle create(MySQLGlobalState globalState) throws Exception { - return new MySQLFuzzer(globalState); - } - - }, - DQP { - @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLDQPOracle(globalState); - } - }; - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java new file mode 100644 index 000000000..206179101 --- /dev/null +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -0,0 +1,64 @@ +package sqlancer.mysql; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.mysql.gen.MySQLExpressionGenerator; +import sqlancer.mysql.oracle.MySQLCERTOracle; +import sqlancer.mysql.oracle.MySQLDQPOracle; +import sqlancer.mysql.oracle.MySQLFuzzer; +import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; + +public enum MySQLOracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }, + PQS { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + + }, + CERT { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + FUZZER { + @Override + public TestOracle create(MySQLGlobalState globalState) throws Exception { + return new MySQLFuzzer(globalState); + } + + }, + DQP { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLDQPOracle(globalState); + } + }; +} diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 287f20a7e..80a23b947 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -21,7 +21,6 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; -import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; diff --git a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java deleted file mode 100644 index 6d0fd1178..000000000 --- a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mysql.MySQLErrors; -import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLSchema; -import sqlancer.mysql.MySQLSchema.MySQLColumn; -import sqlancer.mysql.MySQLSchema.MySQLTable; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLJoin; -import sqlancer.mysql.ast.MySQLSelect; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -public class MySQLTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public MySQLTLPWhereOracle(MySQLGlobalState state) { - MySQLExpressionGenerator gen = new MySQLExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) - .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/oceanbase/OceanBaseGlobalState.java b/src/sqlancer/oceanbase/OceanBaseGlobalState.java index b8baf08fb..fa483072b 100644 --- a/src/sqlancer/oceanbase/OceanBaseGlobalState.java +++ b/src/sqlancer/oceanbase/OceanBaseGlobalState.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import sqlancer.SQLGlobalState; -import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; public class OceanBaseGlobalState extends SQLGlobalState { diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index 86507f769..949e5158b 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -1,6 +1,5 @@ package sqlancer.oceanbase; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,12 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; -import sqlancer.oceanbase.oracle.OceanBaseNoRECOracle; -import sqlancer.oceanbase.oracle.OceanBasePivotedQuerySynthesisOracle; -import sqlancer.oceanbase.oracle.OceanBaseTLPWhereOracle; @Parameters(separators = "=", commandDescription = "OceanBase (default port: " + OceanBaseOptions.DEFAULT_PORT + ", default host: " + OceanBaseOptions.DEFAULT_HOST + ")") @@ -24,34 +17,6 @@ public class OceanBaseOptions implements DBMSSpecificOptions oracles = Arrays.asList(OceanBaseOracleFactory.TLP_WHERE); - public enum OceanBaseOracleFactory implements OracleFactory { - - TLP_WHERE { - @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { - return new OceanBaseTLPWhereOracle(globalState); - } - }, - NoREC { - @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { - return new OceanBaseNoRECOracle(globalState); - } - }, - PQS { - - @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { - return new OceanBasePivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - } - } - @Parameter(names = { "--query-timeout" }, description = "Query timeout") public int queryTimeout = 1000000000; @Parameter(names = { "--transaction-timeout" }, description = "Transaction timeout") diff --git a/src/sqlancer/oceanbase/OceanBaseOracleFactory.java b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java new file mode 100644 index 000000000..b7b115f37 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java @@ -0,0 +1,50 @@ +package sqlancer.oceanbase; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; +import sqlancer.oceanbase.oracle.OceanBasePivotedQuerySynthesisOracle; + +public enum OceanBaseOracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).with("value is out of range").build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + NoREC { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()) + .with("canceling statement due to statement timeout").with("unmatched parentheses") + .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") + .with("range out of order in character class").with("unrecognized character after ") + .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") + .with("malformed number or name after").with("digit expected after").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + return new OceanBasePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + } +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java deleted file mode 100644 index 0ab0e6d2f..000000000 --- a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java +++ /dev/null @@ -1,49 +0,0 @@ -package sqlancer.oceanbase.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.oceanbase.OceanBaseErrors; -import sqlancer.oceanbase.OceanBaseGlobalState; -import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.ast.OceanBaseExpression; -import sqlancer.oceanbase.ast.OceanBaseJoin; -import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; - -public class OceanBaseNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) - .withRegex(OceanBaseErrors.getExpressionErrorsRegex()) - .with("canceling statement due to statement timeout").with("unmatched parentheses") - .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") - .with("range out of order in character class").with("unrecognized character after ") - .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") - .with("malformed number or name after").with("digit expected after").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java deleted file mode 100644 index 08f5fef93..000000000 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.oceanbase.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.oceanbase.OceanBaseErrors; -import sqlancer.oceanbase.OceanBaseGlobalState; -import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.ast.OceanBaseExpression; -import sqlancer.oceanbase.ast.OceanBaseJoin; -import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; - -public class OceanBaseTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public OceanBaseTLPWhereOracle(OceanBaseGlobalState state) { - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) - .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).with("value is out of range").build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index ced65d5da..659dc4ff6 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -1,7 +1,5 @@ package sqlancer.postgres; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,17 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; -import sqlancer.postgres.oracle.PostgresCERTOracle; -import sqlancer.postgres.oracle.PostgresFuzzer; -import sqlancer.postgres.oracle.PostgresNoRECOracle; -import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; @Parameters(separators = "=", commandDescription = "PostgreSQL (default port: " + PostgresOptions.DEFAULT_PORT + ", default host: " + PostgresOptions.DEFAULT_HOST + ")") @@ -43,63 +30,6 @@ public class PostgresOptions implements DBMSSpecificOptions { - NOREC { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresNoRECOracle(globalState); - } - }, - PQS { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresPivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - HAVING { - - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new PostgresTLPWhereOracle(globalState)); - oracles.add(new PostgresTLPHavingOracle(globalState)); - oracles.add(new PostgresTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - CERT { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresCERTOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - FUZZER { - @Override - public TestOracle create(PostgresGlobalState globalState) throws Exception { - return new PostgresFuzzer(globalState); - } - - }; - - } - @Override public List getTestOracleFactory() { return oracle; diff --git a/src/sqlancer/postgres/PostgresOracleFactory.java b/src/sqlancer/postgres/PostgresOracleFactory.java new file mode 100644 index 000000000..45e2190fc --- /dev/null +++ b/src/sqlancer/postgres/PostgresOracleFactory.java @@ -0,0 +1,91 @@ +package sqlancer.postgres; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.oracle.PostgresCERTOracle; +import sqlancer.postgres.oracle.PostgresFuzzer; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; + +public enum PostgresOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()).build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + WHERE { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }, + HAVING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(new PostgresTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CERT { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + FUZZER { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + return new PostgresFuzzer(globalState); + } + + }; + +} diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 62c43cd5a..7c86dc628 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -22,7 +22,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; import sqlancer.postgres.gen.PostgresAlterTableGenerator; import sqlancer.postgres.gen.PostgresAnalyzeGenerator; import sqlancer.postgres.gen.PostgresClusterGenerator; diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java deleted file mode 100644 index 9243b4870..000000000 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ /dev/null @@ -1,79 +0,0 @@ -package sqlancer.postgres.oracle; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresDataType; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.PostgresSchema.PostgresTables; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; -import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; -import sqlancer.postgres.oracle.tlp.PostgresTLPBase; - -public class PostgresNoRECOracle implements TestOracle { - - private final NoRECOracle oracle; - - public PostgresNoRECOracle(PostgresGlobalState globalState) { - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) - .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - public static List getJoinStatements(PostgresGlobalState globalState, List columns, - List tables) { - List joinStatements = new ArrayList<>(); - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); - PostgresTable table = Randomly.fromList(tables); - tables.remove(table); - PostgresJoinType options = PostgresJoinType.getRandom(); - PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); - joinStatements.add(j); - } - // JOIN subqueries - for (int i = 0; i < Randomly.smallNumber(); i++) { - PostgresTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); - PostgresSubquery subquery = PostgresTLPBase.createSubquery(globalState, String.format("sub%d", i), - subqueryTables); - PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); - PostgresJoinType options = PostgresJoinType.getRandom(); - PostgresJoin j = new PostgresJoin(subquery, joinClause, options); - joinStatements.add(j); - } - return joinStatements; - } -} diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 77c4fb022..55b5d2dcf 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -20,13 +20,13 @@ import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.ForClause; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; -import sqlancer.postgres.oracle.PostgresNoRECOracle; public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { @@ -53,8 +53,27 @@ public void check() throws SQLException { protected List getJoinStatements(PostgresGlobalState globalState, List columns, List tables) { - return PostgresNoRECOracle.getJoinStatements(state, columns, tables); - // TODO joins + List joinStatements = new ArrayList<>(); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); + PostgresTable table = Randomly.fromList(tables); + tables.remove(table); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + PostgresTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PostgresSubquery subquery = PostgresTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; } protected void generateSelectBase(List tables, List joins) { diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java deleted file mode 100644 index 3c183457d..000000000 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java +++ /dev/null @@ -1,40 +0,0 @@ -package sqlancer.postgres.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; - -public class PostgresTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public PostgresTLPWhereOracle(PostgresGlobalState state) { - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) - .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) - .build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/presto/PrestoOptions.java b/src/sqlancer/presto/PrestoOptions.java index 78deab27c..f0c557ca5 100644 --- a/src/sqlancer/presto/PrestoOptions.java +++ b/src/sqlancer/presto/PrestoOptions.java @@ -1,24 +1,14 @@ package sqlancer.presto; -import java.util.ArrayList; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.presto.test.PrestoNoRECOracle; -import sqlancer.presto.test.PrestoQueryPartitioningAggregateTester; -import sqlancer.presto.test.PrestoQueryPartitioningDistinctTester; -import sqlancer.presto.test.PrestoQueryPartitioningGroupByTester; -import sqlancer.presto.test.PrestoQueryPartitioningHavingTester; -import sqlancer.presto.test.PrestoQueryPartitioningWhereTester; @Parameters(commandDescription = "Presto") -public class PrestoOptions implements DBMSSpecificOptions { +public class PrestoOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 8080; @@ -109,58 +99,4 @@ public List getTestOracleFactory() { return oracles; } - public enum PrestoOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoNoRECOracle(globalState); - } - - }, - HAVING { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningAggregateTester(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningDistinctTester(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(PrestoGlobalState globalState) { - List> oracles = new ArrayList<>(); - oracles.add(new PrestoQueryPartitioningWhereTester(globalState)); - oracles.add(new PrestoQueryPartitioningHavingTester(globalState)); - oracles.add(new PrestoQueryPartitioningAggregateTester(globalState)); - oracles.add(new PrestoQueryPartitioningDistinctTester(globalState)); - oracles.add(new PrestoQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle<>(oracles, globalState); - } - } - - } - } diff --git a/src/sqlancer/presto/PrestoOracleFactory.java b/src/sqlancer/presto/PrestoOracleFactory.java new file mode 100644 index 000000000..3076357f2 --- /dev/null +++ b/src/sqlancer/presto/PrestoOracleFactory.java @@ -0,0 +1,73 @@ +package sqlancer.presto; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; +import sqlancer.presto.test.PrestoQueryPartitioningAggregateTester; +import sqlancer.presto.test.PrestoQueryPartitioningDistinctTester; +import sqlancer.presto.test.PrestoQueryPartitioningGroupByTester; +import sqlancer.presto.test.PrestoQueryPartitioningHavingTester; +import sqlancer.presto.test.PrestoQueryPartitioningWhereTester; + +public enum PrestoOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PrestoGlobalState globalState) { + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PrestoGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java deleted file mode 100644 index 6ef5e68ce..000000000 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.presto.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.presto.PrestoErrors; -import sqlancer.presto.PrestoGlobalState; -import sqlancer.presto.PrestoSchema; -import sqlancer.presto.PrestoSchema.PrestoColumn; -import sqlancer.presto.PrestoSchema.PrestoTable; -import sqlancer.presto.ast.PrestoExpression; -import sqlancer.presto.ast.PrestoJoin; -import sqlancer.presto.ast.PrestoSelect; -import sqlancer.presto.gen.PrestoTypedExpressionGenerator; - -public class PrestoNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public PrestoNoRECOracle(PrestoGlobalState globalState) { - PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/questdb/QuestDBOptions.java b/src/sqlancer/questdb/QuestDBOptions.java index f84ca3d47..82cf55bd9 100644 --- a/src/sqlancer/questdb/QuestDBOptions.java +++ b/src/sqlancer/questdb/QuestDBOptions.java @@ -1,6 +1,5 @@ package sqlancer.questdb; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,11 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.questdb.QuestDBOptions.QuestDBOracleFactory; -import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; -import sqlancer.questdb.test.QuestDBQueryPartitioningWhereTester; @Parameters(separators = "=", commandDescription = "QuestDB (default port: " + QuestDBOptions.DEFAULT_PORT + " default host: " + QuestDBOptions.DEFAULT_HOST + ")") @@ -23,16 +17,6 @@ public class QuestDBOptions implements DBMSSpecificOptions @Parameter(names = "--oracle") public List oracle = Arrays.asList(QuestDBOracleFactory.WHERE); - public enum QuestDBOracleFactory implements OracleFactory { - // TODO (anxing): implement test oracles - WHERE { - @Override - public TestOracle create(QuestDBGlobalState globalState) throws SQLException { - return new QuestDBQueryPartitioningWhereTester(globalState); - } - } - } - @Parameter(names = "--username", description = "The user name used to log into QuestDB") private String userName = "admin"; // NOPMD diff --git a/src/sqlancer/questdb/QuestDBOracleFactory.java b/src/sqlancer/questdb/QuestDBOracleFactory.java new file mode 100644 index 000000000..52c727278 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBOracleFactory.java @@ -0,0 +1,18 @@ +package sqlancer.questdb; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.questdb.test.QuestDBQueryPartitioningWhereTester; + +public enum QuestDBOracleFactory implements OracleFactory { + // TODO (anxing): implement test oracles + WHERE { + @Override + public TestOracle create(QuestDBProvider.QuestDBGlobalState globalState) + throws SQLException { + return new QuestDBQueryPartitioningWhereTester(globalState); + } + } +} diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index e2c04ed46..ec90f3bca 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -1,7 +1,5 @@ package sqlancer.sqlite3; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,18 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; -import sqlancer.sqlite3.oracle.SQLite3Fuzzer; -import sqlancer.sqlite3.oracle.SQLite3NoRECOracle; -import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPAggregateOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPDistinctOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPGroupByOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPHavingOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPWhereOracle; @Parameters(separators = "=", commandDescription = "SQLite3") public class SQLite3Options implements DBMSSpecificOptions { @@ -93,80 +79,6 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") public int maxNumIndexes = 20; - public enum SQLite3OracleFactory implements OracleFactory { - PQS { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3PivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - - }, - NoREC { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3NoRECOracle(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPAggregateOracle(globalState); - } - - }, - WHERE { - - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPWhereOracle(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPDistinctOracle(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPGroupByOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPHavingOracle(globalState); - } - }, - FUZZER { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3Fuzzer(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new SQLite3TLPWhereOracle(globalState)); - oracles.add(new SQLite3TLPDistinctOracle(globalState)); - oracles.add(new SQLite3TLPGroupByOracle(globalState)); - oracles.add(new SQLite3TLPHavingOracle(globalState)); - oracles.add(new SQLite3TLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return Arrays.asList(oracles); diff --git a/src/sqlancer/sqlite3/SQLite3OracleFactory.java b/src/sqlancer/sqlite3/SQLite3OracleFactory.java new file mode 100644 index 000000000..b1a9a1265 --- /dev/null +++ b/src/sqlancer/sqlite3/SQLite3OracleFactory.java @@ -0,0 +1,101 @@ +package sqlancer.sqlite3; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.oracle.SQLite3Fuzzer; +import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPAggregateOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPDistinctOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPGroupByOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPHavingOracle; + +public enum SQLite3OracleFactory implements OracleFactory { + PQS { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3PivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + + }, + NoREC { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .with(SQLite3Errors.getMatchQueryErrors()).with(SQLite3Errors.getQueryErrors()) + .with("misuse of aggregate", "misuse of window function", + "second argument to nth_value must be a positive integer", "no such table", + "no query solution", "unable to use function MATCH in the requested context") + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + AGGREGATE { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPAggregateOracle(globalState); + } + + }, + WHERE { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .build(); + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }, + DISTINCT { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPDistinctOracle(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPGroupByOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPHavingOracle(globalState); + } + }, + FUZZER { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3Fuzzer(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index b20701c95..5fbd3b471 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -23,7 +23,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; import sqlancer.sqlite3.gen.SQLite3CreateVirtualRtreeTabelGenerator; import sqlancer.sqlite3.gen.SQLite3ExplainGenerator; diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index e07533ed8..1d045982b 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; +import sqlancer.sqlite3.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index 6c27a5d58..7847208a3 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -12,7 +12,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; +import sqlancer.sqlite3.SQLite3OracleFactory; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.schema.SQLite3Schema; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 1c483e59d..bc605a015 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -9,7 +9,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; +import sqlancer.sqlite3.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Select; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java deleted file mode 100644 index 8f6e2d276..000000000 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ /dev/null @@ -1,48 +0,0 @@ -package sqlancer.sqlite3.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.ast.SQLite3Expression.Join; -import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; -import sqlancer.sqlite3.schema.SQLite3Schema; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; - -public class SQLite3NoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public SQLite3NoRECOracle(SQLite3GlobalState globalState) { - SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) - .with(SQLite3Errors.getMatchQueryErrors()).with(SQLite3Errors.getQueryErrors()) - .with("misuse of aggregate", "misuse of window function", - "second argument to nth_value must be a positive integer", "no such table", "no query solution", - "unable to use function MATCH in the requested context") - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java deleted file mode 100644 index 8d850fdcb..000000000 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java +++ /dev/null @@ -1,37 +0,0 @@ -package sqlancer.sqlite3.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; -import sqlancer.sqlite3.schema.SQLite3Schema; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; - -public class SQLite3TLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public SQLite3TLPWhereOracle(SQLite3GlobalState state) { - SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) - .build(); - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 6c69e84a1..1619832d6 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -1,7 +1,5 @@ package sqlancer.tidb; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,15 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; -import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.oracle.TiDBCERTOracle; -import sqlancer.tidb.oracle.TiDBDQPOracle; -import sqlancer.tidb.oracle.TiDBTLPHavingOracle; -import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @Parameters(separators = "=", commandDescription = "TiDB (default port: " + TiDBOptions.DEFAULT_PORT + ", default host: " + TiDBOptions.DEFAULT_HOST + ")") @@ -40,48 +29,6 @@ public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = { "--tiflash" }, description = "Enable TiFlash") public boolean tiflash; - public enum TiDBOracleFactory implements OracleFactory { - HAVING { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBTLPHavingOracle(globalState); - } - }, - WHERE { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBTLPWhereOracle(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new TiDBTLPWhereOracle(globalState)); - oracles.add(new TiDBTLPHavingOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - CERT { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBCERTOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - DQP { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBDQPOracle(globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return oracle; diff --git a/src/sqlancer/tidb/TiDBOracleFactory.java b/src/sqlancer/tidb/TiDBOracleFactory.java new file mode 100644 index 000000000..fac172adf --- /dev/null +++ b/src/sqlancer/tidb/TiDBOracleFactory.java @@ -0,0 +1,64 @@ +package sqlancer.tidb; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.tidb.oracle.TiDBCERTOracle; +import sqlancer.tidb.oracle.TiDBDQPOracle; +import sqlancer.tidb.oracle.TiDBTLPHavingOracle; + +public enum TiDBOracleFactory implements OracleFactory { + HAVING { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + return new TiDBTLPHavingOracle(globalState); + } + }, + WHERE { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CERT { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + return new TiDBCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + DQP { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + return new TiDBDQPOracle(globalState); + } + }; + +} diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 05ef0d9c6..1d117e234 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -23,7 +23,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.gen.TiDBAlterTableGenerator; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java deleted file mode 100644 index 61ec16788..000000000 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.tidb.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.tidb.TiDBErrors; -import sqlancer.tidb.TiDBExpressionGenerator; -import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.TiDBSchema; -import sqlancer.tidb.TiDBSchema.TiDBColumn; -import sqlancer.tidb.TiDBSchema.TiDBTable; -import sqlancer.tidb.ast.TiDBExpression; -import sqlancer.tidb.ast.TiDBJoin; -import sqlancer.tidb.ast.TiDBSelect; - -public class TiDBTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public TiDBTLPWhereOracle(TiDBGlobalState state) { - TiDBExpressionGenerator gen = new TiDBExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/yugabyte/ycql/YCQLOptions.java b/src/sqlancer/yugabyte/ycql/YCQLOptions.java index df4854757..9d865aca2 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLOptions.java +++ b/src/sqlancer/yugabyte/ycql/YCQLOptions.java @@ -1,6 +1,5 @@ package sqlancer.yugabyte.ycql; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,11 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.yugabyte.ycql.YCQLOptions.YCQLOracleFactory; -import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; -import sqlancer.yugabyte.ycql.test.YCQLFuzzer; @Parameters(separators = "=", commandDescription = "YCQL (default port: " + YCQLOptions.DEFAULT_PORT + ", default host: " + YCQLOptions.DEFAULT_HOST) @@ -34,16 +28,6 @@ public class YCQLOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracles = Arrays.asList(YCQLOracleFactory.FUZZER); - public enum YCQLOracleFactory implements OracleFactory { - FUZZER { - @Override - public TestOracle create(YCQLGlobalState globalState) throws SQLException { - return new YCQLFuzzer(globalState); - } - - } - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java b/src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java new file mode 100644 index 000000000..beaf43704 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java @@ -0,0 +1,18 @@ +package sqlancer.yugabyte.ycql; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ycql.test.YCQLFuzzer; + +public enum YCQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YCQLProvider.YCQLGlobalState globalState) + throws SQLException { + return new YCQLFuzzer(globalState); + } + + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLOptions.java b/src/sqlancer/yugabyte/ysql/YSQLOptions.java index 6b46b09bf..71b886ba5 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLOptions.java +++ b/src/sqlancer/yugabyte/ysql/YSQLOptions.java @@ -1,7 +1,5 @@ package sqlancer.yugabyte.ysql; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,17 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.yugabyte.ysql.YSQLOptions.YSQLOracleFactory; -import sqlancer.yugabyte.ysql.oracle.YSQLCatalog; -import sqlancer.yugabyte.ysql.oracle.YSQLFuzzer; -import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; -import sqlancer.yugabyte.ysql.oracle.YSQLPivotedQuerySynthesisOracle; -import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPAggregateOracle; -import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPHavingOracle; -import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPWhereOracle; @Parameters(separators = "=", commandDescription = "YSQL (default port: " + YSQLOptions.DEFAULT_PORT + ", default host: " + YSQLOptions.DEFAULT_HOST) @@ -45,54 +32,4 @@ public List getTestOracleFactory() { return oracle; } - public enum YSQLOracleFactory implements OracleFactory { - FUZZER { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLFuzzer(globalState); - } - }, - CATALOG { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLCatalog(globalState); - } - }, - NOREC { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLNoRECOracle(globalState); - } - }, - PQS { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLPivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - HAVING { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new YSQLTLPWhereOracle(globalState)); - oracles.add(new YSQLTLPHavingOracle(globalState)); - oracles.add(new YSQLTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - } - - } - } diff --git a/src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java b/src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java new file mode 100644 index 000000000..b85f80593 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java @@ -0,0 +1,72 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; +import sqlancer.yugabyte.ysql.oracle.YSQLCatalog; +import sqlancer.yugabyte.ysql.oracle.YSQLFuzzer; +import sqlancer.yugabyte.ysql.oracle.YSQLPivotedQuerySynthesisOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPAggregateOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPHavingOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPWhereOracle; + +public enum YSQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLFuzzer(globalState); + } + }, + CATALOG { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLCatalog(globalState); + } + }, + NOREC { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) + .with(YSQLErrors.getCommonFetchErrors()).with("canceling statement due to statement timeout") + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + HAVING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); + oracles.add(new YSQLTLPWhereOracle(globalState)); + oracles.add(new YSQLTLPHavingOracle(globalState)); + oracles.add(new YSQLTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java deleted file mode 100644 index 70804ce40..000000000 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.yugabyte.ysql.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.yugabyte.ysql.YSQLErrors; -import sqlancer.yugabyte.ysql.YSQLGlobalState; -import sqlancer.yugabyte.ysql.YSQLSchema; -import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; -import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; -import sqlancer.yugabyte.ysql.ast.YSQLExpression; -import sqlancer.yugabyte.ysql.ast.YSQLJoin; -import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; - -public class YSQLNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public YSQLNoRECOracle(YSQLGlobalState globalState) { - YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) - .with(YSQLErrors.getCommonFetchErrors()).with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} From aed8cc1c317e31fe26a4643276b44126f1825ac7 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 25 Sep 2024 12:09:10 +0800 Subject: [PATCH 1002/1284] [YSQL] Implement TLP-where expression generator interface --- .../ysql/gen/YSQLExpressionGenerator.java | 22 ++++++++++++++----- .../ysql/gen/YSQLRandomQueryGenerator.java | 2 +- .../YSQLPivotedQuerySynthesisOracle.java | 2 +- .../oracle/tlp/YSQLTLPAggregateOracle.java | 2 +- .../ysql/oracle/tlp/YSQLTLPHavingOracle.java | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java index 759d4d3ba..80c35c431 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java @@ -12,6 +12,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.yugabyte.ysql.YSQLCompoundDataType; import sqlancer.yugabyte.ysql.YSQLGlobalState; @@ -47,7 +48,8 @@ import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; public class YSQLExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private final int maxDepth; @@ -182,7 +184,8 @@ public YSQLExpression generateExpression(int depth) { return generateExpression(depth, YSQLDataType.getRandomType()); } - public List generateOrderBy() { + @Override + public List generateOrderBys() { List orderBys = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber(); i++) { orderBys.add(new YSQLOrderByTerm(YSQLColumnValue.create(Randomly.fromList(columns), null), @@ -584,7 +587,7 @@ public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); @@ -599,8 +602,7 @@ public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public YSQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -658,7 +660,7 @@ public String generateOptimizedQueryString(YSQLSelect select, YSQLExpression whe YSQLColumnValue allColumns = new YSQLColumnValue(Randomly.fromList(columns), null); select.setFetchColumns(Arrays.asList(allColumns)); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(generateOrderBy()); + select.setOrderByClauses(generateOrderBys()); } select.setWhereClause(whereCondition); } @@ -676,4 +678,12 @@ public String generateUnoptimizedQueryString(YSQLSelect select, YSQLExpression w return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBooleanWithRatherLowProbability()) { + return List.of(new YSQLColumnValue(YSQLColumn.createDummy("*"), null)); + } + return columns.stream().map(c -> new YSQLColumnValue(c, null)).collect(Collectors.toList()); + } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java index 02e87dffd..9821dcb7d 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java @@ -45,7 +45,7 @@ public static YSQLSelect createRandomQuery(int nrColumns, YSQLGlobalState global } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java index ac321d7c3..8f54aa698 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java @@ -134,7 +134,7 @@ public SQLQueryAdapter getRectifiedQuery() throws SQLException { YSQLExpression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } - List orderBy = new YSQLExpressionGenerator(globalState).setColumns(columns).generateOrderBy(); + List orderBy = new YSQLExpressionGenerator(globalState).setColumns(columns).generateOrderBys(); selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(YSQLVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java index 6e9b74f87..3fdaf3f9a 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -61,7 +61,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = YSQLVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java index d4a398cf2..32df70b11 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java @@ -43,7 +43,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setHavingClause(predicate); String firstQueryString = YSQLVisitor.asString(select); From 510df57e895e16ced536a0466e1e8014e7a95130 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 25 Sep 2024 12:09:28 +0800 Subject: [PATCH 1003/1284] [YSQL] Use common TLP-where oracle --- .../ysql/oracle/tlp/YSQLTLPWhereOracle.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java index 0d61dd8c5..265586586 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java @@ -1,45 +1,45 @@ package sqlancer.yugabyte.ysql.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; -import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.YSQLSchema; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; -public class YSQLTLPWhereOracle extends YSQLTLPBase { +public class YSQLTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public YSQLTLPWhereOracle(YSQLGlobalState state) { - super(state); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) + .with(YSQLErrors.getCommonFetchErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - whereCheck(); + oracle.check(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - protected void whereCheck() throws SQLException { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - String originalQueryString = YSQLVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = YSQLVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = YSQLVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = YSQLVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From e0d072bec939074cc2ee0ab0ddaded5d563fd075 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 1 Oct 2024 16:12:46 +0800 Subject: [PATCH 1004/1284] [Postgres] Add CERT test --- .github/workflows/main.yml | 1 + test/sqlancer/dbms/TestPostgresCERT.java | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 test/sqlancer/dbms/TestPostgresCERT.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68ed03dc6..2387644fd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -482,6 +482,7 @@ jobs: POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresPQS test POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresTLP test POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresNoREC test + POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresCERT test presto: name: DBMS Tests (Presto) diff --git a/test/sqlancer/dbms/TestPostgresCERT.java b/test/sqlancer/dbms/TestPostgresCERT.java new file mode 100644 index 000000000..afa5e3801 --- /dev/null +++ b/test/sqlancer/dbms/TestPostgresCERT.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPostgresCERT { + + @Test + public void testCERT() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", + "false", "--oracle", "CERT" })); + } +} From eefcde54c4ec9df85ce0d20c934cdc93067a7ae8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 1 Oct 2024 16:26:43 +0800 Subject: [PATCH 1005/1284] [Postgres] Add bug 18643 --- src/sqlancer/postgres/PostgresBugs.java | 10 ++++++++++ .../postgres/oracle/PostgresCERTOracle.java | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 src/sqlancer/postgres/PostgresBugs.java diff --git a/src/sqlancer/postgres/PostgresBugs.java b/src/sqlancer/postgres/PostgresBugs.java new file mode 100644 index 000000000..352d6c1c5 --- /dev/null +++ b/src/sqlancer/postgres/PostgresBugs.java @@ -0,0 +1,10 @@ +package sqlancer.postgres; + +// do not make the fields final to avoid warnings +public final class PostgresBugs { + public static boolean bug18643 = true; + + private PostgresBugs() { + } + +} diff --git a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java index 9b509834e..d22689816 100644 --- a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java @@ -14,6 +14,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.postgres.PostgresBugs; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -77,14 +78,19 @@ public void check() throws SQLException { // First query row count String queryString1 = PostgresVisitor.asString(select); - int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + long rowCount1 = getRow(state, queryString1, queryPlan1Sequences); // JOIN and LIMIT mutations not added - boolean increase = mutate(Mutator.LIMIT); + boolean increase; + if (PostgresBugs.bug18643) { + increase = mutate(Mutator.LIMIT, Mutator.OR, Mutator.AND); + } else { + increase = mutate(Mutator.LIMIT); + } // Second Query row count String queryString2 = PostgresVisitor.asString(select); - int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + long rowCount2 = getRow(state, queryString2, queryPlan2Sequences); // Check query plan equivalence if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { @@ -216,9 +222,9 @@ protected boolean mutateLimit() { return increase; } - private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + private long getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) throws AssertionError, SQLException { - int row = -1; + long row = -1; String explainQuery = "EXPLAIN " + selectStr; if (globalState.getOptions().logEachSelect()) { @@ -239,7 +245,7 @@ private int getRow(SQLGlobalState globalState, String selectStr, List Date: Wed, 2 Oct 2024 11:55:51 +0800 Subject: [PATCH 1006/1284] [TiDB] Add CERT test --- .github/workflows/main.yml | 4 +++- test/sqlancer/dbms/TestTiDBCERT.java | 19 +++++++++++++++++++ .../dbms/{TestTiDB.java => TestTiDBTLP.java} | 8 +++----- 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 test/sqlancer/dbms/TestTiDBCERT.java rename test/sqlancer/dbms/{TestTiDB.java => TestTiDBTLP.java} (63%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68ed03dc6..2e2dae97e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -574,7 +574,9 @@ jobs: - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests - run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test + run: | + TIDB_AVAILABLE=true mvn -Dtest=TestTiDBTLP test + TIDB_AVAILABLE=true mvn -Dtest=TestTiDBCERT test tidb-qpg: name: QPG Tests (TiDB) diff --git a/test/sqlancer/dbms/TestTiDBCERT.java b/test/sqlancer/dbms/TestTiDBCERT.java new file mode 100644 index 000000000..748444870 --- /dev/null +++ b/test/sqlancer/dbms/TestTiDBCERT.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestTiDBCERT { + + @Test + public void testCERT() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.TIDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-queries", "4", "tidb", "--oracle", "CERT" })); + } + +} diff --git a/test/sqlancer/dbms/TestTiDB.java b/test/sqlancer/dbms/TestTiDBTLP.java similarity index 63% rename from test/sqlancer/dbms/TestTiDB.java rename to test/sqlancer/dbms/TestTiDBTLP.java index b97e59273..11cf7581e 100644 --- a/test/sqlancer/dbms/TestTiDB.java +++ b/test/sqlancer/dbms/TestTiDBTLP.java @@ -7,13 +7,11 @@ import sqlancer.Main; -public class TestTiDB { +public class TestTiDBTLP { @Test - public void testMySQL() { - String tiDB = System.getenv("TIDB_AVAILABLE"); - boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); - assumeTrue(tiDBIsAvailable); + public void testTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.TIDB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "tidb" })); } From 648bc945b2bdf2b169740aa7b950d211d9f6b2b2 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 2 Oct 2024 11:56:25 +0800 Subject: [PATCH 1007/1284] [TiDB] Handle expected errors --- src/sqlancer/tidb/TiDBSchema.java | 2 ++ src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java | 3 ++- src/sqlancer/tidb/gen/TiDBDeleteGenerator.java | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 265cacbda..7c808beee 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -251,6 +251,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { primitiveType = TiDBDataType.INT; size = 1; break; + case "tinyint(2)": case "tinyint(3)": case "tinyint(4)": primitiveType = TiDBDataType.INT; @@ -267,6 +268,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { size = 4; break; case "blob": + case "mediumblob": case "longblob": case "tinyblob": primitiveType = TiDBDataType.BLOB; diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index f78878ac0..8b43c635a 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -5,6 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -14,7 +15,7 @@ private TiDBAnalyzeTableGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - ExpectedErrors errors = new ExpectedErrors(); + ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); StringBuilder sb = new StringBuilder("ANALYZE TABLE "); diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index f6eea4671..c3986f8d9 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -18,7 +18,7 @@ private TiDBDeleteGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - ExpectedErrors errors = new ExpectedErrors(); + ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); StringBuilder sb = new StringBuilder("DELETE "); From bd45822bc8d6083a0bdb6569f91d625de6fba074 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 2 Oct 2024 11:56:54 +0800 Subject: [PATCH 1008/1284] [TiDB] Add bugs 38319 and 51525 --- src/sqlancer/tidb/TiDBBugs.java | 6 ++++++ src/sqlancer/tidb/gen/TiDBViewGenerator.java | 8 +++++++- src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 11 ++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index 81f27b705..3849a3c8c 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -14,6 +14,9 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/38295 public static boolean bug38295 = true; + // https://github.com/pingcap/tidb/issues/38319 + public static boolean bug38319 = true; + // https://github.com/pingcap/tidb/issues/44747 public static boolean bug44747 = true; @@ -32,6 +35,9 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/47348 public static boolean bug47348 = true; + // https://github.com/pingcap/tidb/issues/51525 + public static boolean bug51525 = true; + private TiDBBugs() { } diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index fdbbc0839..79c284ecd 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -4,8 +4,10 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.ast.TiDBSelect; public final class TiDBViewGenerator { @@ -37,7 +39,11 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append(i); } sb.append(") AS "); - sb.append(TiDBRandomQuerySynthesizer.generate(globalState, nrColumns).getQueryString()); + TiDBSelect select = TiDBRandomQuerySynthesizer.generateSelect(globalState, nrColumns); + if (TiDBBugs.bug38319 && !select.getGroupByExpressions().isEmpty()) { + throw new IgnoreMeException(); + } + sb.append(select.asString()); ExpectedErrors errors = new ExpectedErrors(); TiDBErrors.addExpressionErrors(errors); errors.add( diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java index 4fd5c8c54..b55c1df1f 100644 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -14,6 +14,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -82,8 +83,16 @@ public void check() throws SQLException { String queryString1 = TiDBVisitor.asString(select); double rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + List excludes = new ArrayList<>(); + excludes.add(Mutator.DISTINCT); + if (TiDBBugs.bug51525) { + excludes.add(Mutator.OR); + } + if (TiDBBugs.bug38319) { + excludes.add(Mutator.GROUPBY); + } // Mutate the query - boolean increase = mutate(Mutator.DISTINCT); + boolean increase = mutate(excludes.toArray(new Mutator[0])); // Get the result of the second query String queryString2 = TiDBVisitor.asString(select); From 880ec6f3f26985ade2217606a591ee9a2cbe4e47 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 1 Oct 2024 14:23:42 +0800 Subject: [PATCH 1009/1284] [CockroachDB] Add CERT test --- .github/workflows/main.yml | 1 + test/sqlancer/dbms/TestCockroachDBCERT.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 test/sqlancer/dbms/TestCockroachDBCERT.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6842b20cc..881ccf689 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -173,6 +173,7 @@ jobs: run: | COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBNoREC test COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBTLP test + COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBCERT test cockroachdb-qpg: name: QPG Tests (CockroachDB) diff --git a/test/sqlancer/dbms/TestCockroachDBCERT.java b/test/sqlancer/dbms/TestCockroachDBCERT.java new file mode 100644 index 000000000..7e60818c1 --- /dev/null +++ b/test/sqlancer/dbms/TestCockroachDBCERT.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCockroachDBCERT { + + @Test + public void testCockroachDBCERT() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "CERT" })); + } + +} From fb3deab1b537402e6bfea33373caa474fafc5d79 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 1 Oct 2024 14:24:19 +0800 Subject: [PATCH 1010/1284] [CockroachDB] Update CERT oracle --- .../cockroachdb/oracle/CockroachDBCERTOracle.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java index 86d5d75ce..731d6cb14 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -132,11 +132,14 @@ protected boolean mutateJoin() { } JoinType newJoinType = CockroachDBJoin.JoinType.INNER; - if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invariant relation // between LEFT and RIGHT // join - newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); - } else { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS, JoinType.LEFT, + JoinType.RIGHT); + } else if (join.getJoinType() == JoinType.FULL) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS); + } else if (join.getJoinType() != JoinType.CROSS) { newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); } assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT @@ -230,7 +233,7 @@ protected boolean mutateLimit() { private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) throws AssertionError, SQLException { int row = -1; - String explainQuery = "EXPLAIN (VERBOSE) " + selectStr; + String explainQuery = "EXPLAIN " + selectStr; // Log the query if (globalState.getOptions().logEachSelect()) { From 012f4582a97de0119b314d4e35256b394306a270 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 4 Oct 2024 14:12:14 +0800 Subject: [PATCH 1011/1284] [CockroachDB] Add bugs 131640, 131647 and 131875 --- src/sqlancer/cockroachdb/CockroachDBBugs.java | 9 +++++++++ .../oracle/CockroachDBCERTOracle.java | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 6c34bcac4..86d88ee21 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -59,6 +59,15 @@ public final class CockroachDBBugs { // https://github.com/cockroachdb/cockroach/issues/85230 public static boolean bug85230 = true; + // https://github.com/cockroachdb/cockroach/issues/131640 + public static boolean bug131640 = true; + + // https://github.com/cockroachdb/cockroach/issues/131647 + public static boolean bug131647 = true; + + // https://github.com/cockroachdb/cockroach/issues/131875 + public static boolean bug131875 = true; + private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java index 731d6cb14..07a51861f 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLGlobalState; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -76,9 +77,17 @@ public void check() throws SQLException { String queryString1 = CockroachDBVisitor.asString(select); int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); - // Mutate the query + List excludes = new ArrayList<>(); // Disable limit due to its false positive - boolean increase = mutate(); + excludes.add(Mutator.LIMIT); + if (CockroachDBBugs.bug131640) { + excludes.add(Mutator.OR); + } + if (CockroachDBBugs.bug131647) { + excludes.add(Mutator.JOIN); + } + // Mutate the query + boolean increase = mutate(excludes.toArray(new Mutator[0])); // Get the result of the second query String queryString2 = CockroachDBVisitor.asString(select); @@ -262,10 +271,15 @@ private int getRow(SQLGlobalState globalState, String selectStr, List Date: Mon, 7 Oct 2024 09:45:12 +0000 Subject: [PATCH 1012/1284] Bump org.junit.jupiter:junit-jupiter-engine from 5.9.1 to 5.11.2 Bumps [org.junit.jupiter:junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.9.1 to 5.11.2. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.9.1...r5.11.2) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7b97a3a0d..119ef6434 100644 --- a/pom.xml +++ b/pom.xml @@ -324,7 +324,7 @@ org.junit.jupiter junit-jupiter-engine - 5.9.1 + 5.11.2 test From b0f16c9ab63b4178390cc1a711e3792dae6fec26 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 09:55:04 +0800 Subject: [PATCH 1013/1284] [MySQL] Add CERT test --- .github/workflows/main.yml | 1 + test/sqlancer/dbms/TestMySQLCERT.java | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 test/sqlancer/dbms/TestMySQLCERT.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68ed03dc6..b1127faa7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -429,6 +429,7 @@ jobs: run: | MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP + MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLCERT oceanbase: name: DBMS Tests (OceanBase) diff --git a/test/sqlancer/dbms/TestMySQLCERT.java b/test/sqlancer/dbms/TestMySQLCERT.java new file mode 100644 index 000000000..a7704a545 --- /dev/null +++ b/test/sqlancer/dbms/TestMySQLCERT.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMySQLCERT { + + @Test + public void testMySQL() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); + // Run with 0 queries as there are false positives for every mutation + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--max-expression-depth", "1", "--num-threads", "1", "--num-queries", "0", "mysql", "--oracle", + "CERT" })); + } + +} From 7fa58055f394f4511bb0e333686e03c460a7b781 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 14:13:30 +0800 Subject: [PATCH 1014/1284] Add common CERT oracle --- src/sqlancer/common/gen/CERTGenerator.java | 29 +++++ src/sqlancer/common/oracle/CERTOracle.java | 133 +++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 src/sqlancer/common/gen/CERTGenerator.java create mode 100644 src/sqlancer/common/oracle/CERTOracle.java diff --git a/src/sqlancer/common/gen/CERTGenerator.java b/src/sqlancer/common/gen/CERTGenerator.java new file mode 100644 index 000000000..b272ba4b2 --- /dev/null +++ b/src/sqlancer/common/gen/CERTGenerator.java @@ -0,0 +1,29 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface CERTGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + CERTGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + String generateExplainQuery(S select); + + boolean mutate(S select); +} diff --git a/src/sqlancer/common/oracle/CERTOracle.java b/src/sqlancer/common/oracle/CERTOracle.java new file mode 100644 index 000000000..48c174ef2 --- /dev/null +++ b/src/sqlancer/common/oracle/CERTOracle.java @@ -0,0 +1,133 @@ +package sqlancer.common.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class CERTOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private final CheckedFunction> rowCountParser; + private final CheckedFunction> queryPlanParser; + + private CERTGenerator gen; + private final ExpectedErrors errors; + + public CERTOracle(G state, CERTGenerator gen, ExpectedErrors expectedErrors, + CheckedFunction> rowCountParser, + CheckedFunction> queryPlanParser) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + this.rowCountParser = rowCountParser; + this.queryPlanParser = queryPlanParser; + } + + @Override + public void check() throws SQLException { + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + List fetchColumns = gen.generateFetchColumns(false); + + Z select = gen.generateSelect(); + select.setFetchColumns(fetchColumns); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateBooleanExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByClause(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateBooleanExpression()); + } + } + + List queryPlan1Sequences = new ArrayList<>(); + List queryPlan2Sequences = new ArrayList<>(); + + String queryString1 = gen.generateExplainQuery(select); + long rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + boolean increase = gen.mutate(select); + String queryString2 = gen.generateExplainQuery(select); + long rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check the results + if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { + throw new AssertionError("Inconsistent result for query: " + queryString1 + "; --" + rowCount1 + "\n" + + queryString2 + "; --" + rowCount2); + } + } + + private Long getRow(SQLGlobalState globalState, String explainQuery, List queryPlanSequences) + throws AssertionError, SQLException { + Optional row = Optional.empty(); + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + Optional rowCount = rowCountParser.apply(rs); + if (row.isEmpty() && rowCount.isPresent()) { + row = rowCount; + } + + Optional queryPlanSequence = queryPlanParser.apply(rs); + queryPlanSequence.ifPresent(qps -> queryPlanSequences.add(qps)); + } + } + } catch (IgnoreMeException e) { + throw new IgnoreMeException(); + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + + return row.orElseThrow(IgnoreMeException::new); + } + + @FunctionalInterface + public interface CheckedFunction { + R apply(T t) throws SQLException; + } +} From 62a22d7bf80c41fcc9331f03f7c63916c1b7e2c9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 14:14:01 +0800 Subject: [PATCH 1015/1284] [TiDB] Use common CERT oracle --- .../tidb/TiDBExpressionGenerator.java | 278 +++++++++++++----- src/sqlancer/tidb/TiDBOracleFactory.java | 17 +- src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 267 ----------------- 3 files changed, 222 insertions(+), 340 deletions(-) delete mode 100644 src/sqlancer/tidb/oracle/TiDBCERTOracle.java diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index 0db285284..ff6a529d0 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -3,10 +3,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.CERTGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; @@ -31,6 +33,7 @@ import sqlancer.tidb.ast.TiDBFunctionCall; import sqlancer.tidb.ast.TiDBFunctionCall.TiDBFunction; import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBJoin.JoinType; import sqlancer.tidb.ast.TiDBOrderingTerm; import sqlancer.tidb.ast.TiDBRegexOperation; import sqlancer.tidb.ast.TiDBRegexOperation.TiDBRegexOperator; @@ -42,14 +45,8 @@ import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; public class TiDBExpressionGenerator extends UntypedExpressionGenerator - implements TLPWhereGenerator { - - private final TiDBGlobalState globalState; - private List tables; - - public TiDBExpressionGenerator(TiDBGlobalState globalState) { - this.globalState = globalState; - } + implements TLPWhereGenerator, + CERTGenerator { private enum Gen { UNARY_PREFIX, // @@ -60,68 +57,12 @@ private enum Gen { // BINARY_ARITHMETIC } - @Override - protected TiDBExpression generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - if (allowAggregates && Randomly.getBoolean()) { - allowAggregates = false; - TiDBAggregateFunction func = TiDBAggregateFunction.getRandom(); - List args = generateExpressions(func.getNrArgs()); - return new TiDBAggregate(args, func); - } - switch (Randomly.fromOptions(Gen.values())) { - case DEFAULT: - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - TiDBColumn column = Randomly.fromList(columns); - if (column.hasDefault()) { - return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(new TiDBColumnReference(column))); - } - throw new IgnoreMeException(); - case UNARY_POSTFIX: - return new TiDBUnaryPostfixOperation(generateExpression(depth + 1), TiDBUnaryPostfixOperator.getRandom()); - case UNARY_PREFIX: - TiDBUnaryPrefixOperator rand = TiDBUnaryPrefixOperator.getRandom(); - return new TiDBUnaryPrefixOperation(generateExpression(depth + 1), rand); - case COLUMN: - return generateColumn(); - case CONSTANT: - return generateConstant(); - case COMPARISON: - return new TiDBBinaryComparisonOperation(generateExpression(depth + 1), generateExpression(depth + 1), - TiDBComparisonOperator.getRandom()); - case REGEX: - return new TiDBRegexOperation(generateExpression(depth + 1), generateExpression(depth + 1), - TiDBRegexOperator.getRandom()); - case FUNCTION: - TiDBFunction func = TiDBFunction.getRandom(); - return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); - case BINARY_BIT: - return new TiDBBinaryBitOperation(generateExpression(depth + 1), generateExpression(depth + 1), - TiDBBinaryBitOperator.getRandom()); - case BINARY_LOGICAL: - return new TiDBBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), - TiDBBinaryLogicalOperator.getRandom()); - case CAST: - return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions("BINARY", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 - "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 - "DECIMAL", "SIGNED", "UNSIGNED" /* https://github.com/pingcap/tidb/issues/16028 */)); - case CASE: - int nr = Randomly.fromOptions(1, 2); - return new TiDBCase(generateExpression(depth + 1), generateExpressions(nr, depth + 1), - generateExpressions(nr, depth + 1), generateExpression(depth + 1)); - default: - throw new AssertionError(); - } - } + private final TiDBGlobalState globalState; - @Override - protected TiDBExpression generateColumn() { - TiDBColumn column = Randomly.fromList(columns); - return new TiDBColumnReference(column); + private List tables; + + public TiDBExpressionGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; } @Override @@ -199,8 +140,7 @@ public TiDBExpression generateConstant(TiDBDataType type) { } @Override - public TLPWhereGenerator setTablesAndColumns( - AbstractTables tables) { + public TiDBExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -240,4 +180,200 @@ public List generateFetchColumns(boolean shouldCreateDummy) { return Randomly.nonEmptySubset(this.columns).stream().map(c -> new TiDBColumnReference(c)) .collect(Collectors.toList()); } + + @Override + protected TiDBExpression generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBoolean()) { + allowAggregates = false; + TiDBAggregateFunction func = TiDBAggregateFunction.getRandom(); + List args = generateExpressions(func.getNrArgs()); + return new TiDBAggregate(args, func); + } + switch (Randomly.fromOptions(Gen.values())) { + case DEFAULT: + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + TiDBColumn column = Randomly.fromList(columns); + if (column.hasDefault()) { + return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(new TiDBColumnReference(column))); + } + throw new IgnoreMeException(); + case UNARY_POSTFIX: + return new TiDBUnaryPostfixOperation(generateExpression(depth + 1), TiDBUnaryPostfixOperator.getRandom()); + case UNARY_PREFIX: + TiDBUnaryPrefixOperator rand = TiDBUnaryPrefixOperator.getRandom(); + return new TiDBUnaryPrefixOperation(generateExpression(depth + 1), rand); + case COLUMN: + return generateColumn(); + case CONSTANT: + return generateConstant(); + case COMPARISON: + return new TiDBBinaryComparisonOperation(generateExpression(depth + 1), generateExpression(depth + 1), + TiDBComparisonOperator.getRandom()); + case REGEX: + return new TiDBRegexOperation(generateExpression(depth + 1), generateExpression(depth + 1), + TiDBRegexOperator.getRandom()); + case FUNCTION: + TiDBFunction func = TiDBFunction.getRandom(); + return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); + case BINARY_BIT: + return new TiDBBinaryBitOperation(generateExpression(depth + 1), generateExpression(depth + 1), + TiDBBinaryBitOperator.getRandom()); + case BINARY_LOGICAL: + return new TiDBBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), + TiDBBinaryLogicalOperator.getRandom()); + case CAST: + return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions("BINARY", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 + "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 + "DECIMAL", "SIGNED", "UNSIGNED" /* https://github.com/pingcap/tidb/issues/16028 */)); + case CASE: + int nr = Randomly.fromOptions(1, 2); + return new TiDBCase(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(); + } + } + + @Override + protected TiDBExpression generateColumn() { + TiDBColumn column = Randomly.fromList(columns); + return new TiDBColumnReference(column); + } + + @Override + public String generateExplainQuery(TiDBSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(TiDBSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateJoin); + mutators.add(this::mutateWhere); + if (!TiDBBugs.bug38319) { + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + } + mutators.add(this::mutateAnd); + if (!TiDBBugs.bug51525) { + mutators.add(this::mutateOr); + } + mutators.add(this::mutateLimit); + // mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateJoin(TiDBSelect select) { + if (select.getJoinList().isEmpty()) { + return false; + } + TiDBJoin join = (TiDBJoin) Randomly.fromList(select.getJoinList()); + if (join.getJoinType() == JoinType.NATURAL) { + return false; + } + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to + // other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((TiDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((TiDBTableReference) join.getRightTable()).getTable().getColumns()); + TiDBExpressionGenerator joinGen2 = new TiDBExpressionGenerator(globalState).setColumns(columns); + join.setOnCondition(joinGen2.generateExpression()); + } + + JoinType newJoinType = TiDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation + // between LEFT and RIGHT + // join + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); + } else { + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); + } + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + if (newJoinType == JoinType.CROSS) { + join.setOnCondition(null); + } + return increase; + } + + boolean mutateWhere(TiDBSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression()); + } + return increase; + } + + boolean mutateHaving(TiDBSelect select) { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(TiDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + TiDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(TiDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + return false; + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + TiDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + boolean mutateLimit(TiDBSelect select) { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(generateConstant(TiDBDataType.INT)); + } + return increase; + } + + private boolean mutateGroupBy(TiDBSelect select) { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + select.clearHavingClause(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } } diff --git a/src/sqlancer/tidb/TiDBOracleFactory.java b/src/sqlancer/tidb/TiDBOracleFactory.java index fac172adf..173ff7abd 100644 --- a/src/sqlancer/tidb/TiDBOracleFactory.java +++ b/src/sqlancer/tidb/TiDBOracleFactory.java @@ -3,13 +3,15 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import sqlancer.OracleFactory; +import sqlancer.common.oracle.CERTOracle; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.tidb.oracle.TiDBCERTOracle; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.tidb.oracle.TiDBDQPOracle; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; @@ -45,7 +47,18 @@ public TestOracle create(TiDBProvider.TiDBGlobalSt @Override public TestOracle create(TiDBProvider.TiDBGlobalState globalState) throws SQLException { - return new TiDBCERTOracle(globalState); + TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + String content = rs.getString(2); + return Optional.of((long) Double.parseDouble(content)); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String operation = rs.getString(1).split("_")[0]; // Extract operation names for query plans + return Optional.of(operation); + }; + + return new CERTOracle<>(globalState, gen, expectedErrors, rowCountParser, queryPlanParser); } @Override diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java deleted file mode 100644 index b55c1df1f..000000000 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ /dev/null @@ -1,267 +0,0 @@ -package sqlancer.tidb.oracle; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLGlobalState; -import sqlancer.common.DBMSCommon; -import sqlancer.common.oracle.CERTOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.tidb.TiDBBugs; -import sqlancer.tidb.TiDBErrors; -import sqlancer.tidb.TiDBExpressionGenerator; -import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.TiDBSchema.TiDBColumn; -import sqlancer.tidb.TiDBSchema.TiDBDataType; -import sqlancer.tidb.TiDBSchema.TiDBTables; -import sqlancer.tidb.ast.TiDBBinaryLogicalOperation; -import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; -import sqlancer.tidb.ast.TiDBColumnReference; -import sqlancer.tidb.ast.TiDBExpression; -import sqlancer.tidb.ast.TiDBJoin; -import sqlancer.tidb.ast.TiDBJoin.JoinType; -import sqlancer.tidb.ast.TiDBSelect; -import sqlancer.tidb.ast.TiDBTableReference; -import sqlancer.tidb.visitor.TiDBVisitor; - -public class TiDBCERTOracle extends CERTOracleBase implements TestOracle { - private TiDBExpressionGenerator gen; - private TiDBSelect select; - - public TiDBCERTOracle(TiDBGlobalState globalState) { - super(globalState); - TiDBErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - queryPlan1Sequences = new ArrayList<>(); - queryPlan2Sequences = new ArrayList<>(); - - // Randomly generate a query - TiDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); - gen = new TiDBExpressionGenerator(state).setColumns(tables.getColumns()); - select = new TiDBSelect(); - - List fetchColumns = new ArrayList<>(); - fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new TiDBColumnReference(c)) - .collect(Collectors.toList())); - select.setFetchColumns(fetchColumns); - - List tableList = tables.getTables().stream().map(t -> new TiDBTableReference(t)) - .collect(Collectors.toList()); - List joins = TiDBJoin.getJoinsWithoutNature(tableList, state); - select.setJoinList(joins); - select.setFromList(tableList); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - if (Randomly.getBoolean()) { - select.setGroupByExpressions(select.getFetchColumns()); - if (Randomly.getBoolean()) { - select.setHavingClause(gen.generateHavingClause()); - } - } - if (Randomly.getBoolean()) { - select.setLimitClause(gen.generateExpression()); - } - if (Randomly.getBoolean()) { - select.setOffsetClause(gen.generateExpression()); - } - - // Get the result of the first query - String queryString1 = TiDBVisitor.asString(select); - double rowCount1 = getRow(state, queryString1, queryPlan1Sequences); - - List excludes = new ArrayList<>(); - excludes.add(Mutator.DISTINCT); - if (TiDBBugs.bug51525) { - excludes.add(Mutator.OR); - } - if (TiDBBugs.bug38319) { - excludes.add(Mutator.GROUPBY); - } - // Mutate the query - boolean increase = mutate(excludes.toArray(new Mutator[0])); - - // Get the result of the second query - String queryString2 = TiDBVisitor.asString(select); - double rowCount2 = getRow(state, queryString2, queryPlan2Sequences); - - // Check structural equivalence - if (!DBMSCommon.areQueryPlanSequencesSimilar(queryPlan1Sequences, queryPlan2Sequences)) { - return; - } - - /* - * https://github.com/pingcap/tidb/issues/38474 A minor issue in TiDB that some operations would round the - * result while others would not. A false alarm happens when both queries have the same number of estimated rows - * but the restrictued one rounds up to a bigger number. To avoid this issue until it is fixed, we make sure the - * gap between two estimated rows is at least 1. - */ - // Check the results - if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { - throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 - + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); - } - } - - @Override - protected boolean mutateJoin() { - if (select.getJoinList().isEmpty()) { - return false; - } - TiDBJoin join = (TiDBJoin) Randomly.fromList(select.getJoinList()); - - // CROSS does not need ON Condition, while other joins do - // To avoid Null pointer, generating a new new condition when mutating CROSS to - // other joins - if (join.getJoinType() == JoinType.CROSS) { - List columns = new ArrayList<>(); - columns.addAll(((TiDBTableReference) join.getLeftTable()).getTable().getColumns()); - columns.addAll(((TiDBTableReference) join.getRightTable()).getTable().getColumns()); - TiDBExpressionGenerator joinGen2 = new TiDBExpressionGenerator(state).setColumns(columns); - join.setOnCondition(joinGen2.generateExpression()); - } - - JoinType newJoinType = TiDBJoin.JoinType.INNER; - if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation - // between LEFT and RIGHT - // join - newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); - } else { - newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); - } - assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT - boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); - join.setJoinType(newJoinType); - if (newJoinType == JoinType.CROSS) { - join.setOnCondition(null); - } - return increase; - } - - @Override - protected boolean mutateWhere() { - boolean increase = select.getWhereClause() != null; - if (increase) { - select.setWhereClause(null); - } else { - select.setWhereClause(gen.generateExpression()); - } - return increase; - } - - @Override - protected boolean mutateGroupBy() { - boolean increase = select.getGroupByExpressions().size() > 0; - if (increase) { - select.clearGroupByExpressions(); - select.clearHavingClause(); - } else { - select.setGroupByExpressions(select.getFetchColumns()); - } - return increase; - } - - @Override - protected boolean mutateHaving() { - if (select.getGroupByExpressions().size() == 0) { - select.setGroupByExpressions(select.getFetchColumns()); - select.setHavingClause(gen.generateExpression()); - return false; - } else { - if (select.getHavingClause() == null) { - select.setHavingClause(gen.generateExpression()); - return false; - } else { - select.setHavingClause(null); - return true; - } - } - } - - @Override - protected boolean mutateAnd() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression()); - } else { - TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), gen.generateExpression(), - TiDBBinaryLogicalOperator.AND); - select.setWhereClause(newWhere); - } - return false; - } - - @Override - protected boolean mutateOr() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression()); - return false; - } else { - TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), gen.generateExpression(), - TiDBBinaryLogicalOperator.OR); - select.setWhereClause(newWhere); - return true; - } - } - - @Override - protected boolean mutateLimit() { - boolean increase = select.getLimitClause() != null; - if (increase) { - select.setLimitClause(null); - } else { - select.setLimitClause(gen.generateConstant(TiDBDataType.INT)); - } - return increase; - } - - private double getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) - throws AssertionError, SQLException { - double row = -1; - String explainQuery = "EXPLAIN " + selectStr; - - // Log the query - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(explainQuery); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - // Get the row count - SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs != null) { - while (rs.next()) { - double estRows = Double.parseDouble(rs.getString(2)); - if (row == -1) { - row = estRows; - } - String operation = rs.getString(1).split("_")[0]; // Extract operation names for query plans - queryPlanSequences.add(operation); - return estRows; - } - } - } catch (Exception e) { - throw new AssertionError(q.getQueryString(), e); - } - if (row == -1) { - throw new IgnoreMeException(); - } - return row; - } -} From 856259c49ba2e88330dba5ad98d12ac91f549aea Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 15:22:45 +0800 Subject: [PATCH 1016/1284] [Postgres] Use common CERT oracle --- .../postgres/PostgresOracleFactory.java | 30 +- .../gen/PostgresExpressionGenerator.java | 140 ++++++++- .../postgres/oracle/PostgresCERTOracle.java | 272 ------------------ 3 files changed, 167 insertions(+), 275 deletions(-) delete mode 100644 src/sqlancer/postgres/oracle/PostgresCERTOracle.java diff --git a/src/sqlancer/postgres/PostgresOracleFactory.java b/src/sqlancer/postgres/PostgresOracleFactory.java index 45e2190fc..df2185776 100644 --- a/src/sqlancer/postgres/PostgresOracleFactory.java +++ b/src/sqlancer/postgres/PostgresOracleFactory.java @@ -3,16 +3,18 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import sqlancer.OracleFactory; +import sqlancer.common.oracle.CERTOracle; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; -import sqlancer.postgres.oracle.PostgresCERTOracle; import sqlancer.postgres.oracle.PostgresFuzzer; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; @@ -72,7 +74,31 @@ public TestOracle create(PostgresGlobalState globalState) t CERT { @Override public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresCERTOracle(globalState); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .with(PostgresCommon.getCommonFetchErrors()).with(PostgresCommon.getCommonInsertUpdateErrors()) + .with(PostgresCommon.getGroupingErrors()).with(PostgresCommon.getCommonInsertUpdateErrors()) + .with(PostgresCommon.getCommonRangeExpressionErrors()).build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + String content = rs.getString(1).trim(); + if (content.contains("rows=")) { + try { + int ind = content.indexOf("rows="); + long number = Long.parseLong(content.substring(ind + 5).split(" ")[0]); + return Optional.of(number); + } catch (Exception e) { + } + } + return Optional.empty(); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String content = rs.getString(1).trim(); + String[] planPart = content.split("-> "); + String plan = planPart[planPart.length - 1]; + return Optional.of(plan.split(" ")[0].trim()); + }; + return new CERTOracle<>(globalState, gen, errors, rowCountParser, queryPlanParser); } @Override diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 8c0c2352e..5f081ef2a 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -5,15 +5,18 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.CERTGenerator; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; +import sqlancer.postgres.PostgresBugs; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; @@ -63,10 +66,12 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; public class PostgresExpressionGenerator implements ExpressionGenerator, NoRECGenerator, - TLPWhereGenerator { + TLPWhereGenerator, + CERTGenerator { private final int maxDepth; @@ -739,4 +744,137 @@ public String generateUnoptimizedQueryString(PostgresSelect select, PostgresExpr return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public String generateExplainQuery(PostgresSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(PostgresSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateJoin); + mutators.add(this::mutateWhere); + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + if (!PostgresBugs.bug18643) { + mutators.add(this::mutateAnd); + mutators.add(this::mutateOr); + } + // mutators.add(this::mutateLimit); + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateJoin(PostgresSelect select) { + if (select.getJoinList().isEmpty()) { + return false; + } + PostgresJoin join = (PostgresJoin) Randomly.fromList(select.getJoinList()); + + // Exclude CROSS for on condition + if (join.getType() == PostgresJoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((PostgresTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((PostgresTableReference) join.getRightTable()).getTable().getColumns()); + PostgresExpressionGenerator joinGen2 = new PostgresExpressionGenerator(globalState).setColumns(columns); + join.setOnClause(joinGen2.generateExpression(0, PostgresDataType.BOOLEAN)); + } + + PostgresJoinType newJoinType = PostgresJoinType.INNER; + if (join.getType() == PostgresJoinType.LEFT || join.getType() == PostgresJoinType.RIGHT) { + newJoinType = PostgresJoinType.getRandomExcept(PostgresJoinType.LEFT, PostgresJoinType.RIGHT); + } else { + newJoinType = PostgresJoinType.getRandomExcept(join.getType()); + } + boolean increase = join.getType().ordinal() < newJoinType.ordinal(); + join.setType(newJoinType); + if (newJoinType == PostgresJoinType.CROSS) { + join.setOnClause(null); + } + return increase; + } + + boolean mutateDistinct(PostgresSelect select) { + PostgresSelect.SelectType selectType = select.getSelectOption(); + if (selectType != PostgresSelect.SelectType.ALL) { + select.setSelectType(PostgresSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(PostgresSelect.SelectType.DISTINCT); + return false; + } + } + + boolean mutateWhere(PostgresSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression(0, PostgresDataType.BOOLEAN)); + } + return increase; + } + + boolean mutateGroupBy(PostgresSelect select) { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + boolean mutateHaving(PostgresSelect select) { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(PostgresSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(0, PostgresDataType.BOOLEAN)); + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(PostgresSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + boolean mutateLimit(PostgresSelect select) { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + Randomly r = new Randomly(); + select.setLimitClause(PostgresConstant.createIntConstant((int) Math.abs(r.getInteger()))); + } + return increase; + } } diff --git a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java deleted file mode 100644 index d22689816..000000000 --- a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java +++ /dev/null @@ -1,272 +0,0 @@ -package sqlancer.postgres.oracle; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLGlobalState; -import sqlancer.common.DBMSCommon; -import sqlancer.common.oracle.CERTOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.PostgresBugs; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresDataType; -import sqlancer.postgres.PostgresSchema.PostgresTables; -import sqlancer.postgres.PostgresVisitor; -import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; -import sqlancer.postgres.ast.PostgresBinaryLogicalOperation.BinaryLogicalOperator; -import sqlancer.postgres.ast.PostgresColumnReference; -import sqlancer.postgres.ast.PostgresConstant; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.ast.PostgresTableReference; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; - -public class PostgresCERTOracle extends CERTOracleBase implements TestOracle { - private PostgresExpressionGenerator gen; - private PostgresSelect select; - - public PostgresCERTOracle(PostgresGlobalState globalState) { - super(globalState); - PostgresCommon.addCommonExpressionErrors(errors); - PostgresCommon.addCommonInsertUpdateErrors(errors); - PostgresCommon.addGroupingErrors(errors); - PostgresCommon.addCommonInsertUpdateErrors(errors); - PostgresCommon.addCommonRangeExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - queryPlan1Sequences = new ArrayList<>(); - queryPlan2Sequences = new ArrayList<>(); - - // Generate Random Query - PostgresTables tables = state.getSchema().getRandomTableNonEmptyTables(); - List tableList = tables.getTables().stream().map(t -> new PostgresTableReference(t)) - .collect(Collectors.toList()); - gen = new PostgresExpressionGenerator(state).setColumns(tables.getColumns()); - List fetchColumns = new ArrayList<>(); - fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() - .map(c -> new PostgresColumnReference(c)).collect(Collectors.toList())); - - select = new PostgresSelect(); - select.setFetchColumns(fetchColumns); - select.setFromList(tableList); - List joins = PostgresJoin.getJoins(tableList, state); - select.setJoinList(joins); - - select.setSelectType(Randomly.fromOptions(PostgresSelect.SelectType.values())); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - } - if (Randomly.getBoolean()) { - select.setGroupByExpressions(fetchColumns); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - } - } - - // First query row count - String queryString1 = PostgresVisitor.asString(select); - long rowCount1 = getRow(state, queryString1, queryPlan1Sequences); - - // JOIN and LIMIT mutations not added - boolean increase; - if (PostgresBugs.bug18643) { - increase = mutate(Mutator.LIMIT, Mutator.OR, Mutator.AND); - } else { - increase = mutate(Mutator.LIMIT); - } - - // Second Query row count - String queryString2 = PostgresVisitor.asString(select); - long rowCount2 = getRow(state, queryString2, queryPlan2Sequences); - - // Check query plan equivalence - if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { - return; - } - - // Check results - if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { - throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 - + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); - } - } - - @Override - protected boolean mutateJoin() { - if (select.getJoinList().isEmpty()) { - return false; - } - PostgresJoin join = (PostgresJoin) Randomly.fromList(select.getJoinList()); - - // Exclude CROSS for on condition - if (join.getType() == PostgresJoinType.CROSS) { - List columns = new ArrayList<>(); - columns.addAll(((PostgresTableReference) join.getLeftTable()).getTable().getColumns()); - columns.addAll(((PostgresTableReference) join.getRightTable()).getTable().getColumns()); - PostgresExpressionGenerator joinGen2 = new PostgresExpressionGenerator(state).setColumns(columns); - join.setOnClause(joinGen2.generateExpression(0, PostgresDataType.BOOLEAN)); - } - - PostgresJoinType newJoinType = PostgresJoinType.INNER; - if (join.getType() == PostgresJoinType.LEFT || join.getType() == PostgresJoinType.RIGHT) { - newJoinType = PostgresJoinType.getRandomExcept(PostgresJoinType.LEFT, PostgresJoinType.RIGHT); - } else { - newJoinType = PostgresJoinType.getRandomExcept(join.getType()); - } - boolean increase = join.getType().ordinal() < newJoinType.ordinal(); - join.setType(newJoinType); - if (newJoinType == PostgresJoinType.CROSS) { - join.setOnClause(null); - } - return increase; - } - - @Override - protected boolean mutateDistinct() { - PostgresSelect.SelectType selectType = select.getSelectOption(); - if (selectType != PostgresSelect.SelectType.ALL) { - select.setSelectType(PostgresSelect.SelectType.ALL); - return true; - } else { - select.setSelectType(PostgresSelect.SelectType.DISTINCT); - return false; - } - } - - @Override - protected boolean mutateWhere() { - boolean increase = select.getWhereClause() != null; - if (increase) { - select.setWhereClause(null); - } else { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - } - return increase; - } - - @Override - protected boolean mutateGroupBy() { - boolean increase = select.getGroupByExpressions().size() > 0; - if (increase) { - select.clearGroupByExpressions(); - } else { - select.setGroupByExpressions(select.getFetchColumns()); - } - return increase; - } - - @Override - protected boolean mutateHaving() { - if (select.getGroupByExpressions().size() == 0) { - select.setGroupByExpressions(select.getFetchColumns()); - select.setHavingClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - return false; - } else { - if (select.getHavingClause() == null) { - select.setHavingClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - return false; - } else { - select.setHavingClause(null); - return true; - } - } - } - - @Override - protected boolean mutateAnd() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - } else { - PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), - gen.generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.AND); - select.setWhereClause(newWhere); - } - return false; - } - - @Override - protected boolean mutateOr() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - return false; - } else { - PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), - gen.generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.OR); - select.setWhereClause(newWhere); - return true; - } - } - - @Override - protected boolean mutateLimit() { - boolean increase = select.getLimitClause() != null; - if (increase) { - select.setLimitClause(null); - } else { - Randomly r = new Randomly(); - select.setLimitClause(PostgresConstant.createIntConstant((int) Math.abs(r.getInteger()))); - } - return increase; - } - - private long getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) - throws AssertionError, SQLException { - long row = -1; - String explainQuery = "EXPLAIN " + selectStr; - - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(explainQuery); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - // Get row count - SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs != null) { - while (rs.next()) { - String content = rs.getString(1).trim(); - if (content.contains("rows=")) { - try { - int ind = content.indexOf("rows="); - long number = Long.parseLong(content.substring(ind + 5).split(" ")[0]); - if (row == -1) { - row = number; - - } - } catch (Exception e) { - } - } - // Proper Formatting TBD - String[] planPart = content.split("-> "); - String plan = planPart[planPart.length - 1]; - queryPlanSequences.add(plan.split(" ")[0].trim()); - } - - } - } catch (Exception e) { - throw new AssertionError(q.getQueryString(), e); - } - if (row == -1) { - throw new IgnoreMeException(); - } - return row; - - } -} From 2d54f09b1a3b375fded60d5e9b28ddf6a5f985b0 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 15:36:56 +0800 Subject: [PATCH 1017/1284] [MySQL] Use common CERT oracle --- src/sqlancer/mysql/MySQLOracleFactory.java | 19 +- .../mysql/gen/MySQLExpressionGenerator.java | 97 +++++++- .../mysql/oracle/MySQLCERTOracle.java | 217 ------------------ .../postgres/PostgresOracleFactory.java | 2 +- 4 files changed, 112 insertions(+), 223 deletions(-) delete mode 100644 src/sqlancer/mysql/oracle/MySQLCERTOracle.java diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index 206179101..a0170d685 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -1,13 +1,15 @@ package sqlancer.mysql; import java.sql.SQLException; +import java.util.Optional; import sqlancer.OracleFactory; +import sqlancer.common.oracle.CERTOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.mysql.gen.MySQLExpressionGenerator; -import sqlancer.mysql.oracle.MySQLCERTOracle; import sqlancer.mysql.oracle.MySQLDQPOracle; import sqlancer.mysql.oracle.MySQLFuzzer; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; @@ -40,7 +42,20 @@ public boolean requiresAllTablesToContainRows() { CERT { @Override public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLCERTOracle(globalState); + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + int rowCount = rs.getInt(10); + return Optional.of((long) rowCount); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String operation = rs.getString(2); + return Optional.of(operation); + }; + + return new CERTOracle<>(globalState, gen, expectedErrors, rowCountParser, queryPlanParser); + } @Override diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index bddf5946f..0958cca8a 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -2,10 +2,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.CERTGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; @@ -41,7 +43,8 @@ import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; public class MySQLExpressionGenerator extends UntypedExpressionGenerator - implements TLPWhereGenerator { + implements TLPWhereGenerator, + CERTGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; @@ -208,8 +211,7 @@ public List generateOrderBys() { } @Override - public TLPWhereGenerator setTablesAndColumns( - AbstractTables tables) { + public MySQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -240,4 +242,93 @@ public List getTableRefs() { public List generateFetchColumns(boolean shouldCreateDummy) { return columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); } + + @Override + public String generateExplainQuery(MySQLSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(MySQLSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateWhere); + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + mutators.add(this::mutateAnd); + mutators.add(this::mutateOr); + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateDistinct(MySQLSelect select) { + MySQLSelect.SelectType selectType = select.getFromOptions(); + if (selectType != MySQLSelect.SelectType.ALL) { + select.setSelectType(MySQLSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(MySQLSelect.SelectType.DISTINCT); + return false; + } + } + + boolean mutateWhere(MySQLSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression()); + } + return increase; + } + + boolean mutateGroupBy(MySQLSelect select) { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + boolean mutateHaving(MySQLSelect select) { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(MySQLSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + MySQLBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(MySQLSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + return false; + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + MySQLBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } } diff --git a/src/sqlancer/mysql/oracle/MySQLCERTOracle.java b/src/sqlancer/mysql/oracle/MySQLCERTOracle.java deleted file mode 100644 index 2cdf3c7cf..000000000 --- a/src/sqlancer/mysql/oracle/MySQLCERTOracle.java +++ /dev/null @@ -1,217 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLGlobalState; -import sqlancer.common.DBMSCommon; -import sqlancer.common.oracle.CERTOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.mysql.MySQLErrors; -import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLSchema.MySQLTables; -import sqlancer.mysql.MySQLVisitor; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; -import sqlancer.mysql.ast.MySQLColumnReference; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLSelect; -import sqlancer.mysql.ast.MySQLTableReference; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -public class MySQLCERTOracle extends CERTOracleBase implements TestOracle { - private MySQLExpressionGenerator gen; - private MySQLSelect select; - - public MySQLCERTOracle(MySQLGlobalState globalState) { - super(globalState); - MySQLErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - queryPlan1Sequences = new ArrayList<>(); - queryPlan2Sequences = new ArrayList<>(); - - // Randomly generate a query - MySQLTables tables = state.getSchema().getRandomTableNonEmptyTables(); - gen = new MySQLExpressionGenerator(state).setColumns(tables.getColumns()); - List fetchColumns = new ArrayList<>(); - fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() - .map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList())); - List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) - .collect(Collectors.toList()); - - select = new MySQLSelect(); - select.setFetchColumns(fetchColumns); - select.setFromList(tableList); - - select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); - } - if (Randomly.getBoolean()) { - select.setGroupByExpressions(fetchColumns); - if (Randomly.getBoolean()) { - select.setHavingClause(gen.generateExpression()); - } - } - - // Set the join. Todo: to make it random - // List joinExpressions = getJoins(tableList, state); - // select.setJoinList(joinExpressions); - - // Get the result of the first query - String queryString1 = MySQLVisitor.asString(select); - int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); - - boolean increase = mutate(Mutator.JOIN, Mutator.LIMIT); - - // Get the result of the second query - String queryString2 = MySQLVisitor.asString(select); - int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); - - // Check structural equivalence - if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { - return; - } - - // Check the results - if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { - throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 - + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); - } - } - - @Override - protected boolean mutateDistinct() { - MySQLSelect.SelectType selectType = select.getFromOptions(); - if (selectType != MySQLSelect.SelectType.ALL) { - select.setSelectType(MySQLSelect.SelectType.ALL); - return true; - } else { - select.setSelectType(MySQLSelect.SelectType.DISTINCT); - return false; - } - } - - @Override - protected boolean mutateWhere() { - boolean increase = select.getWhereClause() != null; - if (increase) { - select.setWhereClause(null); - } else { - select.setWhereClause(gen.generateExpression()); - } - return increase; - } - - @Override - protected boolean mutateGroupBy() { - boolean increase = select.getGroupByExpressions().size() > 0; - if (increase) { - select.clearGroupByExpressions(); - } else { - select.setGroupByExpressions(select.getFetchColumns()); - } - return increase; - } - - @Override - protected boolean mutateHaving() { - if (select.getGroupByExpressions().size() == 0) { - select.setGroupByExpressions(select.getFetchColumns()); - select.setHavingClause(gen.generateExpression()); - return false; - } else { - if (select.getHavingClause() == null) { - select.setHavingClause(gen.generateExpression()); - return false; - } else { - select.setHavingClause(null); - return true; - } - } - } - - @Override - protected boolean mutateAnd() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression()); - } else { - MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), - gen.generateExpression(), MySQLBinaryLogicalOperator.AND); - select.setWhereClause(newWhere); - } - return false; - } - - @Override - protected boolean mutateOr() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression()); - return false; - } else { - MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), - gen.generateExpression(), MySQLBinaryLogicalOperator.OR); - select.setWhereClause(newWhere); - return true; - } - } - - // The limit clause only accpets positive integers, which is not supported yet - // private boolean mutateLimit() { - // boolean increase = select.getLimitClause() != null; - // if (increase) { - // select.setLimitClause(null); - // } else { - // select.setLimitClause(gen.generateConstant()); - // } - // return increase; - // } - - private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) - throws AssertionError, SQLException { - int row = -1; - String explainQuery = "EXPLAIN " + selectStr; - - // Log the query - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(explainQuery); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - // Get the row count - SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs != null) { - while (rs.next()) { - int estRows = rs.getInt(10); - if (row == -1) { - row = estRows; - } - String operation = rs.getString(2); - queryPlanSequences.add(operation); - } - } - } catch (Exception e) { - throw new AssertionError(q.getQueryString(), e); - } - if (row == -1) { - throw new IgnoreMeException(); - } - return row; - } - -} diff --git a/src/sqlancer/postgres/PostgresOracleFactory.java b/src/sqlancer/postgres/PostgresOracleFactory.java index df2185776..7c00021ed 100644 --- a/src/sqlancer/postgres/PostgresOracleFactory.java +++ b/src/sqlancer/postgres/PostgresOracleFactory.java @@ -82,7 +82,7 @@ public TestOracle create(PostgresGlobalState globalState) t .with(PostgresCommon.getCommonRangeExpressionErrors()).build(); CERTOracle.CheckedFunction> rowCountParser = (rs) -> { String content = rs.getString(1).trim(); - if (content.contains("rows=")) { + if (content.contains("Result") && content.contains("rows=")) { try { int ind = content.indexOf("rows="); long number = Long.parseLong(content.substring(ind + 5).split(" ")[0]); From 0d70f35b97e5b10e482437350c45999bc153fa7c Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 9 Oct 2024 20:46:40 +0800 Subject: [PATCH 1018/1284] [CockroachDB] Use common CERT oracle --- .../cockroachdb/CockroachDBErrors.java | 1 + .../cockroachdb/CockroachDBOracleFactory.java | 33 +++- .../gen/CockroachDBExpressionGenerator.java | 142 +++++++++++++++++- test/sqlancer/dbms/TestCockroachDBCERT.java | 2 +- 4 files changed, 174 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 7a06fe51e..134ad35aa 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -351,6 +351,7 @@ private static List getComputedColumnErrors() { // computed columns errors.add("computed column expressions cannot reference computed columns"); + errors.add("STORED COMPUTED COLUMN expression cannot reference computed columns"); return errors; } diff --git a/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java index bbf261f16..96fbc22ce 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java +++ b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java @@ -3,20 +3,23 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import sqlancer.IgnoreMeException; import sqlancer.OracleFactory; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; +import sqlancer.common.oracle.CERTOracle; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; public enum CockroachDBOracleFactory implements OracleFactory { NOREC { @@ -97,7 +100,33 @@ public TestOracle create( @Override public TestOracle create( CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBCERTOracle(globalState); + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + String content = rs.getString(1); + if (content.contains("count:")) { + try { + long number = Long.parseLong(content.split("count: ")[1].split(" ")[0].replace(",", "")); + return Optional.of(number); + } catch (Exception e) { // To avoid the situation that no number is found + } + } + return Optional.empty(); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String content = rs.getString(1); + if (content.contains("• ")) { + String operation = content.split("• ")[1].split(" ")[0]; + if (CockroachDBBugs.bug131875 && (operation.equals("distinct") || operation.equals("limit"))) { + throw new IgnoreMeException(); + } + return Optional.of(operation); + } + return Optional.empty(); + }; + + return new CERTOracle<>(globalState, gen, expectedErrors, rowCountParser, queryPlanParser); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index 189be028d..52d8f027d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -2,9 +2,11 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; @@ -32,6 +34,7 @@ import sqlancer.cockroachdb.ast.CockroachDBFunction; import sqlancer.cockroachdb.ast.CockroachDBInOperation; import sqlancer.cockroachdb.ast.CockroachDBJoin; +import sqlancer.cockroachdb.ast.CockroachDBJoin.JoinType; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonOperator; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonType; @@ -44,6 +47,7 @@ import sqlancer.cockroachdb.ast.CockroachDBTypeAnnotation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; +import sqlancer.common.gen.CERTGenerator; import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; @@ -52,7 +56,8 @@ public class CockroachDBExpressionGenerator extends TypedExpressionGenerator implements NoRECGenerator, - TLPWhereGenerator { + TLPWhereGenerator, + CERTGenerator { private List tables; private final CockroachDBGlobalState globalState; @@ -458,4 +463,139 @@ public List generateFetchColumns(boolean shouldCreateDumm return Randomly.nonEmptySubset(columns).stream().map(c -> new CockroachDBColumnReference(c)) .collect(Collectors.toList()); } + + @Override + public String generateExplainQuery(CockroachDBSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(CockroachDBSelect select) { + List> mutators = new ArrayList<>(); + + if (!CockroachDBBugs.bug131647) { + mutators.add(this::mutateJoin); + } + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + mutators.add(this::mutateAnd); + if (!CockroachDBBugs.bug131640) { + mutators.add(this::mutateWhere); + mutators.add(this::mutateOr); + } + // mutators.add(this::mutateLimit); + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateJoin(CockroachDBSelect select) { + if (select.getJoinList().isEmpty()) { + return false; + } + + CockroachDBJoin join = (CockroachDBJoin) Randomly.fromList(select.getJoinList()); + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((CockroachDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((CockroachDBTableReference) join.getRightTable()).getTable().getColumns()); + CockroachDBExpressionGenerator joinGen2 = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + join.setOnClause(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); + } + + JoinType newJoinType = CockroachDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invariant relation + // between LEFT and RIGHT + // join + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS, JoinType.LEFT, + JoinType.RIGHT); + } else if (join.getJoinType() == JoinType.FULL) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS); + } else if (join.getJoinType() != JoinType.CROSS) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); + } + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + return increase; + } + + boolean mutateDistinct(CockroachDBSelect select) { + boolean increase = select.isDistinct(); + select.setDistinct(!select.isDistinct()); + return increase; + } + + boolean mutateWhere(CockroachDBSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression(CockroachDBDataType.BOOL.get())); + } + return increase; + } + + boolean mutateGroupBy(CockroachDBSelect select) { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + boolean mutateHaving(CockroachDBSelect select) { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(CockroachDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(CockroachDBDataType.BOOL.get())); + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(CockroachDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + boolean mutateLimit(CockroachDBSelect select) { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(generateConstant(CockroachDBDataType.INT.get())); + } + return increase; + } } diff --git a/test/sqlancer/dbms/TestCockroachDBCERT.java b/test/sqlancer/dbms/TestCockroachDBCERT.java index 7e60818c1..4d7140e40 100644 --- a/test/sqlancer/dbms/TestCockroachDBCERT.java +++ b/test/sqlancer/dbms/TestCockroachDBCERT.java @@ -13,7 +13,7 @@ public class TestCockroachDBCERT { public void testCockroachDBCERT() { assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "CERT" })); + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "CERT" })); } } From 2dd2e1a9396e697877882f252b1a3a447c04614a Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Sun, 8 Dec 2024 20:11:55 +1030 Subject: [PATCH 1019/1284] Ignore database* to gitignore - Skip make package outputs --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a3ecf6076..ff30cb6de 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,7 @@ SQLancer.iml dependency-reduced-pom.xml database0.db databaseconnectiontest.db +database*.log +database*.properties +database*.script +databases/ \ No newline at end of file From e501451c5ef80c303ff4ee3911bfec4102079038 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Jan 2025 14:59:24 +0800 Subject: [PATCH 1020/1284] Remove the Twitter/X link since the account is no longer actively being maintained --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index b0fe4769b..a18c855c8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ [![Build Status](https://github.com/sqlancer/sqlancer/workflows/ci/badge.svg)](https://github.com/sqlancer/sqlancer/actions) -[![Twitter](https://img.shields.io/twitter/follow/sqlancer_dbms?style=social)](https://twitter.com/sqlancer_dbms) -# SQLancer ![SQLancer](media/logo/png/sqlancer_logo_logo_pos_500.png) From 63e69e4deda890417572d758d37920abe9e4e642 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Jan 2025 16:51:57 +0800 Subject: [PATCH 1021/1284] Give a better overview of SQLancer --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a18c855c8..596d6545f 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,11 @@ ![SQLancer](media/logo/png/sqlancer_logo_logo_pos_500.png) -SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Management Systems (DBMS) in order to find logic bugs in their implementation. We refer to logic bugs as those bugs that cause the DBMS to fetch an incorrect result set (e.g., by omitting a record). +SQLancer is a tool to automatically test Database Management Systems (DBMSs) in order to find bugs in their implementation. That is, it finds bugs in the code of the DBMS implementation, rather than in queries written by the user. SQLancer has found hundreds of bugs in mature and widely-known DBMSs. -SQLancer operates in the following two phases: - -1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. -2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. **News: we support Differential Query Plans (DQP) oracle now. See Testing Approaches below.** +SQLancer tackles two essential challenges when automatically testing the DBMSs: +1. **Test input generation**: SQLancer implements approaches for automatically generating SQL statements. It contains various hand-written SQL generators that operate in multiple phases. First, a database schema is created, which refers to a set of tables and their columns. Then, data is inserted into these tables, along with creating various other kinds of database states such as indexes, views, or database-specific options. Finally, queries are generated, which can be validated using one of multiple result validators (also called *test oracles*) that SQLancer provides. Besides the standard approach of creating the statements in an unguided way, SQLancer also supports a test input-generation approach that is feedback-guided and aims to exercise as many unique query plans as possible based on the intuition that doing so would exercise many interesting behaviors in the database system [[ICSE '23]](https://arxiv.org/pdf/2312.17510). +2. **Test oracles**: A key innovation in SQLancer is that it provides ways to find deep kinds of bugs in DBMSs. As a main focus, it can find logic bugs, which are bugs that cause the DBMS to fetch an incorrect result set (e.g., by omitting a record). We have proposed multiple complementary test oracles such as *Ternary Logic Partitioning (TLP)* [[OOPSLA '20]](https://dl.acm.org/doi/pdf/10.1145/3428279), *Non-optimizing Reference Engine Construction (NoREC)* [[ESEC/FSE 2020]](https://arxiv.org/abs/2007.08292), *Pivoted Query Synthesis (PQS)* [[OSDI '20]](https://www.usenix.org/system/files/osdi20-rigger.pdf), *Differential Query Plans (DQP)* [[SIGMOD '24]](https://dl.acm.org/doi/pdf/10.1145/3654991), and *Constant Optimization Driven Database System Testing (CODDTest)* [SIGMOD '25]. It can also find specific categories of performance issues, which refer to cases where a DBMS could reasonably be expected to produce its result more efficiently using a technique called *Cardinality Estimation Restriction Testing (CERT)* [[ICSE '24]](https://arxiv.org/pdf/2306.00355). SQLancer can detect unexpected internal errors (e.g., an error that the database is corrupted) by declaring all potential errors that might be returned by a DBMS for a given query. Finally, SQLancer can find crash bugs, which are bugs that cause the DBMS process to terminate. For this, it uses an implicit test oracle. # Getting Started From 4a9f1d7b2dd62c37bc9b6e80f0d1c5f705654e66 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Jan 2025 17:14:47 +0800 Subject: [PATCH 1022/1284] Update the getting started guide --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 596d6545f..1eb7bac1e 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,11 @@ SQLancer tackles two essential challenges when automatically testing the DBMSs: # Getting Started -Requirements: +Minimum Requirements: * Java 11 or above -* [Maven](https://maven.apache.org/) (`sudo apt install maven` on Ubuntu) -* The DBMS that you want to test (embedded DBMSs such as DuckDB, H2, and SQLite do not require a setup) +* [Maven](https://maven.apache.org/) -The following commands clone SQLancer, create a JAR, and start SQLancer to test SQLite using Non-optimizing Reference Engine Construction (NoREC): +The following commands clone SQLancer, create a JAR, and start SQLancer to test SQLite using [Non-optimizing Reference Engine Construction (NoREC)](https://arxiv.org/abs/2007.08292): ``` git clone https://github.com/sqlancer/sqlancer @@ -26,9 +25,12 @@ cd target java -jar sqlancer-*.jar --num-threads 4 sqlite3 --oracle NoREC ``` -If the execution prints progress information every five seconds, then the tool works as expected. Note that SQLancer might find bugs in SQLite. Before reporting these, be sure to check that they can still be reproduced when using the latest development version. The shortcut CTRL+C can be used to terminate SQLancer manually. If SQLancer does not find any bugs, it executes infinitely. The option `--num-tries` can be used to control after how many bugs SQLancer terminates. Alternatively, the option `--timeout-seconds` can be used to specify the maximum duration that SQLancer is allowed to run. +**Running and terminating.** If the execution prints progress information every five seconds, then the tool works as expected. The shortcut CTRL+C can be used to terminate SQLancer manually. If SQLancer does not find any bugs, it executes infinitely. The option `--num-tries` can be used to control after how many bugs SQLancer terminates. Alternatively, the option `--timeout-seconds` can be used to specify the maximum duration that SQLancer is allowed to run. + +**Parameters.** If you launch SQLancer without parameters, available options and commands are displayed. Note that general options that are supported by all DBMS-testing implementations (e.g., `--num-threads`) need to precede the name of the DBMS to be tested (e.g., `sqlite3`). Options that are supported only for specific DBMS (e.g., `--test-rtree` for SQLite3), or options for which each testing implementation provides different values (e.g. `--oracle NoREC`) need to go after the DBMS name. + +**DBMSs.** To run SQLancer on SQLite, it was not necessary to install and set up a DBMS. The reason for this is that embedded DBMSs run in the same process as the application and thus require no separate installation or setup. Embedded DBMSs supported by SQLancer include DuckDB, H2, and SQLite. Their binaries are included as [JAR dependencies](https://github.com/sqlancer/sqlancer/blob/main/pom.xml). Note that any crashes in these systems will also cause a crash in the JVM on which SQLancer runs. -If you launch SQLancer without parameters, available options and commands are displayed. Note that general options that are supported by all DBMS-testing implementations (e.g., `--num-threads`) need to precede the name of DBMS to be tested (e.g., `sqlite3`). Options that are supported only for specific DBMS (e.g., `--test-rtree` for SQLite3), or options for which each testing implementation provides different values (e.g. `--oracle NoREC`) need to go after the DBMS name. # Testing Approaches @@ -98,6 +100,8 @@ After finding a bug, it is useful to produce a minimal test case before reportin ## Found Bugs +For most DBMSs, SQLancer supports only a previous *release* version. Thus, potential bugs that SQLancer finds could be already fixed in the latest *development* version of the DBMS. If you are not a developer of the DBMS that you are testing, we would like to encourage you to validate that the bug can still be reproduced before reporting it. + We would appreciate it if you mention SQLancer when you report bugs found by it. We would also be excited to know if you are using SQLancer to find bugs, or if you have extended it to test another DBMS (also if you do not plan to contribute it to this project). SQLancer has found over 400 bugs in widely-used DBMS, which are listed [here](https://www.manuelrigger.at/dbms-bugs/). From 002b9a4885d0afb7aa2ae8866a870d1c2b86c69a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Jan 2025 17:39:38 +0800 Subject: [PATCH 1023/1284] Restructure and update the section on how to use SQLancer --- README.md | 59 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 1eb7bac1e..27f9579e1 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,41 @@ java -jar sqlancer-*.jar --num-threads 4 sqlite3 --oracle NoREC **DBMSs.** To run SQLancer on SQLite, it was not necessary to install and set up a DBMS. The reason for this is that embedded DBMSs run in the same process as the application and thus require no separate installation or setup. Embedded DBMSs supported by SQLancer include DuckDB, H2, and SQLite. Their binaries are included as [JAR dependencies](https://github.com/sqlancer/sqlancer/blob/main/pom.xml). Note that any crashes in these systems will also cause a crash in the JVM on which SQLancer runs. +# Using SQLancer + +**Logs.** SQLancer stores logs in the `target/logs` subdirectory. By default, the option `--log-each-select` is enabled, which results in every SQL statement that is sent to the DBMS being logged. The corresponding file names are postfixed with `-cur.log`. In addition, if SQLancer detects a logic bug, it creates a file with the extension `.log`, in which the statements to reproduce the bug are logged, including only the last query that was executed along with the other statements to set up the database state. + +**Reducing bugs.** After finding a bug-inducing test input, the input typically needs to be reduced to be further analyzed, as it might contain many SQL statements that are redundant to reproduce the bug. One option is to do this manually, by removing a statement or feature at a time, replaying the bug-inducing statements, and applying the test oracle (e.g., for test oracles like TLP or NoREC, this would require checking that both queries still produce a different result). This process can be automated using a so-called [delta-debugging approach](https://www.debuggingbook.org/html/DeltaDebugger.html). SQLancer includes an experimental implementation of a delta debugging approach, which can be enabled using `--use-reducer`. In the past, we have successfully used [C-Reduce](https://embed.cs.utah.edu/creduce/), which requires specifying the test oracle in a script that can be executed by C-Reduce. + +**Testing the latest DBMS version.** For most DBMSs, SQLancer supports only a previous *release* version. Thus, potential bugs that SQLancer finds could be already fixed in the latest *development* version of the DBMS. If you are not a developer of the DBMS that you are testing, we would like to encourage you to validate that the bug can still be reproduced before reporting it. We would appreciate it if you could mention SQLancer when you report bugs found by it. We would also be excited to hear about your experience using SQLancer or related use cases or extensions. + +**Options.** SQLancer provides many options that you can use to customize its behavior. Executing `java -jar sqlancer-*.jar --help` will list them and should print output such as the following: +``` +Usage: SQLancer [options] [command] [command options] + Options: + --ast-reducer-max-steps + EXPERIMENTAL Maximum steps the AST-based reducer will do + Default: -1 + --ast-reducer-max-time + EXPERIMENTAL Maximum time duration (secs) the statement reducer will do + Default: -1 + --canonicalize-sql-strings + Should canonicalize query string (add ';' at the end + Default: true + --constant-cache-size + Specifies the size of the constant cache. This option only takes effect + when constant caching is enabled + Default: 100 +... +``` + +**Which SQLancer version to use.** The recommended way to use SQLancer is to use its latest source version on GitHub. Infrequent and irregular official releases are also available on the following platforms: +* [GitHub](https://github.com/sqlancer/sqlancer/releases) +* [Maven Central](https://search.maven.org/artifact/com.sqlancer/sqlancer) +* [DockerHub](https://hub.docker.com/r/mrigger/sqlancer) + +**Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. + # Testing Approaches | Approach | Description | @@ -88,23 +123,6 @@ Some DBMS were once supported but subsequently removed. | StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. | -# Using SQLancer - -## Logs - -SQLancer stores logs in the `target/logs` subdirectory. By default, the option `--log-each-select` is enabled, which results in every SQL statement that is sent to the DBMS being logged. The corresponding file names are postfixed with `-cur.log`. In addition, if SQLancer detects a logic bug, it creates a file with the extension `.log`, in which the statements to reproduce the bug are logged. - -## Reducing a Bug - -After finding a bug, it is useful to produce a minimal test case before reporting the bug, to save the DBMS developers' time and effort. For many test cases, [C-Reduce](https://embed.cs.utah.edu/creduce/) does a great job. - -## Found Bugs - -For most DBMSs, SQLancer supports only a previous *release* version. Thus, potential bugs that SQLancer finds could be already fixed in the latest *development* version of the DBMS. If you are not a developer of the DBMS that you are testing, we would like to encourage you to validate that the bug can still be reproduced before reporting it. - -We would appreciate it if you mention SQLancer when you report bugs found by it. We would also be excited to know if you are using SQLancer to find bugs, or if you have extended it to test another DBMS (also if you do not plan to contribute it to this project). SQLancer has found over 400 bugs in widely-used DBMS, which are listed [here](https://www.manuelrigger.at/dbms-bugs/). - - # Community We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. SQLancer's official Twitter handle is [@sqlancer_dbms](https://twitter.com/sqlancer_dbms). @@ -126,13 +144,6 @@ For some DBMSs, SQLancer expects that a database "test" exists, which it then us * [Contributing to SQLancer](CONTRIBUTING.md) * [Papers and .bib entries](docs/PAPERS.md) -# Releases - -Official release are available on: -* [GitHub](https://github.com/sqlancer/sqlancer/releases) -* [Maven Central](https://search.maven.org/artifact/com.sqlancer/sqlancer) -* [DockerHub](https://hub.docker.com/r/mrigger/sqlancer) - # Additional Resources * A talk on Ternary Logic Partitioning (TLP) and SQLancer is available on [YouTube](https://www.youtube.com/watch?v=Np46NQ6lqP8). From bfad1c33b80ef06b0142c5f0659000b027a7feaa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 23:47:55 +0800 Subject: [PATCH 1024/1284] Bump org.xerial:sqlite-jdbc from 3.46.1.0 to 3.47.2.0 (#1052) Bumps [org.xerial:sqlite-jdbc](https://github.com/xerial/sqlite-jdbc) from 3.46.1.0 to 3.47.2.0. - [Release notes](https://github.com/xerial/sqlite-jdbc/releases) - [Changelog](https://github.com/xerial/sqlite-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/xerial/sqlite-jdbc/compare/3.46.1.0...3.47.2.0) --- updated-dependencies: - dependency-name: org.xerial:sqlite-jdbc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 119ef6434..1bcf92957 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ org.xerial sqlite-jdbc - 3.46.1.0 + 3.47.2.0 mysql From 2814b206e392395b0ef557ca45afa78c2dd069ef Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:11:26 +0100 Subject: [PATCH 1025/1284] fix typo with invalid random number of tables Apply fix from https://github.com/sqlancer/sqlancer/pull/1058 --- src/sqlancer/clickhouse/ClickHouseProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 86f6fd392..c611d77d7 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -87,7 +87,7 @@ protected ClickHouseSchema readSchema() throws SQLException { @Override public void generateDatabase(ClickHouseGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(5); i++) { + for (int i = 0; i < Randomly.fromOptions(1, 2, 3, 4, 5); i++) { boolean success; do { String tableName = ClickHouseCommon.createTableName(i); From 0fb45a2242c2d3037e329b59cde76654bc952493 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 15:47:28 +0800 Subject: [PATCH 1026/1284] Improve the documentation on the currently supported approaches --- README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 27f9579e1..12a879dd7 100644 --- a/README.md +++ b/README.md @@ -67,21 +67,19 @@ Usage: SQLancer [options] [command] [command options] **Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. -# Testing Approaches - -| Approach | Description | -|------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Pivoted Query Synthesis (PQS) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described [here](https://arxiv.org/abs/2001.04174). PQS is the most powerful technique, but also requires more implementation effort than the other two techniques. It is currently unmaintained. | -| Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | -| Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | -| Cardinality Estimation Restriction Testing (CERT) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. It is described [here](https://arxiv.org/abs/2306.00355). It derives a query to a more restrict query, whose estimated cardinality should be no more than that for the original query. An violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. | -| Differential Query Plans (DQP) | DQP aims to find logic bugs in database systems by checking whether the query plans of the same query perform consistently. It is described [here](https://bajinsheng.github.io/assets/pdf/dqp_sigmod24.pdf). DQP supports MySQL, MariaDB, and TiDB.| - -# Generation Approaches -| Approach | Description | -|----------|-------------| -| Random Generation | Random generation is the default test case generation approach in SQLancer. First, random tables are generated. Then queries are randomly generated based on the schemas of the tables. | -| Query Plan Guidance (QPG) | QPG is a test case generation method guided by query plan coverage. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. This approach is enabled by option `--qpg-enable` and now supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. | +# Approaches and Papers + +SQLancer has pioneered and includes multiple approaches for DBMS testing, as outlined below in chronological order. + +| Technique | Venue | Links | Description | +|-----------------------------------------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Pivoted Query Synthesis (PQS) | OSDI 2020 | [Paper](https://www.usenix.org/system/files/osdi20-rigger.pdf) [Video](https://www.youtube.com/watch?v=0aeDyXgzo04 ) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described here. PQS effectively detects bugs, but requires more implementation effort than other testing approaches that follow a metamorphic testing or differential testing methodology. Thus, it is currently unmaintained. | +| Non-optimizing Reference Engine Construction (NoREC) | ESEC/FSE 2020 | [Paper](https://arxiv.org/abs/2007.08292) [Video](https://www.youtube.com/watch?v=4mbzytrWJhQ) | NoREC aims to find optimization bugs. It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. The approach applies primarily to simple queries with a filter predicate. | +| Ternary Logic Partitioning (TLP) | OOPSLA 2020 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3428279) [Video](https://www.youtube.com/watch?v=FN9OLbGh0VI) | TLP partitions a query into three partitioning queries, whose results are composed and compared to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. It is among the most widely adopted testing techniques. | +| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | DQP is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | +| Cardinality Estimation Restriction Testing (CERT) | ICSE 2024 | [Paper](https://arxiv.org/pdf/2306.00355) [Code](https://github.com/sqlancer/sqlancer/issues/822) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. From a given input query, it derives a more restrictive query, whose estimated cardinality should be no more than that of the original query. A violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. CERT is the only test oracle that is part of SQLancer that was designed to find performance issues. | +| Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | +| Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | Please find the `.bib` entries [here](docs/PAPERS.md). @@ -139,12 +137,14 @@ Another reason for many failures on a supported version could be that error mess For some DBMSs, SQLancer expects that a database "test" exists, which it then uses as an initial database to connect to. If you have not yet created such a database, you can use a command such as `CREATE DATABASE test` to create this database (e.g., see the [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-createdatabase.html)). -# Additional Documentation +# Links + +Documentation: * [Contributing to SQLancer](CONTRIBUTING.md) * [Papers and .bib entries](docs/PAPERS.md) -# Additional Resources +Other resources: * A talk on Ternary Logic Partitioning (TLP) and SQLancer is available on [YouTube](https://www.youtube.com/watch?v=Np46NQ6lqP8). * An (older) Pivoted Query Synthesis (PQS) talk is available on [YouTube](https://www.youtube.com/watch?v=yzENTaWe7qg). From b6f81e3f5ac14e9b9aef4a37bb06d2cb5fb254ba Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:09:28 +0800 Subject: [PATCH 1027/1284] Update the resource page --- README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 12a879dd7..0fa2da8e5 100644 --- a/README.md +++ b/README.md @@ -139,14 +139,19 @@ For some DBMSs, SQLancer expects that a database "test" exists, which it then us # Links -Documentation: +Documentation and resources: * [Contributing to SQLancer](CONTRIBUTING.md) * [Papers and .bib entries](docs/PAPERS.md) +* More information on our DBMS testing efforts and the bugs we found is available [here](https://www.manuelrigger.at/dbms-bugs/). -Other resources: +Videos: +* [SQLancer Tutorial Playlist](https://www.youtube.com/playlist?list=PLm7ofmclym1E2LwBeSer_AAhzBSxBYDci) +* [SQLancer Talks](https://youtube.com/playlist?list=PLm7ofmclym1E9-AbYy-PkrMfHpB9VdlZJ) -* A talk on Ternary Logic Partitioning (TLP) and SQLancer is available on [YouTube](https://www.youtube.com/watch?v=Np46NQ6lqP8). -* An (older) Pivoted Query Synthesis (PQS) talk is available on [YouTube](https://www.youtube.com/watch?v=yzENTaWe7qg). -* PingCAP has implemented PQS, NoREC, and TLP in a tool called [go-sqlancer](https://github.com/chaos-mesh/go-sqlancer). -* More information on our DBMS testing efforts and the bugs we found is available [here](https://www.manuelrigger.at/dbms-bugs/). +Closely related tools: +* [go-sqlancer](https://github.com/chaos-mesh/go-sqlancer): re-implementation of some of SQLancer's approaches in Go by PingCAP +* [Jepsen](https://github.com/jepsen-io): testing of distributed (database) systems +* [SQLRight](https://github.com/PSU-Security-Universe/sqlright): coverage-guided DBMS fuzzer, also supporting NoREC and TLP +* [SQLsmith](https://github.com/anse1/sqlsmith): random SQL query generator used for fuzzing +* [Squirrel](https://github.com/s3team/Squirrel): coverage-guided DBMS fuzzer \ No newline at end of file From 324f01a9f9b74f23e374751290799f2045ef658e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:12:39 +0800 Subject: [PATCH 1028/1284] Improve the FAQ formatting --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0fa2da8e5..18314271e 100644 --- a/README.md +++ b/README.md @@ -127,15 +127,11 @@ We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_inv # FAQ -## I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors? - -In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) file to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). +**I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors?** In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) file to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html)). -## When starting SQLancer, I get an error such as "database 'test' does not exist". How can I run SQLancer without this error? - -For some DBMSs, SQLancer expects that a database "test" exists, which it then uses as an initial database to connect to. If you have not yet created such a database, you can use a command such as `CREATE DATABASE test` to create this database (e.g., see the [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-createdatabase.html)). +**When starting SQLancer, I get an error such as "database 'test' does not exist". How can I run SQLancer without this error?** For some DBMSs, SQLancer expects that a database "test" exists, which it then uses as an initial database to connect to. If you have not yet created such a database, you can use a command such as `CREATE DATABASE test` to create this database (e.g., see the [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-createdatabase.html)). # Links From 06a28e929edb2dd5429a278009a88a87219181b5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:21:02 +0800 Subject: [PATCH 1029/1284] Work on the community part --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 18314271e..c18ca202c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ SQLancer tackles two essential challenges when automatically testing the DBMSs: 1. **Test input generation**: SQLancer implements approaches for automatically generating SQL statements. It contains various hand-written SQL generators that operate in multiple phases. First, a database schema is created, which refers to a set of tables and their columns. Then, data is inserted into these tables, along with creating various other kinds of database states such as indexes, views, or database-specific options. Finally, queries are generated, which can be validated using one of multiple result validators (also called *test oracles*) that SQLancer provides. Besides the standard approach of creating the statements in an unguided way, SQLancer also supports a test input-generation approach that is feedback-guided and aims to exercise as many unique query plans as possible based on the intuition that doing so would exercise many interesting behaviors in the database system [[ICSE '23]](https://arxiv.org/pdf/2312.17510). 2. **Test oracles**: A key innovation in SQLancer is that it provides ways to find deep kinds of bugs in DBMSs. As a main focus, it can find logic bugs, which are bugs that cause the DBMS to fetch an incorrect result set (e.g., by omitting a record). We have proposed multiple complementary test oracles such as *Ternary Logic Partitioning (TLP)* [[OOPSLA '20]](https://dl.acm.org/doi/pdf/10.1145/3428279), *Non-optimizing Reference Engine Construction (NoREC)* [[ESEC/FSE 2020]](https://arxiv.org/abs/2007.08292), *Pivoted Query Synthesis (PQS)* [[OSDI '20]](https://www.usenix.org/system/files/osdi20-rigger.pdf), *Differential Query Plans (DQP)* [[SIGMOD '24]](https://dl.acm.org/doi/pdf/10.1145/3654991), and *Constant Optimization Driven Database System Testing (CODDTest)* [SIGMOD '25]. It can also find specific categories of performance issues, which refer to cases where a DBMS could reasonably be expected to produce its result more efficiently using a technique called *Cardinality Estimation Restriction Testing (CERT)* [[ICSE '24]](https://arxiv.org/pdf/2306.00355). SQLancer can detect unexpected internal errors (e.g., an error that the database is corrupted) by declaring all potential errors that might be returned by a DBMS for a given query. Finally, SQLancer can find crash bugs, which are bugs that cause the DBMS process to terminate. For this, it uses an implicit test oracle. +**Community.** We have a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. Previously, SQLancer had an account on Twitter/X [@sqlancer_dbms](https://twitter.com/sqlancer_dbms), which is no longer maintained. We have a [blog](https://sqlancer.github.io/posts/), which, as of now, contains only posts by contributors of the [Google Summer of Code project](https://summerofcode.withgoogle.com/archive/2023/organizations/sqlancer). + # Getting Started Minimum Requirements: @@ -120,11 +122,6 @@ Some DBMS were once supported but subsequently removed. | MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | | StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. | - -# Community - -We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. SQLancer's official Twitter handle is [@sqlancer_dbms](https://twitter.com/sqlancer_dbms). - # FAQ **I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors?** In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) file to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). From 48052e3c763286155a7aa6d8cc9073bb6846ab24 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:32:33 +0800 Subject: [PATCH 1030/1284] Move the list of supported DBMSs to CONTRIBUTING.md --- CONTRIBUTING.md | 38 ++++++++++++++++++++++++++++++++++++++ README.md | 39 +-------------------------------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f2217b1f5..85249f315 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,43 @@ For a permissive DBMS, implementing the expression generator is easier, since th For a strict DBMS, the better approach is typically to attempt to generate expressions of the expected type. For PostgreSQL, the expression generator thus expects an additional type argument (see [PostgreSQLExpressionGenerator](https://github.com/sqlancer/sqlancer/blob/86647df8aa2dd8d167b5c3ce3297290f5b0b2bcd/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java#L251)). This type is propagated recursively. For example, if we require a predicate for the `WHERE` clause, we pass boolean as a type. The expression generator then calls a method `generateBooleanExpression` that attempts to produce a boolean expression, by, for example, generating a comparison (e.g., `<=`). For the comparison's operands, a random type is then selected and propagated. For example, if an integer type is selected, then `generateExpression` is called with this type once for the left operand, and once for the right operand. Note that this process does not guarantee that the expression will indeed have the expected type. It might happen, for example, that the expression generator attempts to produce an integer value, but that it produces a double value instead, namely when an integer overflow occurs, which, depending on the DBMS, implicitly converts the result to a floating-point value. +#### Supported DBMS + +Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. + +| DBMS | Status | Expression Generation | Description | +| ---------------------------- | ----------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | +| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | +| PostgreSQL | Working | Typed | | +| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | +| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | +| CockroachDB | Working | Typed | | +| TiDB | Working | Untyped | | +| DuckDB | Working | Untyped, Generic | | +| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | +| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | +| OceanBase | Working | Untyped | | +| YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | +| Databend | Working | Typed | | +| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | +| CnosDB | Working | Typed | The implementation of CnosDB currently uses Restful API. | +| Materialize | Working | Typed | | +| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | +| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | +| DataFusion | Preliminary | Typed | Only basic SQL features are supported. | + +#### Previously Supported DBMS + +Some DBMS were once supported but subsequently removed. + +| DBMS | Pull Request | Description | +| ---------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ArangoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because ArangoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. + ### Unfixed Bugs Often, some bugs are fixed only after an extended period, meaning that SQLancer will repeatedly report the same bug. In such cases, it might be possible to avoid generating the problematic pattern, or adding an expected error with the internal error message. Rather than, for example, commenting out the code with the bug-inducing pattern, a pattern implemented by the [TiDBBugs class](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java) should be applied. The core idea is to use a public, static flag for each issue, which is set to true as long as the issue persists (e.g., see [bug35652](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java#L55)). The work-around code is then executed—or the problematic pattern should not be generated—if the flag is set to true (e.g., [an expected error is added for bug35652](https://github.com/sqlancer/sqlancer/blob/59564d818d991d54b32fa5a79c9f733799c090f2/src/sqlancer/tidb/TiDBErrors.java#L47)). This makes it easy to later on identify and remove all such work-around code once the issue has been fixed. @@ -109,3 +146,4 @@ Please pay attention to good commit messages (in particular subject lines). As b 3. Use the imperative mood in the subject line. For example, write "Refactor the handling of indexes" rather than "Refactoring" or "Refactor**ed** the handling of indexes". Please also pay attention to a clean commit history. Rather than merging with the main branch, use `git rebase` to rebase your commits on the main branch. Sometimes, it might happen that you discover an issue only after having already created a commit, for example, when an issue is found by `mvn verify` in the CI checks. Do not introduce a separate commit for such issues. If the issue was introduced by the last commit, you can fix the issue, and use `git commit --amend` to change the latest commit. If the change was introduced by one of the previous commits, you can use `git rebase -i` to change the respective commit. If you already have a number of such commits, you can use `git squash` to "collapse" multiple commits into one. For more information, you might want to read [How (and Why!) to Keep Your Git Commit History Clean](https://about.gitlab.com/blog/2018/06/07/keeping-git-commit-history-clean/) written by Kushal Pandya. + diff --git a/README.md b/README.md index c18ca202c..cfb99d7e2 100644 --- a/README.md +++ b/README.md @@ -83,44 +83,7 @@ SQLancer has pioneered and includes multiple approaches for DBMS testing, as out | Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | | Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | -Please find the `.bib` entries [here](docs/PAPERS.md). - -# Supported DBMS - -Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. - -| DBMS | Status | Expression Generation | Description | -| ---------------------------- | ----------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | -| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | -| PostgreSQL | Working | Typed | | -| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | -| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | -| CockroachDB | Working | Typed | | -| TiDB | Working | Untyped | | -| DuckDB | Working | Untyped, Generic | | -| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | -| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | -| OceanBase | Working | Untyped | | -| YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | -| Databend | Working | Typed | | -| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | -| CnosDB | Working | Typed | The implementation of CnosDB currently uses Restful API. | -| Materialize | Working | Typed | | -| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | -| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | -| DataFusion | Preliminary | Typed | Only basic SQL features are supported. | - -## Previously Supported DBMS - -Some DBMS were once supported but subsequently removed. - -| DBMS | Pull Request | Description | -| ---------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| ArangoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because ArangoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | -| Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | -| MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | -| StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. | +Please find the `.bib` entries [here](docs/PAPERS.md). | # FAQ From ef49c7f1b0e980a3bb8253817566ac5a5b577687 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:40:59 +0800 Subject: [PATCH 1031/1284] Add a brief overview of the supported DBMSs in the main README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index cfb99d7e2..039d1e245 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ Usage: SQLancer [options] [command] [command options] **Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. +**Supported DBMSs.** SQLancer requires DBMS-specific code for each DBMS that it supports. As of January 2025, it provides support for Citus, ClickHouse, CnosDB, CockroachDB, Databend, (Apache) DataFusion, (Apache) Doris, DuckDB, H2, HSQLDB, MariaDB, Materialize, MySQL, OceanBase, PostgreSQL, Presto, QuestDB, SQLite3, TiDB, and YugabyteDB. The extent to which the individual DBMSs are supported [differs](https://github.com/sqlancer/sqlancer/blob/documentation-approaches/CONTRIBUTING.md). + # Approaches and Papers SQLancer has pioneered and includes multiple approaches for DBMS testing, as outlined below in chronological order. From 30eb75f1e0d50d85179ff3e297dd50dbc46e1e11 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:48:10 +0800 Subject: [PATCH 1032/1284] Add tutorial links --- CONTRIBUTING.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 85249f315..5db3bdc69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Development -## Working with Eclipse +## Working with Eclipse [[Video Guide]](https://www.youtube.com/watch?v=KsuGrOLKb9Q) Developing SQLancer using Eclipse is expected to work well. You can import SQLancer with a single step: diff --git a/README.md b/README.md index 039d1e245..ce04e254a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ SQLancer tackles two essential challenges when automatically testing the DBMSs: **Community.** We have a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. Previously, SQLancer had an account on Twitter/X [@sqlancer_dbms](https://twitter.com/sqlancer_dbms), which is no longer maintained. We have a [blog](https://sqlancer.github.io/posts/), which, as of now, contains only posts by contributors of the [Google Summer of Code project](https://summerofcode.withgoogle.com/archive/2023/organizations/sqlancer). -# Getting Started +# Getting Started [[Video Guide]](https://www.youtube.com/watch?v=lcZ6LixPH1Y) Minimum Requirements: * Java 11 or above From 702367f0d301baf304a70a8f3c4ed5c5f1bdd3d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jan 2025 09:01:57 +0000 Subject: [PATCH 1033/1284] Bump org.duckdb:duckdb_jdbc from 1.1.0 to 1.1.3 Bumps [org.duckdb:duckdb_jdbc](https://github.com/duckdb/duckdb) from 1.1.0 to 1.1.3. - [Release notes](https://github.com/duckdb/duckdb/releases) - [Commits](https://github.com/duckdb/duckdb/compare/v1.1.0...v1.1.3) --- updated-dependencies: - dependency-name: org.duckdb:duckdb_jdbc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1bcf92957..110f89fc4 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 1.1.0 + 1.1.3 com.facebook.presto From 72f73677471707fc632793a5585d1abcbced6a69 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 17:03:55 +0800 Subject: [PATCH 1034/1284] Remove workaround code for a fixed DuckDB bug --- src/sqlancer/duckdb/DuckDBBugs.java | 3 --- src/sqlancer/duckdb/DuckDBErrors.java | 6 ------ 2 files changed, 9 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBBugs.java b/src/sqlancer/duckdb/DuckDBBugs.java index 001deec80..c91661cb5 100644 --- a/src/sqlancer/duckdb/DuckDBBugs.java +++ b/src/sqlancer/duckdb/DuckDBBugs.java @@ -2,9 +2,6 @@ public final class DuckDBBugs { - // https://github.com/duckdb/duckdb/issues/13933 - public static boolean bug13933 = true; - private DuckDBBugs() { } diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index bea136670..5141ca0b1 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -57,12 +57,6 @@ public static List getExpressionErrors() { errors.add("Cannot subtract infinite timestamps"); errors.add("Timestamp difference is out of bounds"); - if (DuckDBBugs.bug13933) { - errors.add("INTERNAL Error: Unimplemented type for normalify"); - errors.add( - "FATAL Error: Failed: database has been invalidated because of a previous fatal error. The database must be restarted prior to being used again"); - } - return errors; } From c677872349904ba9c64ee06531795559939e184c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 00:30:55 +0800 Subject: [PATCH 1035/1284] Update the H2 version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 110f89fc4..2c616138a 100644 --- a/pom.xml +++ b/pom.xml @@ -340,7 +340,7 @@ com.h2database h2 - 2.1.214 + 2.3.232 org.mongodb From f8101f6299f5e8d53bab622d8e02d73d3475e9a8 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 8 Jan 2025 18:46:04 +0800 Subject: [PATCH 1036/1284] [MySQL] Support aggregation operations --- .../mysql/MySQLExpectedValueVisitor.java | 7 +++ src/sqlancer/mysql/MySQLToStringVisitor.java | 13 +++++ src/sqlancer/mysql/MySQLVisitor.java | 5 ++ src/sqlancer/mysql/ast/MySQLAggregate.java | 47 +++++++++++++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 8 ++++ 5 files changed, 80 insertions(+) create mode 100644 src/sqlancer/mysql/ast/MySQLAggregate.java diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index 0ff8e389d..29143884f 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -1,6 +1,7 @@ package sqlancer.mysql; import sqlancer.IgnoreMeException; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -166,4 +167,10 @@ public void visit(MySQLText text) { print(text); } + @Override + public void visit(MySQLAggregate aggr) { + print(aggr); + visit(aggr.getExpr()); + } + } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index c4bce3559..bd5a80a58 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -5,6 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -25,6 +26,7 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; public class MySQLToStringVisitor extends ToStringVisitor implements MySQLVisitor { @@ -322,4 +324,15 @@ public void visit(MySQLJoin join) { public void visit(MySQLText text) { sb.append(text.getText()); } + + @Override + public void visit(MySQLAggregate aggr) { + MySQLAggregateFunction func = aggr.getFunc(); + + sb.append(func); + sb.append("("); + sb.append(func.getRandomOption()); + visit(aggr.getExpr()); + sb.append(")"); + } } diff --git a/src/sqlancer/mysql/MySQLVisitor.java b/src/sqlancer/mysql/MySQLVisitor.java index 7b8944f28..e413f7668 100644 --- a/src/sqlancer/mysql/MySQLVisitor.java +++ b/src/sqlancer/mysql/MySQLVisitor.java @@ -1,5 +1,6 @@ package sqlancer.mysql; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -58,6 +59,8 @@ public interface MySQLVisitor { void visit(MySQLText text); + void visit(MySQLAggregate aggregate); + default void visit(MySQLExpression expr) { if (expr instanceof MySQLConstant) { visit((MySQLConstant) expr); @@ -95,6 +98,8 @@ default void visit(MySQLExpression expr) { visit((MySQLCollate) expr); } else if (expr instanceof MySQLText) { visit((MySQLText) expr); + } else if (expr instanceof MySQLAggregate) { + visit((MySQLAggregate) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java new file mode 100644 index 000000000..ad158fd8e --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -0,0 +1,47 @@ +package sqlancer.mysql.ast; + +import sqlancer.Randomly; + +public class MySQLAggregate implements MySQLExpression { + + public enum MySQLAggregateFunction { + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_count. + COUNT("DISTINCT"), + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_sum. + SUM("DISTINCT"), + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_min. + MIN("DISTINCT"), + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_max. + MAX("DISTINCT"); + + private final String[] options; + + private MySQLAggregateFunction(String... options) { + this.options = options.clone(); + } + + public String getRandomOption() { + if (options.length == 0 || Randomly.getBoolean()) { + return ""; + } + + return Randomly.fromOptions(options); + } + } + + private final MySQLExpression expr; + private final MySQLAggregateFunction func; + + public MySQLAggregate(MySQLExpression expr, MySQLAggregateFunction func) { + this.expr = expr; + this.func = func; + } + + public MySQLExpression getExpr() { + return expr; + } + + public MySQLAggregateFunction getFunc() { + return func; + } +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 0958cca8a..9aeccc858 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -16,6 +16,7 @@ import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; @@ -40,6 +41,7 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; public class MySQLExpressionGenerator extends UntypedExpressionGenerator @@ -248,6 +250,12 @@ public String generateExplainQuery(MySQLSelect select) { return "EXPLAIN " + select.asString(); } + public MySQLAggregate generateAggregate() { + MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); + MySQLExpression expr = generateExpression(); + return new MySQLAggregate(expr, func); + } + @Override public boolean mutate(MySQLSelect select) { List> mutators = new ArrayList<>(); From 66e8bebcb300949556df15884ca10afd33e27c20 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 8 Jan 2025 18:48:14 +0800 Subject: [PATCH 1037/1284] [MySQL] Generate aggregates in query --- .../gen/MySQLRandomQuerySynthesizer.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java index e1a7da75f..6f2c2a656 100644 --- a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -21,11 +21,22 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) MySQLTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(tables.getColumns()); MySQLSelect select = new MySQLSelect(); - List columns = new ArrayList<>(); + + List allColumns = new ArrayList<>(); + List columnsWithoutAggregations = new ArrayList<>(); select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); - columns.addAll(gen.generateExpressions(nrColumns)); - select.setFetchColumns(columns); + for (int i = 0; i < nrColumns; i++) { + if (Randomly.getBoolean()) { + MySQLExpression expression = gen.generateExpression(); + allColumns.add(expression); + columnsWithoutAggregations.add(expression); + } else { + allColumns.add(gen.generateAggregate()); + } + } + select.setFetchColumns(allColumns); + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) .collect(Collectors.toList()); select.setFromList(tableList); @@ -36,7 +47,7 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setGroupByExpressions(columnsWithoutAggregations); if (Randomly.getBoolean()) { select.setHavingClause(gen.generateHavingClause()); } From 760bc7a7717517d7d732930613624683b8ff6942 Mon Sep 17 00:00:00 2001 From: baishen Date: Wed, 15 Jan 2025 14:45:17 +0800 Subject: [PATCH 1038/1284] Databend support Date and Timestamp type --- src/sqlancer/databend/DatabendSchema.java | 32 +++++++++++-------- .../databend/ast/DatabendConstant.java | 4 +++ .../gen/DatabendNewExpressionGenerator.java | 6 ++++ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 1400d9d5e..af9a3e0ce 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -29,8 +29,7 @@ public class DatabendSchema extends AbstractSchema Date: Wed, 15 Jan 2025 15:49:53 +0800 Subject: [PATCH 1039/1284] [MySQL] Test aggregate generation --- pom.xml | 6 +++ .../mysql/MySQLExpectedValueVisitor.java | 2 - src/sqlancer/mysql/MySQLToStringVisitor.java | 6 ++- src/sqlancer/mysql/ast/MySQLAggregate.java | 22 ++++++----- .../mysql/gen/MySQLExpressionGenerator.java | 8 +++- .../mysql/MySQLToStringVisitorTest.java | 39 +++++++++++++++++++ 6 files changed, 69 insertions(+), 14 deletions(-) create mode 100644 test/sqlancer/mysql/MySQLToStringVisitorTest.java diff --git a/pom.xml b/pom.xml index 2c616138a..1c7214ad5 100644 --- a/pom.xml +++ b/pom.xml @@ -327,6 +327,12 @@ 5.11.2 test + + org.junit.jupiter + junit-jupiter-params + 5.11.2 + test + org.slf4j slf4j-simple diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index 29143884f..c2bea714b 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -169,8 +169,6 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { - print(aggr); - visit(aggr.getExpr()); } } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index bd5a80a58..892286f87 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -328,10 +328,14 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { MySQLAggregateFunction func = aggr.getFunc(); + String option = aggr.getOption(); sb.append(func); sb.append("("); - sb.append(func.getRandomOption()); + if (option != null) { + sb.append(option); + sb.append(" "); + } visit(aggr.getExpr()); sb.append(")"); } diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java index ad158fd8e..911b9ddd3 100644 --- a/src/sqlancer/mysql/ast/MySQLAggregate.java +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -1,6 +1,6 @@ package sqlancer.mysql.ast; -import sqlancer.Randomly; +import java.util.List; public class MySQLAggregate implements MySQLExpression { @@ -14,27 +14,25 @@ public enum MySQLAggregateFunction { // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_max. MAX("DISTINCT"); - private final String[] options; + private final List options; private MySQLAggregateFunction(String... options) { - this.options = options.clone(); + this.options = List.of(options); } - public String getRandomOption() { - if (options.length == 0 || Randomly.getBoolean()) { - return ""; - } - - return Randomly.fromOptions(options); + public List getOptions() { + return options; } } private final MySQLExpression expr; private final MySQLAggregateFunction func; + private final String option; - public MySQLAggregate(MySQLExpression expr, MySQLAggregateFunction func) { + public MySQLAggregate(MySQLExpression expr, MySQLAggregateFunction func, String option) { this.expr = expr; this.func = func; + this.option = option; } public MySQLExpression getExpr() { @@ -44,4 +42,8 @@ public MySQLExpression getExpr() { public MySQLAggregateFunction getFunc() { return func; } + + public String getOption() { + return option; + } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9aeccc858..a35ee0272 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -253,7 +253,13 @@ public String generateExplainQuery(MySQLSelect select) { public MySQLAggregate generateAggregate() { MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); MySQLExpression expr = generateExpression(); - return new MySQLAggregate(expr, func); + + if (Randomly.getBoolean() && func.getOptions().size() > 0) { + String option = Randomly.fromList(func.getOptions()); + return new MySQLAggregate(expr, func, option); + } else { + return new MySQLAggregate(expr, func, null); + } } @Override diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java new file mode 100644 index 000000000..8a335636c --- /dev/null +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -0,0 +1,39 @@ +package sqlancer.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLColumnReference; + +public class MySQLToStringVisitorTest { + + @ParameterizedTest + @EnumSource(MySQLAggregate.MySQLAggregateFunction.class) + void visitAggregateWithOptions(MySQLAggregate.MySQLAggregateFunction function) { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", + MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); + + MySQLToStringVisitor visitor = new MySQLToStringVisitor(); + + for (String option : function.getOptions()) { + visitor.visit(new MySQLAggregate(aRef, function, option)); + assertEquals(String.format("%s(%s a)", function, option), visitor.get()); + } + } + + @ParameterizedTest + @EnumSource(MySQLAggregate.MySQLAggregateFunction.class) + void visitAggregateWithoutOptions(MySQLAggregate.MySQLAggregateFunction function) { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", + MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); + + MySQLToStringVisitor visitor = new MySQLToStringVisitor(); + + visitor.visit(new MySQLAggregate(aRef, function, null)); + assertEquals(String.format("%s(a)", function), visitor.get()); + } +} From 7281cc6aa85220aa48ac912e8f8fade8ceb18068 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 15 Jan 2025 16:47:29 +0800 Subject: [PATCH 1040/1284] [MySQL] Explicate test cases --- pom.xml | 6 -- .../mysql/MySQLExpectedValueVisitor.java | 1 + src/sqlancer/mysql/MySQLToStringVisitor.java | 12 +++- src/sqlancer/mysql/ast/MySQLAggregate.java | 50 +++++++++------ .../mysql/gen/MySQLExpressionGenerator.java | 14 +++-- .../mysql/MySQLToStringVisitorTest.java | 62 ++++++++++++++----- 6 files changed, 94 insertions(+), 51 deletions(-) diff --git a/pom.xml b/pom.xml index 1c7214ad5..2c616138a 100644 --- a/pom.xml +++ b/pom.xml @@ -327,12 +327,6 @@ 5.11.2 test - - org.junit.jupiter - junit-jupiter-params - 5.11.2 - test - org.slf4j slf4j-simple diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index c2bea714b..c9db08306 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -169,6 +169,7 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { + // do nothing } } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 892286f87..7436d7b77 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -328,15 +328,21 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { MySQLAggregateFunction func = aggr.getFunc(); - String option = aggr.getOption(); + String option = func.getOption(); + List exprs = aggr.getExprs(); - sb.append(func); + sb.append(func.getName()); sb.append("("); if (option != null) { sb.append(option); sb.append(" "); } - visit(aggr.getExpr()); + for (int i = 0; i < exprs.size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(exprs.get(i)); + } sb.append(")"); } } diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java index 911b9ddd3..a5ce2932b 100644 --- a/src/sqlancer/mysql/ast/MySQLAggregate.java +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -6,44 +6,54 @@ public class MySQLAggregate implements MySQLExpression { public enum MySQLAggregateFunction { // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_count. - COUNT("DISTINCT"), + COUNT("COUNT", null, false), + COUNT_DISTINCT("COUNT", "DISTINCT", true), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_sum. - SUM("DISTINCT"), + SUM("SUM", null, false), + SUM_DISTINCT("SUM", "DISTINCT", false), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_min. - MIN("DISTINCT"), + MIN("MIN", null, false), + MIN_DISTINCT("MIN", "DISTINCT", false), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_max. - MAX("DISTINCT"); + MAX("MAX", null, false), + MAX_DISTINCT("MAX", "DISTINCT", false); - private final List options; + private final String name; + private final String option; + private final boolean isVariadic; - private MySQLAggregateFunction(String... options) { - this.options = List.of(options); + private MySQLAggregateFunction(String name, String option, boolean isVariadic) { + this.name = name; + this.option = option; + this.isVariadic = isVariadic; } - public List getOptions() { - return options; + public String getName() { + return this.name; + } + + public String getOption() { + return option; + } + + public boolean isVariadic() { + return this.isVariadic; } } - private final MySQLExpression expr; + private final List exprs; private final MySQLAggregateFunction func; - private final String option; - public MySQLAggregate(MySQLExpression expr, MySQLAggregateFunction func, String option) { - this.expr = expr; + public MySQLAggregate(List exprs, MySQLAggregateFunction func) { + this.exprs = exprs; this.func = func; - this.option = option; } - public MySQLExpression getExpr() { - return expr; + public List getExprs() { + return exprs; } public MySQLAggregateFunction getFunc() { return func; } - - public String getOption() { - return option; - } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index a35ee0272..932119a51 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -252,13 +253,16 @@ public String generateExplainQuery(MySQLSelect select) { public MySQLAggregate generateAggregate() { MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); - MySQLExpression expr = generateExpression(); - if (Randomly.getBoolean() && func.getOptions().size() > 0) { - String option = Randomly.fromList(func.getOptions()); - return new MySQLAggregate(expr, func, option); + if (func.isVariadic()) { + int nrExprs = Randomly.smallNumber() + 1; + List exprs = IntStream.range(0, nrExprs) + .mapToObj(index -> generateExpression()) + .collect(Collectors.toList()); + + return new MySQLAggregate(exprs, func); } else { - return new MySQLAggregate(expr, func, null); + return new MySQLAggregate(List.of(generateExpression()), func); } } diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 8a335636c..b5a5d1ffb 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -1,39 +1,67 @@ package sqlancer.mysql; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; + +import java.util.List; + +import org.junit.jupiter.api.Test; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLColumnReference; public class MySQLToStringVisitorTest { - - @ParameterizedTest - @EnumSource(MySQLAggregate.MySQLAggregateFunction.class) - void visitAggregateWithOptions(MySQLAggregate.MySQLAggregateFunction function) { + + @Test + void visitAggregateToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); - MySQLToStringVisitor visitor = new MySQLToStringVisitor(); + MySQLAggregate aggrCount = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT + ); + assertEquals("COUNT(a)", MySQLVisitor.asString(aggrCount)); + + MySQLAggregate aggrSum = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM + ); + assertEquals("SUM(a)", MySQLVisitor.asString(aggrSum)); - for (String option : function.getOptions()) { - visitor.visit(new MySQLAggregate(aRef, function, option)); - assertEquals(String.format("%s(%s a)", function, option), visitor.get()); - } + MySQLAggregate aggrMin = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN + ); + assertEquals("MIN(a)", MySQLVisitor.asString(aggrMin)); + + MySQLAggregate aggrMax = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX + ); + assertEquals("MAX(a)", MySQLVisitor.asString(aggrMax)); } - @ParameterizedTest - @EnumSource(MySQLAggregate.MySQLAggregateFunction.class) - void visitAggregateWithoutOptions(MySQLAggregate.MySQLAggregateFunction function) { + @Test + void visitAggregateWithDistinctToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); - MySQLToStringVisitor visitor = new MySQLToStringVisitor(); + MySQLAggregate aggrCountDistinct = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT_DISTINCT + ); + assertEquals("COUNT(DISTINCT a)", MySQLVisitor.asString(aggrCountDistinct)); + + MySQLAggregate aggrSumDistinct = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM_DISTINCT + ); + assertEquals("SUM(DISTINCT a)", MySQLVisitor.asString(aggrSumDistinct)); + + MySQLAggregate aggrMinDistinct = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN_DISTINCT + ); + assertEquals("MIN(DISTINCT a)", MySQLVisitor.asString(aggrMinDistinct)); - visitor.visit(new MySQLAggregate(aRef, function, null)); - assertEquals(String.format("%s(a)", function), visitor.get()); + MySQLAggregate aggrMaxDistinct = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX_DISTINCT + ); + assertEquals("MAX(DISTINCT a)", MySQLVisitor.asString(aggrMaxDistinct)); } } From 29a2f4d7de1a0f1248c75779a6e5a124a2762461 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 15 Jan 2025 16:54:57 +0800 Subject: [PATCH 1041/1284] [MySQL] Add group by if aggregate has been generated --- src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java index 6f2c2a656..d701072ee 100644 --- a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -25,6 +25,8 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) List allColumns = new ArrayList<>(); List columnsWithoutAggregations = new ArrayList<>(); + boolean hasGeneratedAggregate = false; + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); for (int i = 0; i < nrColumns; i++) { if (Randomly.getBoolean()) { @@ -33,6 +35,7 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) columnsWithoutAggregations.add(expression); } else { allColumns.add(gen.generateAggregate()); + hasGeneratedAggregate = true; } } select.setFetchColumns(allColumns); @@ -46,7 +49,7 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) if (Randomly.getBooleanWithRatherLowProbability()) { select.setOrderByClauses(gen.generateOrderBys()); } - if (Randomly.getBoolean()) { + if (hasGeneratedAggregate || Randomly.getBoolean()) { select.setGroupByExpressions(columnsWithoutAggregations); if (Randomly.getBoolean()) { select.setHavingClause(gen.generateHavingClause()); From 910a0b749ab4e88503a2610bcc45ca4d110279eb Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 15 Jan 2025 19:42:27 +0800 Subject: [PATCH 1042/1284] Format files --- src/sqlancer/mysql/MySQLToStringVisitor.java | 2 +- src/sqlancer/mysql/ast/MySQLAggregate.java | 16 +++---- .../mysql/gen/MySQLExpressionGenerator.java | 9 ++-- .../mysql/MySQLToStringVisitorTest.java | 42 +++++++------------ 4 files changed, 25 insertions(+), 44 deletions(-) diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 7436d7b77..4d9607717 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -6,6 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -26,7 +27,6 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; -import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; public class MySQLToStringVisitor extends ToStringVisitor implements MySQLVisitor { diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java index a5ce2932b..94c4e426b 100644 --- a/src/sqlancer/mysql/ast/MySQLAggregate.java +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -3,26 +3,22 @@ import java.util.List; public class MySQLAggregate implements MySQLExpression { - + public enum MySQLAggregateFunction { // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_count. - COUNT("COUNT", null, false), - COUNT_DISTINCT("COUNT", "DISTINCT", true), + COUNT("COUNT", null, false), COUNT_DISTINCT("COUNT", "DISTINCT", true), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_sum. - SUM("SUM", null, false), - SUM_DISTINCT("SUM", "DISTINCT", false), + SUM("SUM", null, false), SUM_DISTINCT("SUM", "DISTINCT", false), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_min. - MIN("MIN", null, false), - MIN_DISTINCT("MIN", "DISTINCT", false), + MIN("MIN", null, false), MIN_DISTINCT("MIN", "DISTINCT", false), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_max. - MAX("MAX", null, false), - MAX_DISTINCT("MAX", "DISTINCT", false); + MAX("MAX", null, false), MAX_DISTINCT("MAX", "DISTINCT", false); private final String name; private final String option; private final boolean isVariadic; - private MySQLAggregateFunction(String name, String option, boolean isVariadic) { + MySQLAggregateFunction(String name, String option, boolean isVariadic) { this.name = name; this.option = option; this.isVariadic = isVariadic; diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 932119a51..f4b868f61 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -18,6 +18,7 @@ import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; @@ -42,7 +43,6 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; -import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; public class MySQLExpressionGenerator extends UntypedExpressionGenerator @@ -256,10 +256,9 @@ public MySQLAggregate generateAggregate() { if (func.isVariadic()) { int nrExprs = Randomly.smallNumber() + 1; - List exprs = IntStream.range(0, nrExprs) - .mapToObj(index -> generateExpression()) - .collect(Collectors.toList()); - + List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) + .collect(Collectors.toList()); + return new MySQLAggregate(exprs, func); } else { return new MySQLAggregate(List.of(generateExpression()), func); diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index b5a5d1ffb..5aa9b4e14 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -13,55 +13,41 @@ public class MySQLToStringVisitorTest { @Test void visitAggregateToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", - MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); - MySQLAggregate aggrCount = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT - ); + MySQLAggregate aggrCount = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT); assertEquals("COUNT(a)", MySQLVisitor.asString(aggrCount)); - MySQLAggregate aggrSum = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM - ); + MySQLAggregate aggrSum = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM); assertEquals("SUM(a)", MySQLVisitor.asString(aggrSum)); - MySQLAggregate aggrMin = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN - ); + MySQLAggregate aggrMin = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN); assertEquals("MIN(a)", MySQLVisitor.asString(aggrMin)); - MySQLAggregate aggrMax = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX - ); + MySQLAggregate aggrMax = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX); assertEquals("MAX(a)", MySQLVisitor.asString(aggrMax)); } @Test void visitAggregateWithDistinctToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", - MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); - MySQLAggregate aggrCountDistinct = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT_DISTINCT - ); + MySQLAggregate aggrCountDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.COUNT_DISTINCT); assertEquals("COUNT(DISTINCT a)", MySQLVisitor.asString(aggrCountDistinct)); - MySQLAggregate aggrSumDistinct = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM_DISTINCT - ); + MySQLAggregate aggrSumDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.SUM_DISTINCT); assertEquals("SUM(DISTINCT a)", MySQLVisitor.asString(aggrSumDistinct)); - MySQLAggregate aggrMinDistinct = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN_DISTINCT - ); + MySQLAggregate aggrMinDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.MIN_DISTINCT); assertEquals("MIN(DISTINCT a)", MySQLVisitor.asString(aggrMinDistinct)); - MySQLAggregate aggrMaxDistinct = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX_DISTINCT - ); + MySQLAggregate aggrMaxDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.MAX_DISTINCT); assertEquals("MAX(DISTINCT a)", MySQLVisitor.asString(aggrMaxDistinct)); } } From f6c1f033259dc1700da6df4c58f5746ab3bc0409 Mon Sep 17 00:00:00 2001 From: baishen Date: Sun, 19 Jan 2025 11:54:09 +0800 Subject: [PATCH 1043/1284] fix tests --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/ast/DatabendConstant.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e7d69c00a..91188369b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -205,7 +205,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.542-nightly + image: datafuselabs/databend:v1.2.687-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 4c93ac213..142cc3090 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -322,7 +322,8 @@ public static class DatabendDateConstant extends DatabendConstant { public String textRepr; public DatabendDateConstant(long val) { - Timestamp timestamp = new Timestamp(val); + long t = val % 250000000000000l; + Timestamp timestamp = new Timestamp(t); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); textRepr = dateFormat.format(timestamp); } @@ -357,7 +358,8 @@ public static class DatabendTimestampConstant extends DatabendConstant { public String textRepr; public DatabendTimestampConstant(long val) { - Timestamp timestamp = new Timestamp(val); + long t = val % 250000000000000l; + Timestamp timestamp = new Timestamp(t); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); textRepr = dateFormat.format(timestamp); } From d59008dc3134ccdf1158207ca053c1e744accfeb Mon Sep 17 00:00:00 2001 From: baishen Date: Mon, 20 Jan 2025 11:44:57 +0800 Subject: [PATCH 1044/1284] add comments --- src/sqlancer/databend/ast/DatabendConstant.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 142cc3090..f5355df58 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -322,7 +322,9 @@ public static class DatabendDateConstant extends DatabendConstant { public String textRepr; public DatabendDateConstant(long val) { - long t = val % 250000000000000l; + // Databend supports `date` type where the year cannot exceed `9999`, + // the value is truncated to ensure generate legitimate `date` value. + long t = val % 253380000000000L; Timestamp timestamp = new Timestamp(t); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); textRepr = dateFormat.format(timestamp); @@ -358,7 +360,9 @@ public static class DatabendTimestampConstant extends DatabendConstant { public String textRepr; public DatabendTimestampConstant(long val) { - long t = val % 250000000000000l; + // Databend supports `timestamp` type where the year cannot exceed `9999`, + // the value is truncated to ensure generate legitimate `timestamp` value. + long t = val % 253380000000000L; Timestamp timestamp = new Timestamp(t); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); textRepr = dateFormat.format(timestamp); From 292a91b230d94ad433118fbfc5caf34e1c952785 Mon Sep 17 00:00:00 2001 From: baishen Date: Mon, 20 Jan 2025 12:24:14 +0800 Subject: [PATCH 1045/1284] simplify --- src/sqlancer/databend/ast/DatabendConstant.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index f5355df58..919942a8a 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -48,6 +48,13 @@ public double asFloat() { throw new UnsupportedOperationException(this.toString()); } + protected Timestamp truncateTimestamp(long val) { + // Databend supports `date` and `timestamp` type where the year cannot exceed `9999`, + // the value is truncated to ensure generate legitimate `date` and `timestamp` value. + long t = val % 253380000000000L; + return new Timestamp(t); + } + public abstract DatabendConstant isEquals(DatabendConstant rightVal); public abstract DatabendConstant isLessThan(DatabendConstant rightVal); @@ -322,10 +329,7 @@ public static class DatabendDateConstant extends DatabendConstant { public String textRepr; public DatabendDateConstant(long val) { - // Databend supports `date` type where the year cannot exceed `9999`, - // the value is truncated to ensure generate legitimate `date` value. - long t = val % 253380000000000L; - Timestamp timestamp = new Timestamp(t); + Timestamp timestamp = truncateTimestamp(val); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); textRepr = dateFormat.format(timestamp); } @@ -360,10 +364,7 @@ public static class DatabendTimestampConstant extends DatabendConstant { public String textRepr; public DatabendTimestampConstant(long val) { - // Databend supports `timestamp` type where the year cannot exceed `9999`, - // the value is truncated to ensure generate legitimate `timestamp` value. - long t = val % 253380000000000L; - Timestamp timestamp = new Timestamp(t); + Timestamp timestamp = truncateTimestamp(val); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); textRepr = dateFormat.format(timestamp); } From 1458d5e044603aed13514ee2b7368a5fa4fdc1b1 Mon Sep 17 00:00:00 2001 From: MAOXIONGKAI <40223039+MAOXIONGKAI@users.noreply.github.com> Date: Mon, 20 Jan 2025 15:28:52 +0800 Subject: [PATCH 1046/1284] Fix typo in Approaches and Papers section of REAMD.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ce04e254a..f88b77faa 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ SQLancer has pioneered and includes multiple approaches for DBMS testing, as out | Pivoted Query Synthesis (PQS) | OSDI 2020 | [Paper](https://www.usenix.org/system/files/osdi20-rigger.pdf) [Video](https://www.youtube.com/watch?v=0aeDyXgzo04 ) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described here. PQS effectively detects bugs, but requires more implementation effort than other testing approaches that follow a metamorphic testing or differential testing methodology. Thus, it is currently unmaintained. | | Non-optimizing Reference Engine Construction (NoREC) | ESEC/FSE 2020 | [Paper](https://arxiv.org/abs/2007.08292) [Video](https://www.youtube.com/watch?v=4mbzytrWJhQ) | NoREC aims to find optimization bugs. It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. The approach applies primarily to simple queries with a filter predicate. | | Ternary Logic Partitioning (TLP) | OOPSLA 2020 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3428279) [Video](https://www.youtube.com/watch?v=FN9OLbGh0VI) | TLP partitions a query into three partitioning queries, whose results are composed and compared to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. It is among the most widely adopted testing techniques. | -| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | DQP is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | +| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | QPG is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | | Cardinality Estimation Restriction Testing (CERT) | ICSE 2024 | [Paper](https://arxiv.org/pdf/2306.00355) [Code](https://github.com/sqlancer/sqlancer/issues/822) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. From a given input query, it derives a more restrictive query, whose estimated cardinality should be no more than that of the original query. A violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. CERT is the only test oracle that is part of SQLancer that was designed to find performance issues. | | Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | | Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | @@ -112,4 +112,4 @@ Closely related tools: * [Jepsen](https://github.com/jepsen-io): testing of distributed (database) systems * [SQLRight](https://github.com/PSU-Security-Universe/sqlright): coverage-guided DBMS fuzzer, also supporting NoREC and TLP * [SQLsmith](https://github.com/anse1/sqlsmith): random SQL query generator used for fuzzing -* [Squirrel](https://github.com/s3team/Squirrel): coverage-guided DBMS fuzzer \ No newline at end of file +* [Squirrel](https://github.com/s3team/Squirrel): coverage-guided DBMS fuzzer From 6efee976f4c41bfaf053fd4e4baeab8005181ec6 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 5 Feb 2025 10:59:43 +0100 Subject: [PATCH 1047/1284] Update ClickHouseErrors.java (#1071) * Update ClickHouseErrors.java * format --- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index e6a4613d9..2c4d8d1b8 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -55,7 +55,7 @@ public static List getExpectedExpressionErrors() { // AST generation logic "MULTIPLE_EXPRESSIONS_FOR_ALIAS", "AMBIGUOUS_IDENTIFIER", // https://github.com/ClickHouse/ClickHouse/issues/45389 "AMBIGUOUS_COLUMN_NAME", // same https://github.com/ClickHouse/ClickHouse/issues/45389 - "Cannot parse number with multiple sign"); + "No equality condition found in JOIN ON expression", "Cannot parse number with multiple sign"); } public static void addExpectedExpressionErrors(ExpectedErrors errors) { From c7e4ca220789f1e46ef5cc11e209ba3d3e4ac7a0 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 5 Feb 2025 21:26:59 +1030 Subject: [PATCH 1048/1284] Mark support for v13, update test / comment URLs accordingly. As per discussionsin Issue #912 , SQLancer works fine with Postgres v13. This commit updates the test harness to run against Postgres v13 (instead of v12 currently). Also, the documentation links in README & code-comment URLs point to unreleased / outdated postgres documentation versions - For e.g. links were pointing to different postgres version documentations like devel (unreleased) / current / v12 or even v9.5 / v11 (both already EOL). This commit ensures that all www.postgresql.org documentation links in the comments, point to the supported version of postgres (which as of this commit is v13). As and when we advance support for a more recent version, we should update the doc links accordingly too. --- .github/workflows/main.yml | 2 +- src/sqlancer/postgres/PostgresProvider.java | 2 +- .../ast/PostgresFunctionWithUnknownResult.java | 16 ++++++++-------- .../postgres/gen/PostgresSetGenerator.java | 14 +++++++------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91188369b..b42d40aed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -467,7 +467,7 @@ jobs: - name: Set up PostgreSQL uses: harmon758/postgresql-action@v1.0.0 with: - postgresql version: '12' + postgresql version: '13' postgresql user: 'sqlancer' postgresql password: 'sqlancer' postgresql db: 'test' diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 7c86dc628..b255bb542 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -109,7 +109,7 @@ public enum Action implements AbstractAction { RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // COMMENT_ON(PostgresCommentGenerator::generate), // RESET((g) -> new SQLQueryAdapter("RESET ALL") /* - * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also + * https://www.postgresql.org/docs/13/sql-reset.html TODO: also * configuration parameter */), // NOTIFY(PostgresNotifyGenerator::createNotify), // diff --git a/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java b/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java index 3357a7db5..287f46784 100644 --- a/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java +++ b/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java @@ -19,11 +19,11 @@ public enum PostgresFunctionWithUnknownResult { TEXT("text", PostgresDataType.TEXT, PostgresDataType.INET), INET_SAME_FAMILY("inet_same_family", PostgresDataType.BOOLEAN, PostgresDataType.INET, PostgresDataType.INET), - // https://www.postgresql.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-SIGNAL-TABLE + // https://www.postgresql.org/docs/13/functions-admin.html#FUNCTIONS-ADMIN-SIGNAL-TABLE // PG_RELOAD_CONF("pg_reload_conf", PostgresDataType.BOOLEAN), // too much output // PG_ROTATE_LOGFILE("pg_rotate_logfile", PostgresDataType.BOOLEAN), prints warning - // https://www.postgresql.org/docs/devel/functions-info.html#FUNCTIONS-INFO-SESSION-TABLE + // https://www.postgresql.org/docs/13/functions-info.html#FUNCTIONS-INFO-SESSION-TABLE CURRENT_DATABASE("current_database", PostgresDataType.TEXT), // name // CURRENT_QUERY("current_query", PostgresDataType.TEXT), // can generate false positives CURRENT_SCHEMA("current_schema", PostgresDataType.TEXT), // name @@ -87,7 +87,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx TO_HEX("to_hex", PostgresDataType.INT, PostgresDataType.TEXT), TRANSLATE("translate", PostgresDataType.TEXT, PostgresDataType.TEXT, PostgresDataType.TEXT, PostgresDataType.TEXT), // mathematical functions - // https://www.postgresql.org/docs/9.5/functions-math.html + // https://www.postgresql.org/docs/13/functions-math.html ABS("abs", PostgresDataType.REAL, PostgresDataType.REAL), CBRT("cbrt", PostgresDataType.REAL, PostgresDataType.REAL), CEILING("ceiling", PostgresDataType.REAL), // DEGREES("degrees", PostgresDataType.REAL), EXP("exp", PostgresDataType.REAL), LN("ln", PostgresDataType.REAL), @@ -98,7 +98,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx FLOOR("floor", PostgresDataType.REAL), // trigonometric functions - complete - // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-TRIG-TABLE + // https://www.postgresql.org/docs/13/functions-math.html#FUNCTIONS-MATH-TRIG-TABLE ACOS("acos", PostgresDataType.REAL), // ACOSD("acosd", PostgresDataType.REAL), // ASIN("asin", PostgresDataType.REAL), // @@ -117,7 +117,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx TAND("tand", PostgresDataType.REAL), // // hyperbolic functions - complete - // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-HYP-TABLE + // https://www.postgresql.org/docs/13/functions-math.html#FUNCTIONS-MATH-HYP-TABLE SINH("sinh", PostgresDataType.REAL), // COSH("cosh", PostgresDataType.REAL), // TANH("tanh", PostgresDataType.REAL), // @@ -125,12 +125,12 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx ACOSH("acosh", PostgresDataType.REAL), // ATANH("atanh", PostgresDataType.REAL), // - // https://www.postgresql.org/docs/devel/functions-binarystring.html + // https://www.postgresql.org/docs/13/functions-binarystring.html GET_BIT("get_bit", PostgresDataType.INT, PostgresDataType.TEXT, PostgresDataType.INT), GET_BYTE("get_byte", PostgresDataType.INT, PostgresDataType.TEXT, PostgresDataType.INT), // range functions - // https://www.postgresql.org/docs/devel/functions-range.html#RANGE-FUNCTIONS-TABLE + // https://www.postgresql.org/docs/13/functions-range.html#RANGE-FUNCTIONS-TABLE RANGE_LOWER("lower", PostgresDataType.INT, PostgresDataType.RANGE), // RANGE_UPPER("upper", PostgresDataType.INT, PostgresDataType.RANGE), // RANGE_ISEMPTY("isempty", PostgresDataType.BOOLEAN, PostgresDataType.RANGE), // @@ -140,7 +140,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx RANGE_UPPER_INF("upper_inf", PostgresDataType.BOOLEAN, PostgresDataType.RANGE), // RANGE_MERGE("range_merge", PostgresDataType.RANGE, PostgresDataType.RANGE, PostgresDataType.RANGE), // - // https://www.postgresql.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE + // https://www.postgresql.org/docs/13/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE GET_COLUMN_SIZE("get_column_size", PostgresDataType.INT, PostgresDataType.TEXT); // PG_DATABASE_SIZE("pg_database_size", PostgresDataType.INT, PostgresDataType.INT); // PG_SIZE_BYTES("pg_size_bytes", PostgresDataType.INT, PostgresDataType.TEXT); diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index f600f1f04..440172c4b 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -14,7 +14,7 @@ private PostgresSetGenerator() { } private enum ConfigurationOption { - // https://www.postgresql.org/docs/11/runtime-config-wal.html + // https://www.postgresql.org/docs/13/runtime-config-wal.html // This parameter can only be set at server start. // WAL_LEVEL("wal_level", (r) -> Randomly.fromOptions("replica", "minimal", "logical")), // FSYNC("fsync", (r) -> Randomly.fromOptions(1, 0)), @@ -37,7 +37,7 @@ private enum ConfigurationOption { // archive_mode // archive_command // archive_timeout - // https://www.postgresql.org/docs/11/runtime-config-statistics.html + // https://www.postgresql.org/docs/13/runtime-config-statistics.html // 19.9.1. Query and Index Statistics Collector TRACK_ACTIVITIES("track_activities", (r) -> Randomly.fromOptions(1, 0)), // track_activity_query_size @@ -46,7 +46,7 @@ private enum ConfigurationOption { TRACK_FUNCTIONS("track_functions", (r) -> Randomly.fromOptions("'none'", "'pl'", "'all'")), // stats_temp_directory // TODO 19.9.2. Statistics Monitoring - // https://www.postgresql.org/docs/11/runtime-config-autovacuum.html + // https://www.postgresql.org/docs/13/runtime-config-autovacuum.html // all can only be set at server-conf time // 19.11. Client Connection Defaults VACUUM_FREEZE_TABLE_AGE("vacuum_freeze_table_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), @@ -60,7 +60,7 @@ private enum ConfigurationOption { // 19.13. Version and Platform Compatibility DEFAULT_WITH_OIDS("default_with_oids", (r) -> Randomly.fromOptions(0, 1)), SYNCHRONIZED_SEQSCANS("synchronize_seqscans", (r) -> Randomly.fromOptions(0, 1)), - // https://www.postgresql.org/docs/devel/runtime-config-query.html + // https://www.postgresql.org/docs/13/runtime-config-query.html ENABLE_BITMAPSCAN("enable_bitmapscan", (r) -> Randomly.fromOptions(1, 0)), ENABLE_GATHERMERGE("enable_gathermerge", (r) -> Randomly.fromOptions(1, 0)), ENABLE_HASHJOIN("enable_hashjoin", (r) -> Randomly.fromOptions(1, 0)), @@ -78,7 +78,7 @@ private enum ConfigurationOption { ENABLE_SORT("enable_sort", (r) -> Randomly.fromOptions(1, 0)), ENABLE_TIDSCAN("enable_tidscan", (r) -> Randomly.fromOptions(1, 0)), // 19.7.2. Planner Cost Constants (complete as of March 2020) - // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-CONSTANTS + // https://www.postgresql.org/docs/13/runtime-config-query.html#RUNTIME-CONFIG-QUERY-CONSTANTS SEQ_PAGE_COST("seq_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), RANDOM_PAGE_COST("random_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), CPU_TUPLE_COST("cpu_tuple_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), @@ -94,7 +94,7 @@ private enum ConfigurationOption { JIT_OPTIMIZE_ABOVE_COST("jit_optimize_above_cost", (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE))), // 19.7.3. Genetic Query Optimizer (complete as of March 2020) - // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-GEQO + // https://www.postgresql.org/docs/13/runtime-config-query.html#RUNTIME-CONFIG-QUERY-GEQO GEQO("geqo", (r) -> Randomly.fromOptions(1, 0)), GEQO_THRESHOLD("geqo_threshold", (r) -> r.getInteger(2, 2147483647)), GEQO_EFFORT("geqo_effort", (r) -> r.getInteger(1, 10)), @@ -103,7 +103,7 @@ private enum ConfigurationOption { GEQO_SELECTION_BIAS("geqo_selection_bias", (r) -> Randomly.fromOptions(1.5, 1.8, 2.0)), GEQO_SEED("geqo_seed", (r) -> Randomly.fromOptions(0, 0.5, 1)), // 19.7.4. Other Planner Options (complete as of March 2020) - // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-OTHER + // https://www.postgresql.org/docs/13/runtime-config-query.html#RUNTIME-CONFIG-QUERY-OTHER DEFAULT_STATISTICS_TARGET("default_statistics_target", (r) -> r.getInteger(1, 10000)), CONSTRAINT_EXCLUSION("constraint_exclusion", (r) -> Randomly.fromOptions("on", "off", "partition")), CURSOR_TUPLE_FRACTION("cursor_tuple_fraction", From 1a9e496eb1e98a90ebf290d333aa1306df3e2fa1 Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Mon, 3 Mar 2025 15:19:25 +0200 Subject: [PATCH 1049/1284] add postgres explain generator --- .../gen/PostgresExplainGenerator.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/sqlancer/postgres/gen/PostgresExplainGenerator.java diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java new file mode 100644 index 000000000..318e5eead --- /dev/null +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -0,0 +1,36 @@ +package sqlancer.postgres.gen; + +import jnr.ffi.Struct; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresProvider; +import sqlancer.postgres.PostgresProvider.Action; + +public class PostgresExplainGenerator { + + private PostgresExplainGenerator(){ + + } + + public static SQLQueryAdapter explain(PostgresGlobalState globalState) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + Action action; + + do { + action = Randomly.fromOptions(PostgresProvider.Action.values()); + } while (action == Action.EXPLAIN); + SQLQueryAdapter query = action.getQuery(globalState); + sb.append(query); + return new SQLQueryAdapter(sb.toString(),query.getExpectedErrors()); + } + + public static String explain(String selectStr) throws Exception{ + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + sb.append(selectStr); + return sb.toString(); + } + +} From ee8dd08d8a0d459bed4eaa245aa2d727a0e7bb5d Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Mon, 3 Mar 2025 18:45:36 +0200 Subject: [PATCH 1050/1284] add QPG for postgres --- src/sqlancer/postgres/PostgresProvider.java | 78 ++++++++++++++----- .../gen/PostgresExplainGenerator.java | 1 - .../postgres/gen/PostgresInsertGenerator.java | 4 + 3 files changed, 62 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 7c86dc628..f88c2c65b 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -1,5 +1,6 @@ package sqlancer.postgres; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.sql.Connection; @@ -7,6 +8,8 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -22,26 +25,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.gen.PostgresAlterTableGenerator; -import sqlancer.postgres.gen.PostgresAnalyzeGenerator; -import sqlancer.postgres.gen.PostgresClusterGenerator; -import sqlancer.postgres.gen.PostgresCommentGenerator; -import sqlancer.postgres.gen.PostgresDeleteGenerator; -import sqlancer.postgres.gen.PostgresDiscardGenerator; -import sqlancer.postgres.gen.PostgresDropIndexGenerator; -import sqlancer.postgres.gen.PostgresIndexGenerator; -import sqlancer.postgres.gen.PostgresInsertGenerator; -import sqlancer.postgres.gen.PostgresNotifyGenerator; -import sqlancer.postgres.gen.PostgresReindexGenerator; -import sqlancer.postgres.gen.PostgresSequenceGenerator; -import sqlancer.postgres.gen.PostgresSetGenerator; -import sqlancer.postgres.gen.PostgresStatisticsGenerator; -import sqlancer.postgres.gen.PostgresTableGenerator; -import sqlancer.postgres.gen.PostgresTransactionGenerator; -import sqlancer.postgres.gen.PostgresTruncateGenerator; -import sqlancer.postgres.gen.PostgresUpdateGenerator; -import sqlancer.postgres.gen.PostgresVacuumGenerator; -import sqlancer.postgres.gen.PostgresViewGenerator; +import sqlancer.postgres.gen.*; // EXISTS // IN @@ -73,6 +57,7 @@ protected PostgresProvider(Class globalClass, Class { + EXPLAIN(PostgresExplainGenerator::explain), ANALYZE(PostgresAnalyzeGenerator::create), // ALTER_TABLE(g -> PostgresAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g, generateOnlyKnown)), // @@ -146,6 +131,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { nrPerformed = r.getInteger(0, 5); break; case COMMIT: + case EXPLAIN: nrPerformed = r.getInteger(0, 0); break; case ALTER_TABLE: @@ -183,6 +169,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; + default: throw new AssertionError(a); } @@ -349,4 +336,55 @@ public String getDBMSName() { return "postgres"; } + + @Override + public String getQueryPlan(String selectStr, PostgresGlobalState globalState) throws Exception { + String queryPlan = ""; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(selectStr); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + List invalidList = List.of("create", "notify", "discard", "listen", "unlisten", "reset","set","delete","alter","analyze"); + if(invalidList.contains(selectStr.split("\\s+")[0].toLowerCase())){ + return ""; + } + + SQLQueryAdapter q = new SQLQueryAdapter(PostgresExplainGenerator.explain(selectStr),null); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + while (rs.next()) { + queryPlan += rs.getString(1); + } + } catch (SQLException | AssertionError e) { + queryPlan = ""; + } + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[PostgresProvider.Action.values().length]; + } + + @Override + protected void executeMutator(int index, PostgresGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = PostgresProvider.Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + + @Override + protected boolean addRowsToAllTables(PostgresGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (PostgresSchema.PostgresTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = PostgresInsertGenerator.insertRows(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index 318e5eead..9d270c711 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -1,6 +1,5 @@ package sqlancer.postgres.gen; -import jnr.ffi.Struct; import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 41c017b2e..71dc9a852 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; @@ -19,6 +20,9 @@ private PostgresInsertGenerator() { public static SQLQueryAdapter insert(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + return insertRows(globalState,table); + } + public static SQLQueryAdapter insertRows(PostgresGlobalState globalState, PostgresSchema.PostgresTable table) { ExpectedErrors errors = new ExpectedErrors(); errors.add("cannot insert into column"); PostgresCommon.addCommonExpressionErrors(errors); From b4e619c5c051bb0ff97d429bd519f4acf3cdfd6d Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 5 Mar 2025 11:18:01 +0200 Subject: [PATCH 1051/1284] remove code not related to QPG support --- src/sqlancer/postgres/PostgresProvider.java | 2 -- .../gen/PostgresExplainGenerator.java | 21 +------------------ 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index f88c2c65b..eb406a505 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -57,7 +57,6 @@ protected PostgresProvider(Class globalClass, Class { - EXPLAIN(PostgresExplainGenerator::explain), ANALYZE(PostgresAnalyzeGenerator::create), // ALTER_TABLE(g -> PostgresAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g, generateOnlyKnown)), // @@ -131,7 +130,6 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { nrPerformed = r.getInteger(0, 5); break; case COMMIT: - case EXPLAIN: nrPerformed = r.getInteger(0, 0); break; case ALTER_TABLE: diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index 9d270c711..db082ff68 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -1,30 +1,11 @@ package sqlancer.postgres.gen; -import sqlancer.Randomly; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresProvider; -import sqlancer.postgres.PostgresProvider.Action; - public class PostgresExplainGenerator { private PostgresExplainGenerator(){ } - - public static SQLQueryAdapter explain(PostgresGlobalState globalState) throws Exception { - StringBuilder sb = new StringBuilder(); - sb.append("EXPLAIN "); - Action action; - - do { - action = Randomly.fromOptions(PostgresProvider.Action.values()); - } while (action == Action.EXPLAIN); - SQLQueryAdapter query = action.getQuery(globalState); - sb.append(query); - return new SQLQueryAdapter(sb.toString(),query.getExpectedErrors()); - } - + public static String explain(String selectStr) throws Exception{ StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN "); From d3d02ff03658515305efed63cbc0f85e2eb9834a Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 5 Mar 2025 12:15:35 +0200 Subject: [PATCH 1052/1284] format query plan string --- src/sqlancer/postgres/PostgresProvider.java | 25 ++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index eb406a505..fc38b45ff 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -360,7 +360,7 @@ public String getQueryPlan(String selectStr, PostgresGlobalState globalState) th } catch (SQLException | AssertionError e) { queryPlan = ""; } - return queryPlan; + return formatQueryPlan(queryPlan); } @Override @@ -385,4 +385,27 @@ protected boolean addRowsToAllTables(PostgresGlobalState globalState) throws Exc return true; } + private String formatQueryPlan(String queryPlan){ + StringBuilder outQueryPlanFormatted = new StringBuilder(); + boolean insideBrackets = false; + + for(char ch: queryPlan.toCharArray()){ + if(ch == '\n' || ch == ' '){ + continue; + } + if(ch == '('){ + insideBrackets = true; + } + else if(ch == ')'){ + insideBrackets = false; + outQueryPlanFormatted.append(';'); + continue; + } + if (!insideBrackets) + outQueryPlanFormatted.append(ch); + } + + return outQueryPlanFormatted.toString(); + } + } From 9c5509f4144951e0eb6d9dc5367dda90374f1630 Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 5 Mar 2025 12:17:00 +0200 Subject: [PATCH 1053/1284] add integration test for QPG postgres --- .../qpg/postgres/TestPostgresQPG.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test/sqlancer/qpg/postgres/TestPostgresQPG.java diff --git a/test/sqlancer/qpg/postgres/TestPostgresQPG.java b/test/sqlancer/qpg/postgres/TestPostgresQPG.java new file mode 100644 index 000000000..03e7b7ab1 --- /dev/null +++ b/test/sqlancer/qpg/postgres/TestPostgresQPG.java @@ -0,0 +1,21 @@ +package sqlancer.qpg.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestPostgresQPG { + + @Test + public void testPostgresQPG() { + String postgres = System.getenv("POSTGRES_AVAILABLE"); + boolean postgresIsAvailable = postgres != null && postgres.equalsIgnoreCase("true"); + assumeTrue(postgresIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "--username", + "postgres", "postgres", "--oracle", "NOREC"})); + } +} From 645b92a6d9e134ce00e71a2a4f62fdb6f20ad960 Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 5 Mar 2025 12:17:44 +0200 Subject: [PATCH 1054/1284] add unit test for postgres::getQueryPlan --- .../qpg/postgres/TestPostgresQueryPlan.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java diff --git a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java new file mode 100644 index 000000000..5901f84b7 --- /dev/null +++ b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java @@ -0,0 +1,41 @@ +package sqlancer.qpg.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.PostgresProvider; + +public class TestPostgresQueryPlan { + + + @Test + void testPostgresQueryPlan() throws Exception { + String databaseName = "postgres"; + PostgresProvider provider = new PostgresProvider(); + PostgresGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + PostgresOptions postgresOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(postgresOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + assertEquals("NestedLoopLeftJoin;->SeqScanont2;->Materialize;->SeqScanont1;Filter:;", queryPlan); + } + +} From 502a69ba10a55efaf4809b3287557d796a52f7e2 Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Fri, 7 Mar 2025 02:54:49 +0200 Subject: [PATCH 1055/1284] fix checkstyle violations --- src/sqlancer/postgres/PostgresProvider.java | 44 ++++++++++++++----- .../gen/PostgresExplainGenerator.java | 8 ++-- .../postgres/gen/PostgresInsertGenerator.java | 3 +- .../qpg/postgres/TestPostgresQPG.java | 7 +-- .../qpg/postgres/TestPostgresQueryPlan.java | 1 - 5 files changed, 42 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index fc38b45ff..32f74789c 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -25,7 +25,27 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.gen.*; +import sqlancer.postgres.gen.PostgresAlterTableGenerator; +import sqlancer.postgres.gen.PostgresAnalyzeGenerator; +import sqlancer.postgres.gen.PostgresClusterGenerator; +import sqlancer.postgres.gen.PostgresCommentGenerator; +import sqlancer.postgres.gen.PostgresDeleteGenerator; +import sqlancer.postgres.gen.PostgresDiscardGenerator; +import sqlancer.postgres.gen.PostgresDropIndexGenerator; +import sqlancer.postgres.gen.PostgresExplainGenerator; +import sqlancer.postgres.gen.PostgresIndexGenerator; +import sqlancer.postgres.gen.PostgresInsertGenerator; +import sqlancer.postgres.gen.PostgresNotifyGenerator; +import sqlancer.postgres.gen.PostgresReindexGenerator; +import sqlancer.postgres.gen.PostgresSequenceGenerator; +import sqlancer.postgres.gen.PostgresSetGenerator; +import sqlancer.postgres.gen.PostgresStatisticsGenerator; +import sqlancer.postgres.gen.PostgresTableGenerator; +import sqlancer.postgres.gen.PostgresTransactionGenerator; +import sqlancer.postgres.gen.PostgresTruncateGenerator; +import sqlancer.postgres.gen.PostgresUpdateGenerator; +import sqlancer.postgres.gen.PostgresVacuumGenerator; +import sqlancer.postgres.gen.PostgresViewGenerator; // EXISTS // IN @@ -334,7 +354,6 @@ public String getDBMSName() { return "postgres"; } - @Override public String getQueryPlan(String selectStr, PostgresGlobalState globalState) throws Exception { String queryPlan = ""; @@ -347,12 +366,13 @@ public String getQueryPlan(String selectStr, PostgresGlobalState globalState) th } } - List invalidList = List.of("create", "notify", "discard", "listen", "unlisten", "reset","set","delete","alter","analyze"); - if(invalidList.contains(selectStr.split("\\s+")[0].toLowerCase())){ + List invalidList = List.of("create", "notify", "discard", "listen", "unlisten", "reset", "set", + "delete", "alter", "analyze"); + if (invalidList.contains(selectStr.split("\\s+")[0].toLowerCase())) { return ""; } - SQLQueryAdapter q = new SQLQueryAdapter(PostgresExplainGenerator.explain(selectStr),null); + SQLQueryAdapter q = new SQLQueryAdapter(PostgresExplainGenerator.explain(selectStr), null); try (SQLancerResultSet rs = q.executeAndGet(globalState)) { while (rs.next()) { queryPlan += rs.getString(1); @@ -385,24 +405,24 @@ protected boolean addRowsToAllTables(PostgresGlobalState globalState) throws Exc return true; } - private String formatQueryPlan(String queryPlan){ + private String formatQueryPlan(String queryPlan) { StringBuilder outQueryPlanFormatted = new StringBuilder(); boolean insideBrackets = false; - for(char ch: queryPlan.toCharArray()){ - if(ch == '\n' || ch == ' '){ + for (char ch : queryPlan.toCharArray()) { + if (ch == '\n' || ch == ' ') { continue; } - if(ch == '('){ + if (ch == '(') { insideBrackets = true; - } - else if(ch == ')'){ + } else if (ch == ')') { insideBrackets = false; outQueryPlanFormatted.append(';'); continue; } - if (!insideBrackets) + if (!insideBrackets) { outQueryPlanFormatted.append(ch); + } } return outQueryPlanFormatted.toString(); diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index db082ff68..d55d4b72c 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -1,12 +1,12 @@ package sqlancer.postgres.gen; -public class PostgresExplainGenerator { +public final class PostgresExplainGenerator { - private PostgresExplainGenerator(){ + private PostgresExplainGenerator() { } - - public static String explain(String selectStr) throws Exception{ + + public static String explain(String selectStr) throws Exception { StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN "); sb.append(selectStr); diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 71dc9a852..49b94b184 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -20,8 +20,9 @@ private PostgresInsertGenerator() { public static SQLQueryAdapter insert(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); - return insertRows(globalState,table); + return insertRows(globalState, table); } + public static SQLQueryAdapter insertRows(PostgresGlobalState globalState, PostgresSchema.PostgresTable table) { ExpectedErrors errors = new ExpectedErrors(); errors.add("cannot insert into column"); diff --git a/test/sqlancer/qpg/postgres/TestPostgresQPG.java b/test/sqlancer/qpg/postgres/TestPostgresQPG.java index 03e7b7ab1..20ec3e91e 100644 --- a/test/sqlancer/qpg/postgres/TestPostgresQPG.java +++ b/test/sqlancer/qpg/postgres/TestPostgresQPG.java @@ -14,8 +14,9 @@ public void testPostgresQPG() { String postgres = System.getenv("POSTGRES_AVAILABLE"); boolean postgresIsAvailable = postgres != null && postgres.equalsIgnoreCase("true"); assumeTrue(postgresIsAvailable); - assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "--username", - "postgres", "postgres", "--oracle", "NOREC"})); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, + "--username", "postgres", "postgres", "--oracle", "NOREC" })); } } diff --git a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java index 5901f84b7..f6a71df8f 100644 --- a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java +++ b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java @@ -13,7 +13,6 @@ public class TestPostgresQueryPlan { - @Test void testPostgresQueryPlan() throws Exception { String databaseName = "postgres"; From 3c9bcb49d16a3ee54ed79fa5f9301e47782e6b92 Mon Sep 17 00:00:00 2001 From: SharmaNishchay <146124877+SharmaNishchay@users.noreply.github.com> Date: Sun, 9 Mar 2025 17:40:24 +0530 Subject: [PATCH 1056/1284] Refactor: Replace .size() == 0 or .length() > 0 etc checks with .isEmpty() --- .../clickhouse/ast/constant/ClickHouseStringConstant.java | 2 +- src/sqlancer/cnosdb/CnosDBLoggableFactory.java | 2 +- src/sqlancer/cnosdb/CnosDBToStringVisitor.java | 2 +- .../cockroachdb/gen/CockroachDBExpressionGenerator.java | 6 +++--- .../cockroachdb/oracle/CockroachDBCERTOracle.java | 4 ++-- .../cockroachdb/oracle/tlp/CockroachDBTLPBase.java | 2 +- src/sqlancer/common/log/SQLLoggableFactory.java | 2 +- src/sqlancer/doris/gen/DorisDropViewGenerator.java | 2 +- src/sqlancer/mariadb/ast/MariaDBStringVisitor.java | 2 +- src/sqlancer/materialize/MaterializeToStringVisitor.java | 2 +- src/sqlancer/mysql/MySQLToStringVisitor.java | 4 ++-- src/sqlancer/mysql/ast/MySQLBinaryOperation.java | 8 ++++---- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 4 ++-- src/sqlancer/oceanbase/OceanBaseToStringVisitor.java | 6 +++--- src/sqlancer/oceanbase/ast/OceanBaseConstant.java | 2 +- src/sqlancer/postgres/PostgresToStringVisitor.java | 2 +- .../postgres/gen/PostgresExpressionGenerator.java | 4 ++-- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 2 +- src/sqlancer/tidb/TiDBExpressionGenerator.java | 4 ++-- src/sqlancer/tidb/gen/TiDBDropViewGenerator.java | 2 +- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 2 +- src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java | 2 +- 22 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java index bde359f92..fa8ba11f5 100644 --- a/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java @@ -32,7 +32,7 @@ public String toString() { @Override public boolean asBooleanNotNull() { - return value.length() > 0; + return !value.isEmpty(); } @Override diff --git a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java index a3f361b14..407621c8b 100644 --- a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java +++ b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java @@ -19,7 +19,7 @@ protected Loggable createLoggable(String input, String suffix) { if (!input.endsWith(";")) { completeString += ";"; } - if (suffix != null && suffix.length() != 0) { + if (suffix != null && !suffix.isEmpty()) { completeString += suffix; } return new LoggedString(completeString); diff --git a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java index 44d3bf509..388e2ccd8 100644 --- a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java +++ b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java @@ -137,7 +137,7 @@ public void visit(CnosDBSelect s) { sb.append(" WHERE "); visit(s.getWhereClause()); } - if (s.getGroupByExpressions().size() > 0) { + if (!s.getGroupByExpressions().isEmpty()) { sb.append(" GROUP BY "); visit(s.getGroupByExpressions()); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index 52d8f027d..8aeb5492b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -457,7 +457,7 @@ public String generateUnoptimizedQueryString(CockroachDBSelect select, Cockroach @Override public List generateFetchColumns(boolean shouldCreateDummy) { - if (shouldCreateDummy || columns.size() == 0) { + if (shouldCreateDummy || columns.isEmpty()) { return List.of(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); } return Randomly.nonEmptySubset(columns).stream().map(c -> new CockroachDBColumnReference(c)) @@ -541,7 +541,7 @@ boolean mutateWhere(CockroachDBSelect select) { } boolean mutateGroupBy(CockroachDBSelect select) { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); } else { @@ -551,7 +551,7 @@ boolean mutateGroupBy(CockroachDBSelect select) { } boolean mutateHaving(CockroachDBSelect select) { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(generateExpression(CockroachDBDataType.BOOL.get())); return false; diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java index 07a51861f..7d2af5930 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -177,7 +177,7 @@ protected boolean mutateWhere() { @Override protected boolean mutateGroupBy() { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); } else { @@ -188,7 +188,7 @@ protected boolean mutateGroupBy() { @Override protected boolean mutateHaving() { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); return false; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index c4809d231..d84cab56f 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -56,7 +56,7 @@ public void check() throws SQLException { List generateFetchColumns() { List columns = new ArrayList<>(); - if (Randomly.getBoolean() || targetTables.getColumns().size() == 0) { + if (Randomly.getBoolean() || targetTables.getColumns().isEmpty()) { columns.add(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); } else { columns.addAll(Randomly.nonEmptySubset(targetTables.getColumns()).stream() diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java index 7b7dfe4f1..bdcf2253f 100644 --- a/src/sqlancer/common/log/SQLLoggableFactory.java +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -16,7 +16,7 @@ protected Loggable createLoggable(String input, String suffix) { } completeString = completeString.replace("\n", "\\n"); completeString = completeString.replace("\r", "\\r"); - if (suffix != null && suffix.length() != 0) { + if (suffix != null && !suffix.isEmpty()) { completeString += suffix; } return new LoggedString(completeString); diff --git a/src/sqlancer/doris/gen/DorisDropViewGenerator.java b/src/sqlancer/doris/gen/DorisDropViewGenerator.java index 81c1629c2..4f87ba88d 100644 --- a/src/sqlancer/doris/gen/DorisDropViewGenerator.java +++ b/src/sqlancer/doris/gen/DorisDropViewGenerator.java @@ -11,7 +11,7 @@ private DorisDropViewGenerator() { } public static SQLQueryAdapter dropView(DorisGlobalState globalState) { - if (globalState.getSchema().getTables(t -> t.isView()).size() == 0) { + if (globalState.getSchema().getTables(t -> t.isView()).isEmpty()) { throw new IgnoreMeException(); } StringBuilder sb = new StringBuilder("DROP VIEW "); diff --git a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java index e1dbf476f..f7fed0c83 100644 --- a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java @@ -54,7 +54,7 @@ public void visit(MariaDBSelectStatement s) { sb.append(" WHERE "); visit(s.getWhereCondition()); } - if (s.getGroupBys().size() != 0) { + if (!s.getGroupBys().isEmpty()) { sb.append(" GROUP BY "); for (i = 0; i < s.getGroupBys().size(); i++) { if (i != 0) { diff --git a/src/sqlancer/materialize/MaterializeToStringVisitor.java b/src/sqlancer/materialize/MaterializeToStringVisitor.java index b39179cae..285be276f 100644 --- a/src/sqlancer/materialize/MaterializeToStringVisitor.java +++ b/src/sqlancer/materialize/MaterializeToStringVisitor.java @@ -139,7 +139,7 @@ public void visit(MaterializeSelect s) { sb.append(" WHERE "); visit(s.getWhereClause()); } - if (s.getGroupByExpressions().size() > 0) { + if (!s.getGroupByExpressions().isEmpty()) { sb.append(" GROUP BY "); visit(s.getGroupByExpressions()); } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 4d9607717..b459fe81e 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -59,7 +59,7 @@ public void visit(MySQLSelect s) { throw new AssertionError(); } sb.append(s.getModifiers().stream().collect(Collectors.joining(" "))); - if (s.getModifiers().size() > 0) { + if (!s.getModifiers().isEmpty()) { sb.append(" "); } if (s.getFetchColumns() == null) { @@ -92,7 +92,7 @@ public void visit(MySQLSelect s) { sb.append(" WHERE "); visit(whereClause); } - if (s.getGroupByExpressions() != null && s.getGroupByExpressions().size() > 0) { + if (s.getGroupByExpressions() != null && !s.getGroupByExpressions().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); List groupBys = s.getGroupByExpressions(); diff --git a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java index 33293f26b..b3e0714cb 100644 --- a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java +++ b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java @@ -78,20 +78,20 @@ public MySQLConstant getExpectedValue() { /* workaround for https://bugs.mysql.com/bug.php?id=95960 */ if (leftExpected.isString()) { String text = leftExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && text.length() > 0) { + while ((text.startsWith(" ") || text.startsWith("\t")) && !text.isEmpty()) { text = text.substring(1); } - if (text.length() > 0 && (text.startsWith("\n") || text.startsWith("."))) { + if (!text.isEmpty() && (text.startsWith("\n") || text.startsWith("."))) { throw new IgnoreMeException(); } } if (rightExpected.isString()) { String text = rightExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && text.length() > 0) { + while ((text.startsWith(" ") || text.startsWith("\t")) && !text.isEmpty()) { text = text.substring(1); } - if (text.length() > 0 && (text.startsWith("\n") || text.startsWith("."))) { + if (!text.isEmpty() && (text.startsWith("\n") || text.startsWith("."))) { throw new IgnoreMeException(); } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index f4b868f61..f23c48c29 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -301,7 +301,7 @@ boolean mutateWhere(MySQLSelect select) { } boolean mutateGroupBy(MySQLSelect select) { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); } else { @@ -311,7 +311,7 @@ boolean mutateGroupBy(MySQLSelect select) { } boolean mutateHaving(MySQLSelect select) { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(generateExpression()); return false; diff --git a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java index a65ae8ee3..ba306ff90 100644 --- a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java @@ -56,7 +56,7 @@ public void visit(OceanBaseSelect s) { throw new AssertionError(); } sb.append(s.getModifiers().stream().collect(Collectors.joining(" "))); - if (s.getModifiers().size() > 0) { + if (!s.getModifiers().isEmpty()) { sb.append(" "); } if (s.getFetchColumns() == null) { @@ -85,7 +85,7 @@ public void visit(OceanBaseSelect s) { sb.append(" WHERE "); visit(whereClause); } - if (s.getGroupByExpressions() != null && s.getGroupByExpressions().size() > 0) { + if (s.getGroupByExpressions() != null && !s.getGroupByExpressions().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); List groupBys = s.getGroupByExpressions(); @@ -260,7 +260,7 @@ public void visit(OceanBaseStringExpression op) { sb.append(op.getStr()); } else { String str = op.getStr(); - if (str.length() > 0) { + if (!str.isEmpty()) { sb.append(r.getInteger(0, 100000)); } else { sb.append(r.getInteger(0, 1000000)); diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java index 91a481fc6..84dadb9e4 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -194,7 +194,7 @@ public boolean isNull() { @Override public boolean isEmpty() { // "" " " - if (value.length() == 0) { + if (value.isEmpty()) { return true; } else { for (int i = 0; i < value.length(); i++) { diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 17327ab9c..78475a0af 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -161,7 +161,7 @@ public void visit(PostgresSelect s) { sb.append(" WHERE "); visit(s.getWhereClause()); } - if (s.getGroupByExpressions().size() > 0) { + if (!s.getGroupByExpressions().isEmpty()) { sb.append(" GROUP BY "); visit(s.getGroupByExpressions()); } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 5f081ef2a..c6189acc5 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -819,7 +819,7 @@ boolean mutateWhere(PostgresSelect select) { } boolean mutateGroupBy(PostgresSelect select) { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); } else { @@ -829,7 +829,7 @@ boolean mutateGroupBy(PostgresSelect select) { } boolean mutateHaving(PostgresSelect select) { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(generateExpression(0, PostgresDataType.BOOLEAN)); return false; diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index b8b860943..09eb224fb 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -139,7 +139,7 @@ public void visit(SQLite3Select s, boolean inner) { visit(whereClause); sb.append(")"); } - if (s.getGroupByClause().size() > 0) { + if (!s.getGroupByClause().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); visit(s.getGroupByClause()); diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index ff6a529d0..8eaca35f0 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -318,7 +318,7 @@ boolean mutateWhere(TiDBSelect select) { } boolean mutateHaving(TiDBSelect select) { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(generateExpression()); return false; @@ -367,7 +367,7 @@ boolean mutateLimit(TiDBSelect select) { } private boolean mutateGroupBy(TiDBSelect select) { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); select.clearHavingClause(); diff --git a/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java index 48bfc8a46..486b5f873 100644 --- a/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java @@ -11,7 +11,7 @@ private TiDBDropViewGenerator() { } public static SQLQueryAdapter dropView(TiDBGlobalState globalState) { - if (globalState.getSchema().getTables(t -> t.isView()).size() == 0) { + if (globalState.getSchema().getTables(t -> t.isView()).isEmpty()) { throw new IgnoreMeException(); } StringBuilder sb = new StringBuilder("DROP VIEW "); diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index af9ad4457..451681409 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -47,7 +47,7 @@ public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException StringBuilder sb = new StringBuilder("CREATE TABLE "); sb.append(tableName); - if (Randomly.getBoolean() && globalState.getSchema().getDatabaseTables().size() > 0) { + if (Randomly.getBoolean() && !globalState.getSchema().getDatabaseTables().isEmpty()) { sb.append(" LIKE "); TiDBTable otherTable = globalState.getSchema().getRandomTable(); sb.append(otherTable.getName()); diff --git a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java index eef4c7d9f..54e718768 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java +++ b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java @@ -127,7 +127,7 @@ public void visit(YSQLSelect s) { sb.append(" WHERE "); visit(s.getWhereClause()); } - if (s.getGroupByExpressions().size() > 0) { + if (!s.getGroupByExpressions().isEmpty()) { sb.append(" GROUP BY "); visit(s.getGroupByExpressions()); } From 37e72a04c578d5ee55efb7bfac013e28264b8fe3 Mon Sep 17 00:00:00 2001 From: Nishchay Sharma <146124877+SharmaNishchay@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:22:46 +0530 Subject: [PATCH 1057/1284] Fixes the typo on line 598 (#1097) --- src/sqlancer/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 46841095b..7df750bf1 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -595,7 +595,7 @@ public void run() { System.out.println( formatInteger(nrSuccessfulActions.get()) + " successfully-executed statements"); System.out.println( - formatInteger(nrUnsuccessfulActions.get()) + " unsuccessfuly-executed statements"); + formatInteger(nrUnsuccessfulActions.get()) + " unsuccessfully-executed statements"); } private String formatInteger(long intValue) { From 52d7224f13ed9be81c8befb1f0dd19b620e3122c Mon Sep 17 00:00:00 2001 From: SharmaNishchay <146124877+SharmaNishchay@users.noreply.github.com> Date: Tue, 11 Mar 2025 21:51:14 +0530 Subject: [PATCH 1058/1284] Remove redundant text.length() checks --- src/sqlancer/mysql/ast/MySQLBinaryOperation.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java index b3e0714cb..af0b0ccf7 100644 --- a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java +++ b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java @@ -78,20 +78,20 @@ public MySQLConstant getExpectedValue() { /* workaround for https://bugs.mysql.com/bug.php?id=95960 */ if (leftExpected.isString()) { String text = leftExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && !text.isEmpty()) { + while (text.startsWith(" ") || text.startsWith("\t")) { text = text.substring(1); } - if (!text.isEmpty() && (text.startsWith("\n") || text.startsWith("."))) { + if (text.startsWith("\n") || text.startsWith(".")) { throw new IgnoreMeException(); } } if (rightExpected.isString()) { String text = rightExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && !text.isEmpty()) { + while (text.startsWith(" ") || text.startsWith("\t")) { text = text.substring(1); } - if (!text.isEmpty() && (text.startsWith("\n") || text.startsWith("."))) { + if (text.startsWith("\n") || text.startsWith(".")) { throw new IgnoreMeException(); } } From 049176b73af1e0f5af7c24fb2e581b9a265424b1 Mon Sep 17 00:00:00 2001 From: Amr Riyad Date: Tue, 11 Mar 2025 20:43:40 +0200 Subject: [PATCH 1059/1284] [CockroachDB] Add 'excluded' support for 'ON CONFLICT' clause --- .../cockroachdb/gen/CockroachDBInsertGenerator.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java index decbe927f..9f823c59b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java @@ -81,8 +81,6 @@ public static SQLQueryAdapter insert(CockroachDBGlobalState globalState, Cockroa if (Randomly.getBoolean()) { sb.append(" NOTHING "); } else { - // TODO: also support excluded. (see - // https://www.cockroachlabs.com/docs/stable/insert.html) sb.append(" UPDATE SET "); List columns = table.getRandomNonEmptyColumnSubset(); int i = 0; @@ -92,7 +90,12 @@ public static SQLQueryAdapter insert(CockroachDBGlobalState globalState, Cockroa } sb.append(c.getName()); sb.append(" = "); - sb.append(CockroachDBVisitor.asString(gen.generateConstant(c.getType()))); + if (Randomly.getBoolean()) { + sb.append(CockroachDBVisitor.asString(gen.generateConstant(c.getType()))); + } else { + sb.append("excluded."); + sb.append(c.getName()); + } } errors.add("UPSERT or INSERT...ON CONFLICT command cannot affect row a second time"); } From a2757eebc61aee4bf6d96d841ba8f80470978d7b Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Tue, 11 Mar 2025 23:34:18 +0200 Subject: [PATCH 1060/1284] remove unnecessary line --- src/sqlancer/postgres/PostgresProvider.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 32f74789c..8d76867c8 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -187,7 +187,6 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; - default: throw new AssertionError(a); } From 8604997fdfed123bf3dbf8afb662cfcfdb23b64e Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 12 Mar 2025 02:11:23 +0200 Subject: [PATCH 1061/1284] extract node type using BFS --- src/sqlancer/postgres/PostgresProvider.java | 50 +++---- .../gen/PostgresExplainGenerator.java | 2 +- .../qpg/postgres/TestPostgresQueryPlan.java | 127 +++++++++++++++++- 3 files changed, 153 insertions(+), 26 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 8d76867c8..78d404b47 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -7,10 +7,15 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedList; import java.util.List; +import java.util.Queue; import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.auto.service.AutoService; import sqlancer.AbstractAction; @@ -364,13 +369,6 @@ public String getQueryPlan(String selectStr, PostgresGlobalState globalState) th e.printStackTrace(); } } - - List invalidList = List.of("create", "notify", "discard", "listen", "unlisten", "reset", "set", - "delete", "alter", "analyze"); - if (invalidList.contains(selectStr.split("\\s+")[0].toLowerCase())) { - return ""; - } - SQLQueryAdapter q = new SQLQueryAdapter(PostgresExplainGenerator.explain(selectStr), null); try (SQLancerResultSet rs = q.executeAndGet(globalState)) { while (rs.next()) { @@ -404,27 +402,31 @@ protected boolean addRowsToAllTables(PostgresGlobalState globalState) throws Exc return true; } - private String formatQueryPlan(String queryPlan) { - StringBuilder outQueryPlanFormatted = new StringBuilder(); - boolean insideBrackets = false; + public String formatQueryPlan(String queryPlan) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = mapper.readTree(queryPlan).get(0).get("Plan"); + // Extract nodes using BFS algorithm + List nodeTypes = extractNodeTypesIterative(root); + return String.join(" ", nodeTypes); + } - for (char ch : queryPlan.toCharArray()) { - if (ch == '\n' || ch == ' ') { - continue; + // BFS algorithm for traversing the Json Query Plan + private static List extractNodeTypesIterative(JsonNode root) { + List result = new ArrayList<>(); + Queue queue = new LinkedList<>(); + queue.add(root); + while (!queue.isEmpty()) { + JsonNode node = queue.poll(); + if (node.has("Node Type")) { + result.add(node.get("Node Type").asText()); } - if (ch == '(') { - insideBrackets = true; - } else if (ch == ')') { - insideBrackets = false; - outQueryPlanFormatted.append(';'); - continue; - } - if (!insideBrackets) { - outQueryPlanFormatted.append(ch); + if (node.has("Plans") && node.get("Plans").isArray()) { + for (JsonNode plan : node.get("Plans")) { + queue.add(plan); + } } } - - return outQueryPlanFormatted.toString(); + return result; } } diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index d55d4b72c..d3039394b 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -8,7 +8,7 @@ private PostgresExplainGenerator() { public static String explain(String selectStr) throws Exception { StringBuilder sb = new StringBuilder(); - sb.append("EXPLAIN "); + sb.append("EXPLAIN (FORMAT JSON) "); sb.append(selectStr); return sb.toString(); } diff --git a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java index f6a71df8f..c883e87d8 100644 --- a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java +++ b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java @@ -34,7 +34,132 @@ void testPostgresQueryPlan() throws Exception { q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); q.execute(state); String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); - assertEquals("NestedLoopLeftJoin;->SeqScanont2;->Materialize;->SeqScanont1;Filter:;", queryPlan); + assertEquals("Nested Loop Seq Scan Materialize Seq Scan", queryPlan); + } + + @Test + void testFormatQueryPlan() throws Exception { + + PostgresProvider provider = new PostgresProvider(); + + String queryPlan = "[\n" + " {\n" + " \"Plan\": {\n" + " \"Node Type\": \"Aggregate\",\n" + + " \"Strategy\": \"Hashed\",\n" + " \"Partial Mode\": \"Simple\",\n" + + " \"Parallel Aware\": false,\n" + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 62998.82,\n" + " \"Total Cost\": 63009.32,\n" + + " \"Plan Rows\": 1050,\n" + " \"Plan Width\": 4,\n" + " \"Output\": [\"t1.c0\"],\n" + + " \"Group Key\": [\"t1.c0\"],\n" + " \"Planned Partitions\": 0,\n" + " \"Plans\": [\n" + + " {\n" + " \"Node Type\": \"Append\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + " \"Startup Cost\": 27150.40,\n" + + " \"Total Cost\": 62996.20,\n" + " \"Plan Rows\": 1050,\n" + + " \"Plan Width\": 4,\n" + " \"Subplans Removed\": 0,\n" + " \"Plans\": [\n" + + " {\n" + " \"Node Type\": \"Group\",\n" + + " \"Parent Relationship\": \"Member\",\n" + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + " \"Startup Cost\": 27150.40,\n" + + " \"Total Cost\": 62949.08,\n" + " \"Plan Rows\": 200,\n" + + " \"Plan Width\": 4,\n" + " \"Output\": [\"t1.c0\"],\n" + + " \"Group Key\": [\"t1.c0\"],\n" + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Gather Merge\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 27150.40,\n" + " \"Total Cost\": 62948.08,\n" + + " \"Plan Rows\": 400,\n" + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"],\n" + " \"Workers Planned\": 2,\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Group\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 26150.38,\n" + + " \"Total Cost\": 61901.89,\n" + " \"Plan Rows\": 200,\n" + + " \"Plan Width\": 4,\n" + " \"Output\": [\"t1.c0\"],\n" + + " \"Group Key\": [\"t1.c0\"],\n" + " \"Plans\": [\n" + + " {\n" + " \"Node Type\": \"Merge Join\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Join Type\": \"Inner\",\n" + + " \"Startup Cost\": 26150.38,\n" + + " \"Total Cost\": 56906.48,\n" + + " \"Plan Rows\": 1998164,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"],\n" + + " \"Inner Unique\": false,\n" + + " \"Merge Cond\": \"(t0.c0 = t1.c0)\",\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Sort\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 25970.60,\n" + + " \"Total Cost\": 26362.39,\n" + + " \"Plan Rows\": 156719,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t0.c0\"],\n" + + " \"Sort Key\": [\"t0.c0\"],\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Seq Scan\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": true,\n" + + " \"Async Capable\": false,\n" + + " \"Relation Name\": \"t0\",\n" + + " \"Schema\": \"public\",\n" + + " \"Alias\": \"t0\",\n" + + " \"Startup Cost\": 0.00,\n" + + " \"Total Cost\": 10301.95,\n" + + " \"Plan Rows\": 156719,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t0.c0\"],\n" + + " \"Filter\": \"(t0.c0 < 100)\"\n" + + " }\n" + " ]\n" + + " },\n" + " {\n" + + " \"Node Type\": \"Sort\",\n" + + " \"Parent Relationship\": \"Inner\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 179.78,\n" + + " \"Total Cost\": 186.16,\n" + + " \"Plan Rows\": 2550,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"],\n" + + " \"Sort Key\": [\"t1.c0\"],\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Seq Scan\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Relation Name\": \"t1\",\n" + + " \"Schema\": \"public\",\n" + + " \"Alias\": \"t1\",\n" + + " \"Startup Cost\": 0.00,\n" + + " \"Total Cost\": 35.50,\n" + + " \"Plan Rows\": 2550,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"]\n" + " }\n" + + " ]\n" + " }\n" + + " ]\n" + " }\n" + " ]\n" + + " }\n" + " ]\n" + " }\n" + " ]\n" + + " },\n" + " {\n" + " \"Node Type\": \"Bitmap Heap Scan\",\n" + + " \"Parent Relationship\": \"Member\",\n" + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + " \"Relation Name\": \"t2\",\n" + + " \"Schema\": \"public\",\n" + " \"Alias\": \"t2\",\n" + + " \"Startup Cost\": 10.74,\n" + " \"Total Cost\": 31.37,\n" + + " \"Plan Rows\": 850,\n" + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t2.c0\"],\n" + " \"Recheck Cond\": \"(t2.c0 < 10)\",\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Bitmap Index Scan\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + " \"Async Capable\": false,\n" + + " \"Index Name\": \"t2_pkey\",\n" + " \"Startup Cost\": 0.00,\n" + + " \"Total Cost\": 10.53,\n" + " \"Plan Rows\": 850,\n" + + " \"Plan Width\": 0,\n" + " \"Index Cond\": \"(t2.c0 < 10)\"\n" + + " }\n" + " ]\n" + " }\n" + " ]\n" + " }\n" + + " ]\n" + " },\n" + " \"Planning Time\": 1.954\n" + " }\n" + "]\n"; + + String formatedQueryPlan = provider.formatQueryPlan(queryPlan); + assertEquals( + "Aggregate Append Group Bitmap Heap Scan Gather Merge Bitmap Index Scan Group Merge Join Sort Sort Seq Scan Seq Scan", + formatedQueryPlan); } } From e68056a694a8ebc31734d8e23da14e7626158898 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Wed, 12 Mar 2025 16:24:45 -0400 Subject: [PATCH 1062/1284] Feat(postgres): Add Window Function Support For Query Generation Implement comprehensive window function support for PostgreSQL testing, including: - Window function expressions with partition by, order by, ow_number, rank, and frame clauses - Named window definitions via window clause - Integration with existing expression generation framework The implementation allows SQLancer to generate and test queries containing window functions, expanding test coverage for PostgreSQL's analytical capabilities. Window functions are now included in the expression generation options and can be mutated during testing like other query components. Key changes: - Add PostgresWindowFunction and supporting classes - Enhance PostgresSelect to handle window clauses - Update expression generator for window function creation - Modify visitor for proper SQL generation --- .../postgres/PostgresToStringVisitor.java | 33 ++++++ src/sqlancer/postgres/PostgresVisitor.java | 7 +- src/sqlancer/postgres/ast/PostgresSelect.java | 49 +++++++++ .../postgres/ast/PostgresWindowFunction.java | 104 ++++++++++++++++++ .../gen/PostgresExpressionGenerator.java | 102 ++++++++++++++++- .../gen/PostgresWindowFunctionGenerator.java | 96 ++++++++++++++++ 6 files changed, 388 insertions(+), 3 deletions(-) create mode 100644 src/sqlancer/postgres/ast/PostgresWindowFunction.java create mode 100644 src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 78475a0af..b746bc733 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -362,4 +362,37 @@ public void visit(PostgresLikeOperation op) { super.visit((BinaryOperation) op); } + @Override + public void visit(PostgresWindowFunction windowFunction) { + sb.append(windowFunction.getFunctionName()); + sb.append("("); + visit(windowFunction.getArguments()); + sb.append(") OVER ("); + + WindowSpecification spec = windowFunction.getWindowSpec(); + if (!spec.getPartitionBy().isEmpty()) { + sb.append("PARTITION BY "); + visit(spec.getPartitionBy()); + } + + if (!spec.getOrderBy().isEmpty()) { + if (!spec.getPartitionBy().isEmpty()) { + sb.append(" "); + } + sb.append("ORDER BY "); + visit(spec.getOrderBy()); + } + + if (spec.getFrame() != null) { + sb.append(" "); + WindowFrame frame = spec.getFrame(); + sb.append(frame.getType().getSQL()); + sb.append(" BETWEEN "); + visit(frame.getStartExpr()); + sb.append(" AND "); + visit(frame.getEndExpr()); + } + + sb.append(")"); + } } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index d9c82232c..5951b1893 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -72,6 +72,8 @@ public interface PostgresVisitor { void visit(PostgresLikeOperation op); + void visit(PostgresWindowFunction windowFunction); + default void visit(PostgresExpression expression) { if (expression instanceof PostgresConstant) { visit((PostgresConstant) expression); @@ -113,7 +115,10 @@ default void visit(PostgresExpression expression) { visit((PostgresColumnReference) expression); } else if (expression instanceof PostgresTableReference) { visit((PostgresTableReference) expression); - } else { + } else if (expression instanceof PostgresWindowFunction) { + visit((PostgresWindowFunction) expression); + } + else { throw new AssertionError(expression); } } diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index c864a56e8..4b7773214 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -37,6 +37,55 @@ public static ForClause getRandom() { } } + public static class WindowDefinition { + private final List partitionBy; + private final List orderBy; + private final WindowFrame frame; + + public WindowDefinition(List partitionBy, + List orderBy, + WindowFrame frame) { + this.partitionBy = partitionBy; + this.orderBy = orderBy; + this.frame = frame; + } + + public List getPartitionBy() { + return partitionBy; + } + + public List getOrderBy() { + return orderBy; + } + + public WindowFrame getFrame() { + return frame; + } + } + + // Getters setters for windowfunctions + public List getWindowFunctions() { + return windowFunctions; + } + + public void setWindowFunctions(List windowFunctions) { + this.windowFunctions = windowFunctions != null ? windowFunctions : new ArrayList<>(); + } + + // Add methods for window definitions + public void addWindowDefinition(String name, WindowDefinition definition) { + windowDefinitions.put(name, definition); + } + + public WindowDefinition getWindowDefinition(String name) { + return windowDefinitions.get(name); + } + + public Map getWindowDefinitions() { + return windowDefinitions; + } + + public static class PostgresFromTable implements PostgresExpression { private final PostgresTable t; private final boolean only; diff --git a/src/sqlancer/postgres/ast/PostgresWindowFunction.java b/src/sqlancer/postgres/ast/PostgresWindowFunction.java new file mode 100644 index 000000000..dd92abbae --- /dev/null +++ b/src/sqlancer/postgres/ast/PostgresWindowFunction.java @@ -0,0 +1,104 @@ +package sqlancer.postgres.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.postgres.PostgresSchema.PostgresDataType; + +public class PostgresWindowFunction implements PostgresExpression { + + private final String functionName; + private final List arguments; + private final WindowSpecification windowSpec; + private final PostgresDataType returnType; + + public static class WindowSpecification { + private final List partitionBy; + private final List orderBy; + private final WindowFrame frame; + + public WindowSpecification(List partitionBy, + List orderBy, + WindowFrame frame) { + this.partitionBy = partitionBy; + this.orderBy = orderBy; + this.frame = frame; + } + + public List getPartitionBy() { + return partitionBy; + } + + public List getOrderBy() { + return orderBy; + } + + public WindowFrame getFrame() { + return frame; + } + } + + public static class WindowFrame { + public enum FrameType { + ROWS("ROWS"), + RANGE("RANGE"); + + private final String sql; + + FrameType(String sql) { + this.sql = sql; + } + + public String getSQL() { + return sql; + } + } + + private final FrameType type; + private final PostgresExpression startExpr; + private final PostgresExpression endExpr; + + public WindowFrame(FrameType type, PostgresExpression startExpr, PostgresExpression endExpr) { + this.type = type; + this.startExpr = startExpr; + this.endExpr = endExpr; + } + + public FrameType getType() { + return type; + } + + public PostgresExpression getStartExpr() { + return startExpr; + } + + public PostgresExpression getEndExpr() { + return endExpr; + } + } + + public PostgresWindowFunction(String functionName, List arguments, + WindowSpecification windowSpec, PostgresDataType returnType) { + this.functionName = functionName; + this.arguments = arguments; + this.windowSpec = windowSpec; + this.returnType = returnType; + } + + public String getFunctionName() { + return functionName; + } + + public List getArguments() { + return arguments; + } + + public WindowSpecification getWindowSpec() { + return windowSpec; + } + + @Override + public PostgresDataType getExpressionType() { + return returnType; + } +} \ No newline at end of file diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index c6189acc5..29b02e484 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -422,6 +422,53 @@ private PostgresExpression generateTextExpression(int depth) { throw new AssertionError(); } } + + private PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { + // Generate window function arguments + List arguments = new ArrayList<>(); + if (Randomly.getBoolean()) { + arguments.add(generateExpression(depth + 1)); + } + + // Generate PARTITION BY expressions + List partitionBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + partitionBy.add(generateExpression(depth + 1)); + } + } + + // Generate ORDER BY expressions + List orderBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + PostgresExpression expr = generateExpression(depth + 1); + orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + } + } + + // Generate window frame + WindowFrame frame = null; + if (Randomly.getBoolean()) { + WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); + PostgresExpression startExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + PostgresExpression endExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + frame = new WindowFrame(frameType, startExpr, endExpr); + } + + WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + + // Select a window function + String functionName = Randomly.fromList(Arrays.asList( + "row_number", "rank", "dense_rank", "percent_rank", + "cume_dist", "ntile", "lag", "lead", "first_value", + "last_value", "nth_value" + )); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } private PostgresExpression generateConcat(int depth) { PostgresExpression left = generateExpression(depth + 1, PostgresDataType.TEXT); @@ -447,7 +494,7 @@ private PostgresExpression generateBitExpression(int depth) { } private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION } private PostgresExpression generateIntExpression(int depth) { @@ -667,7 +714,21 @@ public PostgresExpression generateBooleanExpression() { @Override public PostgresSelect generateSelect() { - return new PostgresSelect(); + PostgresSelect select = new PostgresSelect(); + + // Add window functions to fetch columns if appropriate + if (Randomly.getBoolean()) { + List windowFunctions = new ArrayList<>(); + int numWindowFunctions = Randomly.smallNumber(); + for (int i = 0; i < numWindowFunctions; i++) { + windowFunctions.add(generateWindowFunction(0, Randomly.fromList(Arrays.asList( + PostgresDataType.INT, PostgresDataType.FLOAT + )))); + } + select.setWindowFunctions(windowFunctions); + } + + return select; } @Override @@ -768,6 +829,43 @@ public boolean mutate(PostgresSelect select) { return Randomly.fromList(mutators).apply(select); } + + @Override + public boolean mutate(PostgresSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateJoin); + mutators.add(this::mutateWhere); + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + mutators.add(this::mutateWindowFunction); // Add window function mutation + if (!PostgresBugs.bug18643) { + mutators.add(this::mutateAnd); + mutators.add(this::mutateOr); + } + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + private boolean mutateWindowFunction(PostgresSelect select) { + List windowFunctions = select.getWindowFunctions(); + if (windowFunctions == null || windowFunctions.isEmpty()) { + // Add a new window function + windowFunctions = new ArrayList<>(); + windowFunctions.add(generateWindowFunction(0, PostgresDataType.INT)); + select.setWindowFunctions(windowFunctions); + return false; + } else { + // Remove a random window function + windowFunctions.remove(Randomly.fromList(windowFunctions)); + if (windowFunctions.isEmpty()) { + select.setWindowFunctions(null); + } + return true; + } + } + boolean mutateJoin(PostgresSelect select) { if (select.getJoinList().isEmpty()) { return false; diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java new file mode 100644 index 000000000..7537db9cb --- /dev/null +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -0,0 +1,96 @@ +package sqlancer.postgres.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresOrderByTerm; +import sqlancer.postgres.ast.PostgresWindowFunction; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; + +public class PostgresWindowFunctionGenerator { + + private static final List WINDOW_FUNCTIONS = Arrays.asList( + "row_number", "rank", "dense_rank", "percent_rank", + "cume_dist", "ntile", "lag", "lead", "first_value", + "last_value", "nth_value" + ); + + public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState globalState, + List availableExpr) { + + String functionName = Randomly.fromList(WINDOW_FUNCTIONS); + List arguments = new ArrayList<>(); + + // Generate function arguments based on function name + switch (functionName) { + case "ntile": + arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); + break; + case "lag": + case "lead": + case "nth_value": + arguments.add(Randomly.fromList(availableExpr)); + if (Randomly.getBoolean()) { + arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); + } + break; + case "first_value": + case "last_value": + arguments.add(Randomly.fromList(availableExpr)); + break; + default: + // No arguments needed for other functions + break; + } + + // Generate partition by clause + List partitionBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + partitionBy.add(Randomly.fromList(availableExpr)); + } + } + + // Generate order by clause + List orderBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + orderBy.add(new PostgresOrderByTerm(Randomly.fromList(availableExpr), + Randomly.getBoolean())); + } + } + + // Generate window frame + WindowFrame frame = null; + if (Randomly.getBoolean()) { + WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); + PostgresExpression startExpr = PostgresExpressionGenerator.generateConstant(globalState.getRandomly()); + PostgresExpression endExpr = PostgresExpressionGenerator.generateConstant(globalState.getRandomly()); + frame = new WindowFrame(frameType, startExpr, endExpr); + } + + WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + + // Determine return type based on function + PostgresDataType returnType; + switch (functionName) { + case "percent_rank": + case "cume_dist": + returnType = PostgresDataType.FLOAT; + break; + default: + returnType = PostgresDataType.INT; + break; + } + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } +} \ No newline at end of file From 0c46e61ee9d157a775c16bcc044f64c3a26d3b77 Mon Sep 17 00:00:00 2001 From: Victor Ma Date: Wed, 12 Mar 2025 17:57:11 -0400 Subject: [PATCH 1063/1284] Bump duckdb version to 1.2.0 --- pom.xml | 2 +- src/sqlancer/duckdb/DuckDBErrors.java | 2 +- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 2c616138a..836d84fa5 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 1.1.3 + 1.2.0 com.facebook.presto diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 5141ca0b1..f5cf3eb94 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -115,7 +115,7 @@ public static List getInsertErrors() { errors.addAll(getFunctionErrors()); errors.add("NOT NULL constraint failed"); - errors.add("PRIMARY KEY or UNIQUE constraint violated"); + errors.add("PRIMARY KEY or UNIQUE constraint violation"); errors.add("Duplicate key"); errors.add("can't be cast because the value is out of range for the destination type"); errors.add("Could not convert string"); diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 80bbfd61c..597fda19f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -42,7 +42,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(")"); errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { - errors.add("Cannot create an index on the rowid!"); + errors.add("cannot create an index on the rowid"); } return new SQLQueryAdapter(sb.toString(), errors, true); } From ecc9a4a388066786adafb4e4c94b879ad0f00dd0 Mon Sep 17 00:00:00 2001 From: Milan P Samuel Date: Thu, 13 Mar 2025 15:04:12 +0000 Subject: [PATCH 1064/1284] updated version of mariadb from 11.3.2 to 11.7.2 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91188369b..373ea3896 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -322,7 +322,7 @@ jobs: runs-on: ubuntu-latest services: mysql: - image: mariadb:11.3.2 + image: mariadb:11.7.2 env: MYSQL_ROOT_PASSWORD: root ports: From d399dfcb7a8ad4c02ba320c33ca12e7e68fdefe9 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Thu, 13 Mar 2025 15:41:02 -0400 Subject: [PATCH 1065/1284] Refactor Window Function Generation For Better Readability Adjust probabilities for a lower query generation in PostgreSQL window functions --- .../gen/PostgresExpressionGenerator.java | 103 +++++++++++------- .../gen/PostgresWindowFunctionGenerator.java | 83 +++++++++----- 2 files changed, 118 insertions(+), 68 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 29b02e484..9cddda042 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -424,48 +424,65 @@ private PostgresExpression generateTextExpression(int depth) { } private PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { - // Generate window function arguments - List arguments = new ArrayList<>(); - if (Randomly.getBoolean()) { - arguments.add(generateExpression(depth + 1)); - } + List arguments = generateWindowFunctionArguments(depth); + List partitionBy = generatePartitionByExpressions(depth); + List orderBy = generateOrderByExpressions(depth); + WindowFrame frame = generateWindowFrame(); + + WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + String functionName = selectWindowFunctionName(); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); +} - // Generate PARTITION BY expressions - List partitionBy = new ArrayList<>(); - if (Randomly.getBoolean()) { - int count = Randomly.smallNumber(); - for (int i = 0; i < count; i++) { - partitionBy.add(generateExpression(depth + 1)); + private List generateWindowFunctionArguments(int depth) { + List arguments = new ArrayList<>(); + if (Randomly.getBoolean()) { + arguments.add(generateExpression(depth + 1)); } + return arguments; } - // Generate ORDER BY expressions - List orderBy = new ArrayList<>(); - if (Randomly.getBoolean()) { - int count = Randomly.smallNumber(); - for (int i = 0; i < count; i++) { - PostgresExpression expr = generateExpression(depth + 1); - orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + private List generatePartitionByExpressions(int depth) { + List partitionBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + partitionBy.add(generateExpression(depth + 1)); + } } + return partitionBy; } - // Generate window frame - WindowFrame frame = null; - if (Randomly.getBoolean()) { - WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); - PostgresExpression startExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); - PostgresExpression endExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); - frame = new WindowFrame(frameType, startExpr, endExpr); + private List generateOrderByExpressions(int depth) { + List orderBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + PostgresExpression expr = generateExpression(depth + 1); + orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + } + } + return orderBy; } - WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + private WindowFrame generateWindowFrame() { + if (Randomly.getBoolean()) { + WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); + PostgresExpression startExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + PostgresExpression endExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + return new WindowFrame(frameType, startExpr, endExpr); + } + return null; + } - // Select a window function - String functionName = Randomly.fromList(Arrays.asList( - "row_number", "rank", "dense_rank", "percent_rank", - "cume_dist", "ntile", "lag", "lead", "first_value", - "last_value", "nth_value" - )); + private String selectWindowFunctionName() { + return Randomly.fromList(Arrays.asList( + "row_number", "rank", "dense_rank", "percent_rank", + "cume_dist", "ntile", "lag", "lead", "first_value", + "last_value", "nth_value" + )); + } return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); } @@ -716,21 +733,25 @@ public PostgresExpression generateBooleanExpression() { public PostgresSelect generateSelect() { PostgresSelect select = new PostgresSelect(); - // Add window functions to fetch columns if appropriate - if (Randomly.getBoolean()) { - List windowFunctions = new ArrayList<>(); - int numWindowFunctions = Randomly.smallNumber(); - for (int i = 0; i < numWindowFunctions; i++) { - windowFunctions.add(generateWindowFunction(0, Randomly.fromList(Arrays.asList( - PostgresDataType.INT, PostgresDataType.FLOAT - )))); - } + if (Randomly.getBooleanWithRatherLowProbability()) { + List windowFunctions = generateWindowFunctions(); select.setWindowFunctions(windowFunctions); } return select; } + private List generateWindowFunctions() { + List windowFunctions = new ArrayList<>(); + int numWindowFunctions = Randomly.smallNumber(); + for (int i = 0; i < numWindowFunctions; i++) { + windowFunctions.add(generateWindowFunction(0, Randomly.fromList(Arrays.asList( + PostgresDataType.INT, PostgresDataType.FLOAT + )))); + } + return windowFunctions; + } + @Override public List getRandomJoinClauses() { List joinStatements = new ArrayList<>(); diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java index 7537db9cb..ef9b1ca30 100644 --- a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -24,10 +24,22 @@ public class PostgresWindowFunctionGenerator { public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState globalState, List availableExpr) { - String functionName = Randomly.fromList(WINDOW_FUNCTIONS); + String functionName = selectRandomWindowFunction(); + List arguments = generateFunctionArguments(functionName, globalState, availableExpr); + WindowSpecification windowSpec = generateWindowSpecification(globalState, availableExpr); + PostgresDataType returnType = determineReturnType(functionName); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } + + private static String selectRandomWindowFunction() { + return Randomly.fromList(WINDOW_FUNCTIONS); + } + + private static List generateFunctionArguments(String functionName, + PostgresGlobalState globalState, List availableExpr) { List arguments = new ArrayList<>(); - // Generate function arguments based on function name switch (functionName) { case "ntile": arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); @@ -44,53 +56,70 @@ public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState case "last_value": arguments.add(Randomly.fromList(availableExpr)); break; - default: - // No arguments needed for other functions - break; } + + return arguments; + } - // Generate partition by clause + private static WindowSpecification generateWindowSpecification(PostgresGlobalState globalState, + List availableExpr) { + List partitionBy = generatePartitionByClause(availableExpr); + List orderBy = generateOrderByClause(availableExpr); + WindowFrame frame = generateWindowFrame(globalState); + + return new WindowSpecification(partitionBy, orderBy, frame); + } + + private static List generatePartitionByClause(List availableExpr) { List partitionBy = new ArrayList<>(); - if (Randomly.getBoolean()) { + if (Randomly.getBooleanWithRatherLowProbability()) { int count = Randomly.smallNumber(); for (int i = 0; i < count; i++) { partitionBy.add(Randomly.fromList(availableExpr)); } } + return partitionBy; + } - // Generate order by clause + private static List generateOrderByClause(List availableExpr) { List orderBy = new ArrayList<>(); - if (Randomly.getBoolean()) { + if (Randomly.getBooleanWithRatherLowProbability()) { int count = Randomly.smallNumber(); for (int i = 0; i < count; i++) { - orderBy.add(new PostgresOrderByTerm(Randomly.fromList(availableExpr), - Randomly.getBoolean())); + orderBy.add(new PostgresOrderByTerm( + Randomly.fromList(availableExpr), + Randomly.getBoolean() + )); } } + return orderBy; + } - // Generate window frame - WindowFrame frame = null; - if (Randomly.getBoolean()) { + private static WindowFrame generateWindowFrame(PostgresGlobalState globalState) { + if (Randomly.getBooleanWithRatherLowProbability()) { WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); - PostgresExpression startExpr = PostgresExpressionGenerator.generateConstant(globalState.getRandomly()); - PostgresExpression endExpr = PostgresExpressionGenerator.generateConstant(globalState.getRandomly()); - frame = new WindowFrame(frameType, startExpr, endExpr); + PostgresExpression startExpr = generateFrameBound(globalState); + PostgresExpression endExpr = generateFrameBound(globalState); + return new WindowFrame(frameType, startExpr, endExpr); } + return null; + } - WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); - - // Determine return type based on function - PostgresDataType returnType; + private static PostgresExpression generateFrameBound(PostgresGlobalState globalState) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return PostgresConstant.createIntConstant(0); // CURRENT ROW + } else { + return PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT); + } + } + + private static PostgresDataType determineReturnType(String functionName) { switch (functionName) { case "percent_rank": case "cume_dist": - returnType = PostgresDataType.FLOAT; - break; + return PostgresDataType.FLOAT; default: - returnType = PostgresDataType.INT; - break; + return PostgresDataType.INT; } - - return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); } } \ No newline at end of file From d70a9a67cb4bfb8e4c5dbf42d8713308e5734dce Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 15 Mar 2025 10:49:19 -0400 Subject: [PATCH 1066/1284] Fix compilation errors and warnings Add more support for PostgresOrderByTerm --- .../PostgresExpectedValueVisitor.java | 22 +++++--- .../postgres/PostgresToStringVisitor.java | 52 +++++++++++++++---- src/sqlancer/postgres/PostgresVisitor.java | 1 + .../postgres/ast/PostgresOrderByTerm.java | 34 +++++++++--- src/sqlancer/postgres/ast/PostgresSelect.java | 10 +++- .../postgres/ast/PostgresWindowFunction.java | 52 +++++++++---------- .../gen/PostgresExpressionGenerator.java | 40 +++++--------- .../gen/PostgresWindowFunctionGenerator.java | 34 ++++++++---- 8 files changed, 154 insertions(+), 91 deletions(-) diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index efc1b7442..016576af3 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -22,6 +22,8 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.IgnoreMeException; +import sqlancer.postgres.ast.PostgresWindowFunction; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -42,13 +44,13 @@ private void print(PostgresExpression expr) { // @Override // public void visit(PostgresExpression expr) { - // nrTabs++; - // try { - // super.visit(expr); - // } catch (IgnoreMeException e) { - // - // } - // nrTabs--; + // nrTabs++; + // try { + // super.visit(expr); + // } catch (IgnoreMeException e) { + + // } + // nrTabs--; // } @Override @@ -96,6 +98,12 @@ public void visit(PostgresOrderByTerm op) { } + + @Override + public void visit(PostgresWindowFunction windowFunction) { + throw new IgnoreMeException(); + } + @Override public void visit(PostgresFunction f) { print(f); diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index b746bc733..e3cf46c7e 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -1,6 +1,7 @@ package sqlancer.postgres; import java.util.Optional; +import java.util.List; import sqlancer.Randomly; import sqlancer.common.visitor.BinaryOperation; @@ -30,6 +31,9 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; public final class PostgresToStringVisitor extends ToStringVisitor implements PostgresVisitor { @@ -185,13 +189,19 @@ public void visit(PostgresSelect s) { } } + // @Override + // public void visit(PostgresOrderByTerm op) { + // visit(op.getExpr()); + // sb.append(" "); + // sb.append(op.getOrder()); + // } + @Override - public void visit(PostgresOrderByTerm op) { - visit(op.getExpr()); - sb.append(" "); - sb.append(op.getOrder()); + public void visit(PostgresOrderByTerm term) { + visit(term.getExpr()); + sb.append(term.isAscending() ? " ASC" : " DESC"); } - + @Override public void visit(PostgresFunction f) { sb.append(f.getFunctionName()); @@ -366,13 +376,14 @@ public void visit(PostgresLikeOperation op) { public void visit(PostgresWindowFunction windowFunction) { sb.append(windowFunction.getFunctionName()); sb.append("("); - visit(windowFunction.getArguments()); + // Fix: Use visitList instead of visit for a list of expressions + visitList(windowFunction.getArguments()); sb.append(") OVER ("); WindowSpecification spec = windowFunction.getWindowSpec(); if (!spec.getPartitionBy().isEmpty()) { sb.append("PARTITION BY "); - visit(spec.getPartitionBy()); + visitList(spec.getPartitionBy()); } if (!spec.getOrderBy().isEmpty()) { @@ -380,9 +391,10 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(" "); } sb.append("ORDER BY "); - visit(spec.getOrderBy()); + // Fix: Create a method to handle order by terms specifically + visitOrderByList(spec.getOrderBy()); } - + if (spec.getFrame() != null) { sb.append(" "); WindowFrame frame = spec.getFrame(); @@ -395,4 +407,26 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(")"); } + + // this method handles lists of expressions + private void visitList(List expressions) { + int i = 0; + for (PostgresExpression expr : expressions) { + if (i++ != 0) { + sb.append(", "); + } + visit(expr); + } + } + + // this method handles lists of order by terms + private void visitOrderByList(List orderBy) { + int i = 0; + for (PostgresOrderByTerm term : orderBy) { + if (i++ != 0) { + sb.append(", "); + } + visit(term); + } + } } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 5951b1893..4e563824c 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -2,6 +2,7 @@ import java.util.List; +import sqlancer.postgres.ast.PostgresWindowFunction; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; diff --git a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java index 20f93536e..3c4f39fb7 100644 --- a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java +++ b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java @@ -4,9 +4,9 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresOrderByTerm implements PostgresExpression { - - private final PostgresOrder order; + private final PostgresExpression expr; + private final PostgresOrder order; public enum PostgresOrder { ASC, DESC; @@ -17,18 +17,34 @@ public static PostgresOrder getRandomOrder() { } public PostgresOrderByTerm(PostgresExpression expr, PostgresOrder order) { + if (expr == null) { + throw new IllegalArgumentException("Expression cannot be null"); + } this.expr = expr; this.order = order; } - - public PostgresOrder getOrder() { - return order; + + // Constructor for window functions, might be removed in the future to have only one constructor + public PostgresOrderByTerm(PostgresExpression expr, boolean ascending) { + if (expr == null) { + throw new IllegalArgumentException("Expression cannot be null"); + } + this.expr = expr; + this.order = ascending ? PostgresOrder.ASC : PostgresOrder.DESC; } public PostgresExpression getExpr() { return expr; } + public PostgresOrder getOrder() { + return order; + } + + public boolean isAscending() { + return order == PostgresOrder.ASC; + } + @Override public PostgresConstant getExpectedValue() { throw new AssertionError(this); @@ -38,5 +54,9 @@ public PostgresConstant getExpectedValue() { public PostgresDataType getExpressionType() { return null; } - -} + + @Override + public String toString() { + return String.format("%s %s", expr, order); + } +} \ No newline at end of file diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index 4b7773214..a51c85e72 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -2,6 +2,9 @@ import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.ArrayList; +import java.util.HashMap; import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; @@ -10,6 +13,7 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; public class PostgresSelect extends SelectBase implements PostgresExpression, Select { @@ -18,6 +22,8 @@ public class PostgresSelect extends SelectBase private List joinClauses = Collections.emptyList(); private PostgresExpression distinctOnClause; private ForClause forClause; + private List windowFunctions = new ArrayList<>(); + private Map windowDefinitions = new HashMap<>(); public enum ForClause { UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); @@ -69,7 +75,7 @@ public List getWindowFunctions() { } public void setWindowFunctions(List windowFunctions) { - this.windowFunctions = windowFunctions != null ? windowFunctions : new ArrayList<>(); + this.windowFunctions = windowFunctions; } // Add methods for window definitions @@ -191,4 +197,4 @@ public ForClause getForClause() { public String asString() { return PostgresVisitor.asString(this); } -} +} \ No newline at end of file diff --git a/src/sqlancer/postgres/ast/PostgresWindowFunction.java b/src/sqlancer/postgres/ast/PostgresWindowFunction.java index dd92abbae..71ac6853a 100644 --- a/src/sqlancer/postgres/ast/PostgresWindowFunction.java +++ b/src/sqlancer/postgres/ast/PostgresWindowFunction.java @@ -1,8 +1,6 @@ package sqlancer.postgres.ast; import java.util.List; - -import sqlancer.Randomly; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresWindowFunction implements PostgresExpression { @@ -12,6 +10,31 @@ public class PostgresWindowFunction implements PostgresExpression { private final WindowSpecification windowSpec; private final PostgresDataType returnType; + public PostgresWindowFunction(String functionName, List arguments, + WindowSpecification windowSpec, PostgresDataType returnType) { + this.functionName = functionName; + this.arguments = arguments; + this.windowSpec = windowSpec; + this.returnType = returnType; + } + + public String getFunctionName() { + return functionName; + } + + public List getArguments() { + return arguments; + } + + public WindowSpecification getWindowSpec() { + return windowSpec; + } + + @Override + public PostgresDataType getExpressionType() { + return returnType; + } + public static class WindowSpecification { private final List partitionBy; private final List orderBy; @@ -76,29 +99,4 @@ public PostgresExpression getEndExpr() { return endExpr; } } - - public PostgresWindowFunction(String functionName, List arguments, - WindowSpecification windowSpec, PostgresDataType returnType) { - this.functionName = functionName; - this.arguments = arguments; - this.windowSpec = windowSpec; - this.returnType = returnType; - } - - public String getFunctionName() { - return functionName; - } - - public List getArguments() { - return arguments; - } - - public WindowSpecification getWindowSpec() { - return windowSpec; - } - - @Override - public PostgresDataType getExpressionType() { - return returnType; - } } \ No newline at end of file diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 9cddda042..83f9096d0 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -52,7 +52,7 @@ import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; -import sqlancer.postgres.ast.PostgresOrderByTerm.PostgresOrder; +// import sqlancer.postgres.ast.PostgresOrderByTerm.PostgresOrder; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression.POSIXRegex; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -67,6 +67,10 @@ import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; + public class PostgresExpressionGenerator implements ExpressionGenerator, NoRECGenerator, @@ -92,7 +96,7 @@ public class PostgresExpressionGenerator implements ExpressionGenerator functionsAndTypes; private final List allowedFunctionTypes; - + public PostgresExpressionGenerator(PostgresGlobalState globalState) { this.r = globalState.getRandomly(); this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); @@ -119,8 +123,8 @@ public PostgresExpression generateExpression(int depth) { public List generateOrderBys() { List orderBys = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber(); i++) { - orderBys.add(new PostgresOrderByTerm(PostgresColumnValue.create(Randomly.fromList(columns), null), - PostgresOrder.getRandomOrder())); + PostgresExpression expr = PostgresColumnValue.create(Randomly.fromList(columns), null); + orderBys.add(expr); } return orderBys; } @@ -442,7 +446,6 @@ private List generateWindowFunctionArguments(int depth) { } return arguments; } - private List generatePartitionByExpressions(int depth) { List partitionBy = new ArrayList<>(); if (Randomly.getBoolean()) { @@ -460,6 +463,7 @@ private List generateOrderByExpressions(int depth) { int count = Randomly.smallNumber(); for (int i = 0; i < count; i++) { PostgresExpression expr = generateExpression(depth + 1); + // Call the second constructor in PostgresOrderByTerm, might be removed in the future to have only one constructor orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); } } @@ -484,8 +488,6 @@ private String selectWindowFunctionName() { )); } - return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); - } private PostgresExpression generateConcat(int depth) { PostgresExpression left = generateExpression(depth + 1, PostgresDataType.TEXT); @@ -511,7 +513,7 @@ private PostgresExpression generateBitExpression(int depth) { } private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION } private PostgresExpression generateIntExpression(int depth) { @@ -529,11 +531,12 @@ private PostgresExpression generateIntExpression(int depth) { case BINARY_ARITHMETIC_EXPRESSION: return new PostgresBinaryArithmeticOperation(generateExpression(depth + 1, PostgresDataType.INT), generateExpression(depth + 1, PostgresDataType.INT), PostgresBinaryOperator.getRandom()); + case WINDOW_FUNCTION: + return generateWindowFunction(depth + 1, PostgresDataType.INT); default: throw new AssertionError(); } } - private PostgresExpression createColumnOfType(PostgresDataType type) { List columns = filterColumns(type); PostgresColumn fromList = Randomly.fromList(columns); @@ -831,25 +834,6 @@ public String generateUnoptimizedQueryString(PostgresSelect select, PostgresExpr public String generateExplainQuery(PostgresSelect select) { return "EXPLAIN " + select.asString(); } - - @Override - public boolean mutate(PostgresSelect select) { - List> mutators = new ArrayList<>(); - - mutators.add(this::mutateJoin); - mutators.add(this::mutateWhere); - mutators.add(this::mutateGroupBy); - mutators.add(this::mutateHaving); - if (!PostgresBugs.bug18643) { - mutators.add(this::mutateAnd); - mutators.add(this::mutateOr); - } - // mutators.add(this::mutateLimit); - mutators.add(this::mutateDistinct); - - return Randomly.fromList(mutators).apply(select); - } - @Override public boolean mutate(PostgresSelect select) { diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java index ef9b1ca30..1b9462d2b 100644 --- a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -6,6 +6,7 @@ import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresOrderByTerm; @@ -14,7 +15,8 @@ import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; public class PostgresWindowFunctionGenerator { - + + private static final List WINDOW_FUNCTIONS = Arrays.asList( "row_number", "rank", "dense_rank", "percent_rank", "cume_dist", "ntile", "lag", "lead", "first_value", @@ -23,7 +25,7 @@ public class PostgresWindowFunctionGenerator { public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState globalState, List availableExpr) { - + String functionName = selectRandomWindowFunction(); List arguments = generateFunctionArguments(functionName, globalState, availableExpr); WindowSpecification windowSpec = generateWindowSpecification(globalState, availableExpr); @@ -39,17 +41,18 @@ private static String selectRandomWindowFunction() { private static List generateFunctionArguments(String functionName, PostgresGlobalState globalState, List availableExpr) { List arguments = new ArrayList<>(); - + switch (functionName) { case "ntile": - arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); + + arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); break; case "lag": case "lead": case "nth_value": arguments.add(Randomly.fromList(availableExpr)); if (Randomly.getBoolean()) { - arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); + arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); } break; case "first_value": @@ -57,7 +60,7 @@ private static List generateFunctionArguments(String functio arguments.add(Randomly.fromList(availableExpr)); break; } - + return arguments; } @@ -66,7 +69,7 @@ private static WindowSpecification generateWindowSpecification(PostgresGlobalSta List partitionBy = generatePartitionByClause(availableExpr); List orderBy = generateOrderByClause(availableExpr); WindowFrame frame = generateWindowFrame(globalState); - + return new WindowSpecification(partitionBy, orderBy, frame); } @@ -94,7 +97,6 @@ private static List generateOrderByClause(List Date: Sun, 16 Mar 2025 01:07:28 +0530 Subject: [PATCH 1067/1284] Added IllegalArgumentException to handle the possible exception. --- src/sqlancer/common/query/ExpectedErrors.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index f71d185e1..38d5c9757 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -43,11 +43,17 @@ public ExpectedErrors addRegex(Pattern errorPattern) { } public ExpectedErrors addRegexString(String errorPattern) { + if (errorPattern == null) { + throw new IllegalArgumentException(); + } regexes.add(Pattern.compile(errorPattern)); return this; } public ExpectedErrors addAll(Collection list) { + if (list == null) { + throw new IllegalArgumentException(); + } errors.addAll(list); return this; } From 9bf6f1b7b0fb32d4237f00ed560843274eb9f5bf Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 15 Mar 2025 17:33:35 -0400 Subject: [PATCH 1068/1284] Fix style checks --- .../PostgresExpectedValueVisitor.java | 17 +++-- .../postgres/PostgresToStringVisitor.java | 18 ++--- src/sqlancer/postgres/PostgresVisitor.java | 5 +- .../postgres/ast/PostgresOrderByTerm.java | 10 +-- src/sqlancer/postgres/ast/PostgresSelect.java | 14 ++-- .../postgres/ast/PostgresWindowFunction.java | 17 +++-- .../gen/PostgresExpressionGenerator.java | 51 +++++++------ .../gen/PostgresWindowFunctionGenerator.java | 71 ++++++++++--------- 8 files changed, 98 insertions(+), 105 deletions(-) diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index 016576af3..8985caf0b 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -1,5 +1,6 @@ package sqlancer.postgres; +import sqlancer.IgnoreMeException; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresBetweenOperation; import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; @@ -22,7 +23,6 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; -import sqlancer.IgnoreMeException; import sqlancer.postgres.ast.PostgresWindowFunction; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -44,13 +44,13 @@ private void print(PostgresExpression expr) { // @Override // public void visit(PostgresExpression expr) { - // nrTabs++; - // try { - // super.visit(expr); - // } catch (IgnoreMeException e) { - - // } - // nrTabs--; + // nrTabs++; + // try { + // super.visit(expr); + // } catch (IgnoreMeException e) { + + // } + // nrTabs--; // } @Override @@ -98,7 +98,6 @@ public void visit(PostgresOrderByTerm op) { } - @Override public void visit(PostgresWindowFunction windowFunction) { throw new IgnoreMeException(); diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index e3cf46c7e..5df87aa5a 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -1,7 +1,7 @@ package sqlancer.postgres; -import java.util.Optional; import java.util.List; +import java.util.Optional; import sqlancer.Randomly; import sqlancer.common.visitor.BinaryOperation; @@ -191,9 +191,9 @@ public void visit(PostgresSelect s) { // @Override // public void visit(PostgresOrderByTerm op) { - // visit(op.getExpr()); - // sb.append(" "); - // sb.append(op.getOrder()); + // visit(op.getExpr()); + // sb.append(" "); + // sb.append(op.getOrder()); // } @Override @@ -201,7 +201,7 @@ public void visit(PostgresOrderByTerm term) { visit(term.getExpr()); sb.append(term.isAscending() ? " ASC" : " DESC"); } - + @Override public void visit(PostgresFunction f) { sb.append(f.getFunctionName()); @@ -379,13 +379,13 @@ public void visit(PostgresWindowFunction windowFunction) { // Fix: Use visitList instead of visit for a list of expressions visitList(windowFunction.getArguments()); sb.append(") OVER ("); - + WindowSpecification spec = windowFunction.getWindowSpec(); if (!spec.getPartitionBy().isEmpty()) { sb.append("PARTITION BY "); visitList(spec.getPartitionBy()); } - + if (!spec.getOrderBy().isEmpty()) { if (!spec.getPartitionBy().isEmpty()) { sb.append(" "); @@ -394,7 +394,7 @@ public void visit(PostgresWindowFunction windowFunction) { // Fix: Create a method to handle order by terms specifically visitOrderByList(spec.getOrderBy()); } - + if (spec.getFrame() != null) { sb.append(" "); WindowFrame frame = spec.getFrame(); @@ -404,7 +404,7 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(" AND "); visit(frame.getEndExpr()); } - + sb.append(")"); } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 4e563824c..d5bf71e81 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -2,7 +2,6 @@ import java.util.List; -import sqlancer.postgres.ast.PostgresWindowFunction; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; @@ -27,6 +26,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; import sqlancer.postgres.gen.PostgresExpressionGenerator; public interface PostgresVisitor { @@ -118,8 +118,7 @@ default void visit(PostgresExpression expression) { visit((PostgresTableReference) expression); } else if (expression instanceof PostgresWindowFunction) { visit((PostgresWindowFunction) expression); - } - else { + } else { throw new AssertionError(expression); } } diff --git a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java index 3c4f39fb7..96debf741 100644 --- a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java +++ b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java @@ -4,7 +4,7 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresOrderByTerm implements PostgresExpression { - + private final PostgresExpression expr; private final PostgresOrder order; @@ -23,7 +23,7 @@ public PostgresOrderByTerm(PostgresExpression expr, PostgresOrder order) { this.expr = expr; this.order = order; } - + // Constructor for window functions, might be removed in the future to have only one constructor public PostgresOrderByTerm(PostgresExpression expr, boolean ascending) { if (expr == null) { @@ -40,7 +40,7 @@ public PostgresExpression getExpr() { public PostgresOrder getOrder() { return order; } - + public boolean isAscending() { return order == PostgresOrder.ASC; } @@ -54,9 +54,9 @@ public PostgresConstant getExpectedValue() { public PostgresDataType getExpressionType() { return null; } - + @Override public String toString() { return String.format("%s %s", expr, order); } -} \ No newline at end of file +} diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index a51c85e72..1d7172e3e 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -1,10 +1,10 @@ package sqlancer.postgres.ast; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.ArrayList; -import java.util.HashMap; import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; @@ -23,7 +23,7 @@ public class PostgresSelect extends SelectBase private PostgresExpression distinctOnClause; private ForClause forClause; private List windowFunctions = new ArrayList<>(); - private Map windowDefinitions = new HashMap<>(); + private final Map windowDefinitions = new HashMap<>(); public enum ForClause { UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); @@ -48,9 +48,8 @@ public static class WindowDefinition { private final List orderBy; private final WindowFrame frame; - public WindowDefinition(List partitionBy, - List orderBy, - WindowFrame frame) { + public WindowDefinition(List partitionBy, List orderBy, + WindowFrame frame) { this.partitionBy = partitionBy; this.orderBy = orderBy; this.frame = frame; @@ -91,7 +90,6 @@ public Map getWindowDefinitions() { return windowDefinitions; } - public static class PostgresFromTable implements PostgresExpression { private final PostgresTable t; private final boolean only; @@ -197,4 +195,4 @@ public ForClause getForClause() { public String asString() { return PostgresVisitor.asString(this); } -} \ No newline at end of file +} diff --git a/src/sqlancer/postgres/ast/PostgresWindowFunction.java b/src/sqlancer/postgres/ast/PostgresWindowFunction.java index 71ac6853a..15f87364f 100644 --- a/src/sqlancer/postgres/ast/PostgresWindowFunction.java +++ b/src/sqlancer/postgres/ast/PostgresWindowFunction.java @@ -1,17 +1,18 @@ package sqlancer.postgres.ast; import java.util.List; + import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresWindowFunction implements PostgresExpression { - + private final String functionName; private final List arguments; private final WindowSpecification windowSpec; private final PostgresDataType returnType; - public PostgresWindowFunction(String functionName, List arguments, - WindowSpecification windowSpec, PostgresDataType returnType) { + public PostgresWindowFunction(String functionName, List arguments, + WindowSpecification windowSpec, PostgresDataType returnType) { this.functionName = functionName; this.arguments = arguments; this.windowSpec = windowSpec; @@ -40,9 +41,8 @@ public static class WindowSpecification { private final List orderBy; private final WindowFrame frame; - public WindowSpecification(List partitionBy, - List orderBy, - WindowFrame frame) { + public WindowSpecification(List partitionBy, List orderBy, + WindowFrame frame) { this.partitionBy = partitionBy; this.orderBy = orderBy; this.frame = frame; @@ -63,8 +63,7 @@ public WindowFrame getFrame() { public static class WindowFrame { public enum FrameType { - ROWS("ROWS"), - RANGE("RANGE"); + ROWS("ROWS"), RANGE("RANGE"); private final String sql; @@ -99,4 +98,4 @@ public PostgresExpression getEndExpr() { return endExpr; } } -} \ No newline at end of file +} diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 83f9096d0..9f3251b1d 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -71,7 +71,6 @@ import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; - public class PostgresExpressionGenerator implements ExpressionGenerator, NoRECGenerator, TLPWhereGenerator, @@ -96,7 +95,7 @@ public class PostgresExpressionGenerator implements ExpressionGenerator functionsAndTypes; private final List allowedFunctionTypes; - + public PostgresExpressionGenerator(PostgresGlobalState globalState) { this.r = globalState.getRandomly(); this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); @@ -426,18 +425,18 @@ private PostgresExpression generateTextExpression(int depth) { throw new AssertionError(); } } - + private PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { - List arguments = generateWindowFunctionArguments(depth); - List partitionBy = generatePartitionByExpressions(depth); - List orderBy = generateOrderByExpressions(depth); - WindowFrame frame = generateWindowFrame(); - - WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); - String functionName = selectWindowFunctionName(); - - return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); -} + List arguments = generateWindowFunctionArguments(depth); + List partitionBy = generatePartitionByExpressions(depth); + List orderBy = generateOrderByExpressions(depth); + WindowFrame frame = generateWindowFrame(); + + WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + String functionName = selectWindowFunctionName(); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } private List generateWindowFunctionArguments(int depth) { List arguments = new ArrayList<>(); @@ -446,6 +445,7 @@ private List generateWindowFunctionArguments(int depth) { } return arguments; } + private List generatePartitionByExpressions(int depth) { List partitionBy = new ArrayList<>(); if (Randomly.getBoolean()) { @@ -463,7 +463,8 @@ private List generateOrderByExpressions(int depth) { int count = Randomly.smallNumber(); for (int i = 0; i < count; i++) { PostgresExpression expr = generateExpression(depth + 1); - // Call the second constructor in PostgresOrderByTerm, might be removed in the future to have only one constructor + // Call the second constructor in PostgresOrderByTerm, might be removed in the future to have only one + // constructor orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); } } @@ -481,14 +482,10 @@ private WindowFrame generateWindowFrame() { } private String selectWindowFunctionName() { - return Randomly.fromList(Arrays.asList( - "row_number", "rank", "dense_rank", "percent_rank", - "cume_dist", "ntile", "lag", "lead", "first_value", - "last_value", "nth_value" - )); + return Randomly.fromList(Arrays.asList("row_number", "rank", "dense_rank", "percent_rank", "cume_dist", "ntile", + "lag", "lead", "first_value", "last_value", "nth_value")); } - private PostgresExpression generateConcat(int depth) { PostgresExpression left = generateExpression(depth + 1, PostgresDataType.TEXT); PostgresExpression right = generateExpression(depth + 1); @@ -513,7 +510,7 @@ private PostgresExpression generateBitExpression(int depth) { } private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION } private PostgresExpression generateIntExpression(int depth) { @@ -537,6 +534,7 @@ private PostgresExpression generateIntExpression(int depth) { throw new AssertionError(); } } + private PostgresExpression createColumnOfType(PostgresDataType type) { List columns = filterColumns(type); PostgresColumn fromList = Randomly.fromList(columns); @@ -735,12 +733,12 @@ public PostgresExpression generateBooleanExpression() { @Override public PostgresSelect generateSelect() { PostgresSelect select = new PostgresSelect(); - + if (Randomly.getBooleanWithRatherLowProbability()) { List windowFunctions = generateWindowFunctions(); select.setWindowFunctions(windowFunctions); } - + return select; } @@ -748,9 +746,8 @@ private List generateWindowFunctions() { List windowFunctions = new ArrayList<>(); int numWindowFunctions = Randomly.smallNumber(); for (int i = 0; i < numWindowFunctions; i++) { - windowFunctions.add(generateWindowFunction(0, Randomly.fromList(Arrays.asList( - PostgresDataType.INT, PostgresDataType.FLOAT - )))); + windowFunctions.add(generateWindowFunction(0, + Randomly.fromList(Arrays.asList(PostgresDataType.INT, PostgresDataType.FLOAT)))); } return windowFunctions; } @@ -834,7 +831,7 @@ public String generateUnoptimizedQueryString(PostgresSelect select, PostgresExpr public String generateExplainQuery(PostgresSelect select) { return "EXPLAIN " + select.asString(); } - + @Override public boolean mutate(PostgresSelect select) { List> mutators = new ArrayList<>(); diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java index 1b9462d2b..a2aec3f7d 100644 --- a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -6,22 +6,23 @@ import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresWindowFunction; import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; -public class PostgresWindowFunctionGenerator { +public final class PostgresWindowFunctionGenerator { + private static final List WINDOW_FUNCTIONS = Arrays.asList("row_number", "rank", "dense_rank", + "percent_rank", "cume_dist", "ntile", "lag", "lead", "first_value", "last_value", "nth_value"); - private static final List WINDOW_FUNCTIONS = Arrays.asList( - "row_number", "rank", "dense_rank", "percent_rank", - "cume_dist", "ntile", "lag", "lead", "first_value", - "last_value", "nth_value" - ); + // Private constructor to prevent instantiation + private PostgresWindowFunctionGenerator() { + throw new AssertionError("Utility class should not be instantiated"); + } public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState globalState, List availableExpr) { @@ -38,27 +39,31 @@ private static String selectRandomWindowFunction() { return Randomly.fromList(WINDOW_FUNCTIONS); } - private static List generateFunctionArguments(String functionName, + private static List generateFunctionArguments(String functionName, PostgresGlobalState globalState, List availableExpr) { List arguments = new ArrayList<>(); switch (functionName) { - case "ntile": - - arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); - break; - case "lag": - case "lead": - case "nth_value": - arguments.add(Randomly.fromList(availableExpr)); - if (Randomly.getBoolean()) { - arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); - } - break; - case "first_value": - case "last_value": - arguments.add(Randomly.fromList(availableExpr)); - break; + case "ntile": + arguments + .add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); + break; + case "lag": + case "lead": + case "nth_value": + arguments.add(Randomly.fromList(availableExpr)); + if (Randomly.getBoolean()) { + arguments.add( + PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); + } + break; + case "first_value": + case "last_value": + arguments.add(Randomly.fromList(availableExpr)); + break; + default: + // No arguments needed for other window functions + break; } return arguments; @@ -89,14 +94,12 @@ private static List generateOrderByClause(List Date: Sun, 16 Mar 2025 10:38:24 -0400 Subject: [PATCH 1069/1284] Remove commented code and refactor PostgresWindowFunctionGenerator and PostgresToStringVisitor --- .../postgres/PostgresToStringVisitor.java | 38 ++----------------- .../gen/PostgresExpressionGenerator.java | 1 - .../gen/PostgresWindowFunctionGenerator.java | 23 ++++------- 3 files changed, 12 insertions(+), 50 deletions(-) diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 5df87aa5a..87bd3c429 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -189,13 +189,6 @@ public void visit(PostgresSelect s) { } } - // @Override - // public void visit(PostgresOrderByTerm op) { - // visit(op.getExpr()); - // sb.append(" "); - // sb.append(op.getOrder()); - // } - @Override public void visit(PostgresOrderByTerm term) { visit(term.getExpr()); @@ -373,17 +366,17 @@ public void visit(PostgresLikeOperation op) { } @Override + @SuppressWarnings("unchecked") public void visit(PostgresWindowFunction windowFunction) { sb.append(windowFunction.getFunctionName()); sb.append("("); - // Fix: Use visitList instead of visit for a list of expressions - visitList(windowFunction.getArguments()); + visit(windowFunction.getArguments()); sb.append(") OVER ("); WindowSpecification spec = windowFunction.getWindowSpec(); if (!spec.getPartitionBy().isEmpty()) { sb.append("PARTITION BY "); - visitList(spec.getPartitionBy()); + visit(spec.getPartitionBy()); } if (!spec.getOrderBy().isEmpty()) { @@ -391,8 +384,7 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(" "); } sb.append("ORDER BY "); - // Fix: Create a method to handle order by terms specifically - visitOrderByList(spec.getOrderBy()); + visit((List) (List) spec.getOrderBy()); } if (spec.getFrame() != null) { @@ -407,26 +399,4 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(")"); } - - // this method handles lists of expressions - private void visitList(List expressions) { - int i = 0; - for (PostgresExpression expr : expressions) { - if (i++ != 0) { - sb.append(", "); - } - visit(expr); - } - } - - // this method handles lists of order by terms - private void visitOrderByList(List orderBy) { - int i = 0; - for (PostgresOrderByTerm term : orderBy) { - if (i++ != 0) { - sb.append(", "); - } - visit(term); - } - } } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 9f3251b1d..39a31182a 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -52,7 +52,6 @@ import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; -// import sqlancer.postgres.ast.PostgresOrderByTerm.PostgresOrder; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression.POSIXRegex; import sqlancer.postgres.ast.PostgresPostfixOperation; diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java index a2aec3f7d..6daf66c3c 100644 --- a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -19,7 +19,6 @@ public final class PostgresWindowFunctionGenerator { private static final List WINDOW_FUNCTIONS = Arrays.asList("row_number", "rank", "dense_rank", "percent_rank", "cume_dist", "ntile", "lag", "lead", "first_value", "last_value", "nth_value"); - // Private constructor to prevent instantiation private PostgresWindowFunctionGenerator() { throw new AssertionError("Utility class should not be instantiated"); } @@ -72,10 +71,15 @@ private static List generateFunctionArguments(String functio private static WindowSpecification generateWindowSpecification(PostgresGlobalState globalState, List availableExpr) { List partitionBy = generatePartitionByClause(availableExpr); - List orderBy = generateOrderByClause(availableExpr); - WindowFrame frame = generateWindowFrame(globalState); + PostgresExpressionGenerator exprGen = new PostgresExpressionGenerator(globalState); + List orderBys = exprGen.generateOrderBys(); + List orderByTerms = new ArrayList<>(); + for (PostgresExpression expr : orderBys) { + orderByTerms.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + } - return new WindowSpecification(partitionBy, orderBy, frame); + WindowFrame frame = generateWindowFrame(globalState); + return new WindowSpecification(partitionBy, orderByTerms, frame); } private static List generatePartitionByClause(List availableExpr) { @@ -89,17 +93,6 @@ private static List generatePartitionByClause(List generateOrderByClause(List availableExpr) { - List orderBy = new ArrayList<>(); - if (Randomly.getBooleanWithRatherLowProbability()) { - int count = Randomly.smallNumber(); - for (int i = 0; i < count; i++) { - orderBy.add(new PostgresOrderByTerm(Randomly.fromList(availableExpr), Randomly.getBoolean())); - } - } - return orderBy; - } - private static WindowFrame generateWindowFrame(PostgresGlobalState globalState) { if (Randomly.getBooleanWithRatherLowProbability()) { WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); From 89ebbf58298895117582ecf6b6bf691056917810 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 16 Mar 2025 14:58:55 -0400 Subject: [PATCH 1070/1284] Remove window function generation from generateIntExpression --- .../gen/PostgresExpressionGenerator.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 39a31182a..bad87affc 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -15,7 +15,6 @@ import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TLPWhereGenerator; -import sqlancer.common.schema.AbstractTables; import sqlancer.postgres.PostgresBugs; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; @@ -231,8 +230,7 @@ private PostgresExpression generateBooleanExpression(int depth) { } private PostgresDataType getMeaningfulType() { - // make it more likely that the expression does not only consist of constant - // expressions + // make it more likely that the expression does not only consist of constant expressions if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { return PostgresDataType.getRandomType(); } else { @@ -425,7 +423,7 @@ private PostgresExpression generateTextExpression(int depth) { } } - private PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { + public PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { List arguments = generateWindowFunctionArguments(depth); List partitionBy = generatePartitionByExpressions(depth); List orderBy = generateOrderByExpressions(depth); @@ -508,8 +506,9 @@ private PostgresExpression generateBitExpression(int depth) { } } + // Removed WINDOW_FUNCTION option from the integer expression generation. private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION } private PostgresExpression generateIntExpression(int depth) { @@ -527,8 +526,6 @@ private PostgresExpression generateIntExpression(int depth) { case BINARY_ARITHMETIC_EXPRESSION: return new PostgresBinaryArithmeticOperation(generateExpression(depth + 1, PostgresDataType.INT), generateExpression(depth + 1, PostgresDataType.INT), PostgresBinaryOperator.getRandom()); - case WINDOW_FUNCTION: - return generateWindowFunction(depth + 1, PostgresDataType.INT); default: throw new AssertionError(); } @@ -718,7 +715,8 @@ public PostgresExpression isNull(PostgresExpression expr) { } @Override - public PostgresExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + public PostgresExpressionGenerator setTablesAndColumns( + sqlancer.common.schema.AbstractTables targetTables) { this.targetTables = targetTables.getTables(); this.columns = targetTables.getColumns(); return this; @@ -806,7 +804,7 @@ public String generateOptimizedQueryString(PostgresSelect select, PostgresExpres select.setFetchColumns(Arrays.asList(allColumns)); } select.setWhereClause(whereCondition); - if (Randomly.getBooleanWithSmallProbability()) { + if (Randomly.getBooleanWithRatherLowProbability()) { select.setOrderByClauses(generateOrderBys()); } select.setSelectType(SelectType.ALL); @@ -839,7 +837,7 @@ public boolean mutate(PostgresSelect select) { mutators.add(this::mutateWhere); mutators.add(this::mutateGroupBy); mutators.add(this::mutateHaving); - mutators.add(this::mutateWindowFunction); // Add window function mutation + mutators.add(this::mutateWindowFunction); if (!PostgresBugs.bug18643) { mutators.add(this::mutateAnd); mutators.add(this::mutateOr); @@ -852,13 +850,11 @@ public boolean mutate(PostgresSelect select) { private boolean mutateWindowFunction(PostgresSelect select) { List windowFunctions = select.getWindowFunctions(); if (windowFunctions == null || windowFunctions.isEmpty()) { - // Add a new window function windowFunctions = new ArrayList<>(); windowFunctions.add(generateWindowFunction(0, PostgresDataType.INT)); select.setWindowFunctions(windowFunctions); return false; } else { - // Remove a random window function windowFunctions.remove(Randomly.fromList(windowFunctions)); if (windowFunctions.isEmpty()) { select.setWindowFunctions(null); From c5bc6dd654b6dfc0ea2a86238671b7f7a7cb2778 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Wed, 19 Mar 2025 14:09:23 -0400 Subject: [PATCH 1071/1284] Add test coverage for Randomly class --- test/sqlancer/TestRandomly.java | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/test/sqlancer/TestRandomly.java b/test/sqlancer/TestRandomly.java index 6be6b1bd4..5a58ca50b 100644 --- a/test/sqlancer/TestRandomly.java +++ b/test/sqlancer/TestRandomly.java @@ -234,4 +234,60 @@ private List getRandomValueList(Randomly r) { return values; } + @Test + public void testGetPercentage() { + for (int i = 0; i < NR_MIN_RUNS; i++) { + double percentage = Randomly.getPercentage(); + assertTrue(percentage >= 0.0); + assertTrue(percentage <= 1.0); + } + } + + @Test + public void testGetChar() { + Randomly r = new Randomly(); + boolean encounteredAlphabetic = false; + boolean encounteredNumeric = false; + boolean encounteredSpecial = false; + int i = 0; + do { + String c = r.getChar(); + assertEquals(1, c.length()); + if (Character.isAlphabetic(c.charAt(0))) { + encounteredAlphabetic = true; + } else if (Character.isDigit(c.charAt(0))) { + encounteredNumeric = true; + } else { + encounteredSpecial = true; + } + } while (!encounteredAlphabetic || !encounteredNumeric || !encounteredSpecial || i++ < NR_MIN_RUNS); + } + + @Test + public void testGetAlphabeticChar() { + Randomly r = new Randomly(); + for (int i = 0; i < NR_MIN_RUNS; i++) { + String c = r.getAlphabeticChar(); + assertEquals(1, c.length()); + assertTrue(Character.isAlphabetic(c.charAt(0))); + } + } + + @Test + public void testGetBooleanWithSmallProbability() { + int trueCount = 0; + int totalRuns = NR_MIN_RUNS; + + for (int i = 0; i < totalRuns; i++) { + if (Randomly.getBooleanWithSmallProbability()) { + trueCount++; + } + } + + double trueRatio = (double) trueCount / totalRuns; + assertTrue(trueRatio > 0.005); + assertTrue(trueRatio < 0.015); + + } + } From e2f5495c85a0f3d919a6b9b63fa66aefb7dff26d Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Thu, 20 Mar 2025 10:01:57 -0400 Subject: [PATCH 1072/1284] Add assert in testGetChar --- test/sqlancer/TestRandomly.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/sqlancer/TestRandomly.java b/test/sqlancer/TestRandomly.java index 5a58ca50b..ad0899420 100644 --- a/test/sqlancer/TestRandomly.java +++ b/test/sqlancer/TestRandomly.java @@ -261,6 +261,9 @@ public void testGetChar() { encounteredSpecial = true; } } while (!encounteredAlphabetic || !encounteredNumeric || !encounteredSpecial || i++ < NR_MIN_RUNS); + assertTrue(encounteredAlphabetic, "Never encounter an alphabetic character."); + assertTrue(encounteredNumeric, "Never encounter a numeric character."); + assertTrue(encounteredSpecial, "Did not encounter a special character."); } @Test From 25048b1a10a5c2be747b25a1d3c8649969e8fd90 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Thu, 20 Mar 2025 10:04:05 -0400 Subject: [PATCH 1073/1284] Add assert in testGetChar --- test/sqlancer/TestRandomly.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sqlancer/TestRandomly.java b/test/sqlancer/TestRandomly.java index ad0899420..284f0083a 100644 --- a/test/sqlancer/TestRandomly.java +++ b/test/sqlancer/TestRandomly.java @@ -263,7 +263,7 @@ public void testGetChar() { } while (!encounteredAlphabetic || !encounteredNumeric || !encounteredSpecial || i++ < NR_MIN_RUNS); assertTrue(encounteredAlphabetic, "Never encounter an alphabetic character."); assertTrue(encounteredNumeric, "Never encounter a numeric character."); - assertTrue(encounteredSpecial, "Did not encounter a special character."); + assertTrue(encounteredSpecial, "Never encounter a special character."); } @Test From b8a1162e5469f4e35eefd20ca4beac42bc1a2e9f Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Thu, 20 Mar 2025 11:19:42 -0400 Subject: [PATCH 1074/1284] Add assertions to test methods in testRandomly --- test/sqlancer/TestRandomly.java | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/test/sqlancer/TestRandomly.java b/test/sqlancer/TestRandomly.java index 6be6b1bd4..595f0337b 100644 --- a/test/sqlancer/TestRandomly.java +++ b/test/sqlancer/TestRandomly.java @@ -38,6 +38,7 @@ public void testSubset() { boolean encounteredStrictSubsetNonEmpty = false; Integer[] options = { 1, 2, 3 }; List optionList = new ArrayList<>(Arrays.asList(options)); + int i = 0; do { List subset = Randomly.subset(optionList); assertEquals(optionList.size(), 3); // check that the original set hasn't been modified @@ -49,7 +50,13 @@ public void testSubset() { } else { encounteredStrictSubsetNonEmpty = true; } - } while (!encounteredEmptySubset || !encounteredOriginalSet || !encounteredStrictSubsetNonEmpty); + } while (!encounteredEmptySubset || !encounteredOriginalSet || !encounteredStrictSubsetNonEmpty + || i++ < NR_MIN_RUNS); + + assertTrue(encounteredEmptySubset, "Empty subset was not encountered"); + assertTrue(encounteredOriginalSet, "Original set was not encountered"); + assertTrue(encounteredStrictSubsetNonEmpty, "Strict subset was not encountered"); + } @Test @@ -75,6 +82,10 @@ public void testString() { } } while (!encounteredInteger || !encounteredAscii || !encounteredNonAscii || !encounteredSpace || i++ < NR_MIN_RUNS); + assertTrue(encounteredInteger, "Integer was not encountered"); + assertTrue(encounteredAscii, "Ascii was not encountered"); + assertTrue(encounteredNonAscii, "Non ascii was not encountered"); + assertTrue(encounteredSpace, "Space was not encountered"); } @Test // TODO: also generate and check for NaN @@ -84,6 +95,7 @@ public void testDouble() { boolean encounteredPositive = false; boolean encounteredNegative = false; boolean encounteredInfinity = false; + int i = 0; do { double doubleVal = r.getDouble(); if (doubleVal == 0) { @@ -97,7 +109,12 @@ public void testDouble() { } else { fail(String.valueOf(doubleVal)); } - } while (!encounteredZero || !encounteredPositive || !encounteredNegative || !encounteredInfinity); + } while (!encounteredZero || !encounteredPositive || !encounteredNegative || !encounteredInfinity + || i++ < NR_MIN_RUNS); + assertTrue(encounteredZero, "Zero was not encountered"); + assertTrue(encounteredPositive, "Positive was not encountered"); + assertTrue(encounteredNegative, "Negative was not encountered"); + assertTrue(encounteredInfinity, "Infinity was not encountered"); } @Test @@ -123,6 +140,8 @@ public void testNonZeroInteger() { encounteredNegative = true; } } while (!encounteredPositive || !encounteredNegative || i++ < NR_MIN_RUNS); + assertTrue(encounteredPositive, "Positive integer was not encountered"); + assertTrue(encounteredNegative, "Negative integer was not encountered"); } @Test @@ -140,6 +159,8 @@ public void testPositiveInteger() { encounteredMaxValue = true; } } while (!encounteredZero || !encounteredMaxValue || i++ < NR_MIN_RUNS); + assertTrue(encounteredZero, "Zero was not encountered"); + assertTrue(encounteredMaxValue, "Max value was not encountered"); } @Test @@ -159,6 +180,9 @@ public void testBytes() { encounteredMax = true; } } while (!encounteredAllZeroes || !encounteredMax || !encounteredZeroLength || i++ < NR_MIN_RUNS); + assertTrue(encounteredAllZeroes, "All zeroes were not encountered"); + assertTrue(encounteredMax, "Max value was not encountered"); + assertTrue(encounteredZeroLength, "Zero length was not encountered"); } @Test From fb2c6b014abf2dd5206314fa14988aed23e04e3e Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Fri, 21 Mar 2025 12:38:18 +0000 Subject: [PATCH 1075/1284] Add unit tests for schema classes --- test/sqlancer/TestCommonSchema.java | 156 ++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 test/sqlancer/TestCommonSchema.java diff --git a/test/sqlancer/TestCommonSchema.java b/test/sqlancer/TestCommonSchema.java new file mode 100644 index 000000000..764e94ea4 --- /dev/null +++ b/test/sqlancer/TestCommonSchema.java @@ -0,0 +1,156 @@ +package sqlancer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestCommonSchema { + static class TestTable extends AbstractTable> { + TestTable(String name, List columns, List indexes, boolean isView) { + super(name, columns, indexes, isView); + } + + @Override + public long getNrRows(GlobalState globalState) { + return 0; + } + } + + static class TestTableColumn extends AbstractTableColumn { + TestTableColumn(String name, TestTable table, String type) { + super(name, table, type); + } + } + + static class TestSchema extends AbstractSchema, TestTable> { + TestSchema(List tables) { + super(tables); + } + } + + static class TestIndex extends TableIndex { + TestIndex(String name) { + super(name); + } + } + + private TestTable createTestTable(String name, List indexes, boolean isView, String... columns) { + List cols = Arrays.stream(columns) + .map(col -> new TestTableColumn(col, null, "VARCHAR")) + .collect(Collectors.toList()); + return new TestTable(name, cols, indexes, isView); + } + + private TestTableColumn createTestColumn(String name, TestTable table, String type) { + return new TestTableColumn(name, table, type); + } + + private TestSchema createTestSchema(TestTable... tables) { + return new TestSchema(Arrays.asList(tables)); + } + + @Test + void testColumnManagement() { + TestTable table = createTestTable("products", Collections.emptyList(), false, "sku", "price"); + TestTableColumn randomCol = table.getRandomColumn(); + + assertEquals("sku", table.getColumns().get(0).getName()); + assertEquals("VARCHAR", table.getColumns().get(1).getType()); + + assertTrue(table.getColumns().contains(randomCol)); + } + + @Test + void testIndexManagement() { + TestIndex idx1 = new TestIndex("idx_sku"); + TestIndex idx2 = new TestIndex("idx_price"); + TestTable table = createTestTable("products", Arrays.asList(idx1, idx2), false, "sku", "price"); + TableIndex randomIndex = table.getRandomIndex(); + + assertTrue(table.hasIndexes()); + assertEquals(2, table.getIndexes().size()); + assertTrue(table.getIndexes().contains(randomIndex)); + } + + @Test + void testViewManagement() { + TestTable view1 = createTestTable("v1", Collections.emptyList(),true, "col1"); + TestTable view2 = createTestTable("v2", Collections.emptyList(),true, "col2"); + TestTable table = createTestTable("t1", Collections.emptyList(), false, "col3"); + TestSchema schema = createTestSchema(view1, view2, table); + + assertAll( + () -> assertEquals(2, schema.getViews().size(), "Should detect 2 views"), + () -> assertEquals(1, schema.getDatabaseTablesWithoutViews().size(), "Should detect 1 normal table"), + () -> assertEquals("t1", schema.getDatabaseTablesWithoutViews().get(0).getName()) + ); + } + + @Test + void testFreeColumnNameGeneration() { + TestTable table = createTestTable("users", Collections.emptyList(), false, "id", "name"); + Set generatedNames = new HashSet<>(); + + for (int i = 0; i < 100; i++) { + String newName = table.getFreeColumnName(); + assertTrue(generatedNames.add(newName), "Duplicate: " + newName); + + List newColumns = new ArrayList<>(table.getColumns()); + newColumns.add(new TestTableColumn(newName, table, "TEXT")); + table = new TestTable(table.getName(), newColumns, table.getIndexes(), table.isView()); + } + } + + @Test + void testObjectComparison() { + TestTable tableA = createTestTable("A", Collections.emptyList(), false, "x", "y"); + TestTable tableB = createTestTable("B", Collections.emptyList(), false, "b"); + + TestTableColumn colA1 = new TestTableColumn("x", tableA, "INT"); + TestTableColumn colA2 = new TestTableColumn("y", tableA, "INT"); + TestTableColumn colB1 = new TestTableColumn("b", tableB, "TEXT"); + + assertAll( + () -> assertTrue(colA1.compareTo(colA2) < 0, "Columns should be ordered by name"), + () -> assertTrue(colA1.compareTo(colB1) < 0, "Columns should be ordered by name"), + () -> assertTrue(tableA.compareTo(tableB) > 0, "Tables should be ordered reverse-alphabetically"), + () -> assertEquals(0, tableA.compareTo(tableA), "Same table should be equal") + ); + } + + @Test + void testEquality() { + TestTable table1 = createTestTable("t1", Collections.emptyList(), false, "id"); + TestTable table2 = createTestTable("t2", Collections.emptyList(), false, "id"); + + TestTableColumn col1 = new TestTableColumn("id", table1, "INT"); + TestTableColumn col2 = new TestTableColumn("id", table1, "INT"); + TestTableColumn col3 = new TestTableColumn("id", table2, "INT"); + TestTableColumn col4 = new TestTableColumn("name", table1, "TEXT"); + + assertAll( + () -> assertEquals(col1, col2, "Same table/column should be equal"), + () -> assertNotEquals(col1, col3, "Different tables should not be equal"), + () -> assertNotEquals(col1, col4, "Different columns should not be equal"), + () -> assertNotEquals(col1, "invalid_object", "Different types should not be equal") + ); + } + + @Test + void testBoundaryConditions() { + String longName = "a".repeat(256); + TestTableColumn longCol = new TestTableColumn(longName, null, "TEXT"); + assertEquals(longName, longCol.getName()); + + TestTableColumn col2 = createTestColumn("orphan", null, "UNKNOWN"); + assertEquals("orphan", col2.getFullQualifiedName()); + assertNull(col2.getTable()); + } +} From 17952a41abe734da4e401faad501e626811c9158 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Fri, 3 Jan 2025 20:02:31 +0800 Subject: [PATCH 1076/1284] add the implementation of CODDTest for SQLite3 --- src/sqlancer/common/oracle/CODDTestBase.java | 25 + .../common/schema/AbstractTables.java | 24 + .../sqlite3/SQLite3ExpectedValueVisitor.java | 46 + src/sqlancer/sqlite3/SQLite3Options.java | 3 + .../sqlite3/SQLite3OracleFactory.java | 12 + .../sqlite3/SQLite3ToStringVisitor.java | 158 ++- src/sqlancer/sqlite3/SQLite3Visitor.java | 36 + .../sqlite3/ast/SQLite3Expression.java | 201 +++- src/sqlancer/sqlite3/ast/SQLite3Select.java | 32 + .../sqlite3/oracle/SQLite3CODDTestOracle.java | 1002 +++++++++++++++++ .../sqlite3/schema/SQLite3DataType.java | 17 + 11 files changed, 1552 insertions(+), 4 deletions(-) create mode 100644 src/sqlancer/common/oracle/CODDTestBase.java create mode 100644 src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java diff --git a/src/sqlancer/common/oracle/CODDTestBase.java b/src/sqlancer/common/oracle/CODDTestBase.java new file mode 100644 index 000000000..162a8eaed --- /dev/null +++ b/src/sqlancer/common/oracle/CODDTestBase.java @@ -0,0 +1,25 @@ +package sqlancer.common.oracle; + +import sqlancer.Main.StateLogger; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; + +public abstract class CODDTestBase> implements TestOracle { + protected final S state; + protected final ExpectedErrors errors = new ExpectedErrors(); + protected final StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + protected String auxiliaryQueryString; + protected String foldedQueryString; + protected String originalQueryString; + + public CODDTestBase(S state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } +} \ No newline at end of file diff --git a/src/sqlancer/common/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java index 2afff82ab..c75cb30e1 100644 --- a/src/sqlancer/common/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -34,4 +34,28 @@ public String columnNamesAsString(Function function) { return getColumns().stream().map(function).collect(Collectors.joining(", ")); } + + public void addTable(T table) { + if (!this.tables.contains(table)) { + this.tables.add(table); + columns.addAll(table.getColumns()); + } + } + + public void removeTable(T table) { + if (this.tables.contains(table)) { + this.tables.remove(table); + for (C c : table.getColumns()) { + columns.remove(c); + } + } + } + + public Boolean isContained(T table) { + return this.tables.contains(table); + } + + public int getSize() { + return this.tables.size(); + } } diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 1c148ee86..40f8ae20a 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -14,14 +14,21 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -303,4 +310,43 @@ public void visit(SQLite3SetClause set) { visit(set.getRight()); } + @Override + public void visit(SQLite3Alias alias) { + print(alias); + print(alias.getOrigonalExpression()); + print(alias.getAliasExpression()); + } + + @Override + public void visit(SQLite3WithClasure withClasure) { + print(withClasure); + print(withClasure.getLeft()); + print(withClasure.getRight()); + } + + @Override + public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { + print(tableAndColumnRef); + } + + @Override + public void visit(SQLite3Values values) { + print(values); + } + + @Override + public void visit(SQLite3ExpressionBag expr) { + print(expr); + print(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3Typeof expr) { + print(expr); + print(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3ResultMap tableSummary) { + } } diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index ec90f3bca..47a80e832 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -79,6 +79,9 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") public int maxNumIndexes = 20; + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") + public String coddTestModel = "random"; + @Override public List getTestOracleFactory() { return Arrays.asList(oracles); diff --git a/src/sqlancer/sqlite3/SQLite3OracleFactory.java b/src/sqlancer/sqlite3/SQLite3OracleFactory.java index b1a9a1265..2e2f9f0f7 100644 --- a/src/sqlancer/sqlite3/SQLite3OracleFactory.java +++ b/src/sqlancer/sqlite3/SQLite3OracleFactory.java @@ -11,6 +11,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.oracle.SQLite3CODDTestOracle; import sqlancer.sqlite3.oracle.SQLite3Fuzzer; import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; import sqlancer.sqlite3.oracle.tlp.SQLite3TLPAggregateOracle; @@ -96,6 +97,17 @@ public TestOracle create(SQLite3GlobalState globalState) thr oracles.add(AGGREGATE.create(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + CODDTest { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3CODDTestOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } }; } diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 09eb224fb..44aa0d1e0 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -1,7 +1,11 @@ package sqlancer.sqlite3; import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; @@ -11,6 +15,7 @@ import sqlancer.sqlite3.ast.SQLite3Case.SQLite3CaseWithoutBaseExpression; import sqlancer.sqlite3.ast.SQLite3Cast; import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.ast.SQLite3Constant.SQLite3NullConstant; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.BetweenOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Cast; @@ -19,12 +24,19 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; @@ -35,6 +47,7 @@ import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecBetween; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecTerm; +import sqlancer.sqlite3.schema.SQLite3DataType; public class SQLite3ToStringVisitor extends ToStringVisitor implements SQLite3Visitor { @@ -99,6 +112,10 @@ public void visit(SQLite3Select s, boolean inner) { if (inner) { sb.append("("); } + if (s.getWithClasure() != null) { + visit(s.getWithClasure()); + sb.append(" "); + } sb.append("SELECT "); switch (s.getFromOptions()) { case DISTINCT: @@ -288,13 +305,20 @@ public void visit(InOperation op) { sb.append("("); visit(op.getLeft()); sb.append(" IN "); - sb.append("("); if (op.getRightExpressionList() != null) { + sb.append("("); visit(op.getRightExpressionList()); + sb.append(")"); } else { - visit(op.getRightSelect()); + if (op.getRightSelect() instanceof SQLite3Expression.SQLite3TableReference) { + visit(op.getRightSelect()); + } else { + sb.append("("); + visit(op.getRightSelect()); + sb.append(")"); + } } - sb.append(")"); + sb.append(")"); } @@ -305,6 +329,9 @@ public void visit(Subquery query) { @Override public void visit(SQLite3Exist exist) { + if (exist.getNegated()) { + sb.append(" NOT"); + } sb.append(" EXISTS "); if (exist.getExpression() instanceof SQLite3SetClause) { sb.append("("); @@ -482,4 +509,129 @@ public void visit(SQLite3SetClause set) { sb.append(SQLite3Visitor.asString(set.getRight())); } + @Override + public void visit(SQLite3Alias alias) { + sb.append("("); + visit(alias.getOrigonalExpression()); + sb.append(")"); + sb.append(" AS "); + visit(alias.getAliasExpression()); + } + + @Override + public void visit(SQLite3WithClasure withClasure) { + sb.append("WITH "); + visit(withClasure.getLeft()); + sb.append(" AS "); + visit(withClasure.getRight()); + } + + @Override + public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { + sb.append(tableAndColumnRef.getString()); + } + + @Override + public void visit(SQLite3Values values) { + Map> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + sb.append("(VALUES "); + for (int i = 0; i < size; i++) { + sb.append("("); + for (String name : columnNames) { + if (vs.get(name).get(i).getDataType() == SQLite3DataType.NULL) { + visit(vs.get(name).get(i)); + } else { + sb.append("(CAST("); + visit(vs.get(name).get(i)); + sb.append(" AS "); + switch(vs.get(name).get(i).getDataType()) { + case BINARY: + sb.append("BLOB))"); + break; + case INT: + sb.append("INT))"); + break; + case TEXT: + sb.append("TEXT))"); + break; + case REAL: + sb.append("REAL))"); + break; + default: + throw new IgnoreMeException(); + } + } + + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("), "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + } + + @Override + public void visit(SQLite3ExpressionBag expr) { + visit(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3Typeof expr) { + sb.append("typeof("); + visit(expr.getInnerExpr()); + sb.append(")"); + } + + @Override + public void visit(SQLite3ResultMap tableSummary) { + // we utlize CASE WHEN THEN END here + SQLite3Values values = tableSummary.getValues(); + List columnRefs = tableSummary.getColumns(); + List summary = tableSummary.getSummary(); + + Map> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + if (size == 0) { + sb.append("("); + for (int j = 0; j < columnRefs.size(); ++j) { + visit(columnRefs.get(j)); + sb.append(" IS NULL AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return; + } + List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + sb.append(" CASE "); + for (int i = 0; i < size; i++) { + sb.append("WHEN "); + for (int j = 0; j < columnNames.size(); ++j) { + visit(columnRefs.get(j)); + if (vs.get(columnNames.get(j)).get(i) instanceof SQLite3NullConstant) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(vs.get(columnNames.get(j)).get(i).toString()); + } + sb.append(" AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("THEN "); + visit(summary.get(i)); + sb.append(" "); + } + sb.append("END "); + } } diff --git a/src/sqlancer/sqlite3/SQLite3Visitor.java b/src/sqlancer/sqlite3/SQLite3Visitor.java index f891c17f0..d5ebce096 100644 --- a/src/sqlancer/sqlite3/SQLite3Visitor.java +++ b/src/sqlancer/sqlite3/SQLite3Visitor.java @@ -13,14 +13,21 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -130,6 +137,21 @@ default void visit(SQLite3PostfixUnaryOperation exp) { void visit(SQLite3WindowFunctionFrameSpecBetween between); + + void visit(SQLite3Alias alias); + + void visit(SQLite3WithClasure withClasure); + + void visit(SQLite3TableAndColumnRef tableAndColumnRef); + + void visit(SQLite3Values values); + + void visit(SQLite3ExpressionBag expr); + + void visit(SQLite3Typeof expr); + + void visit(SQLite3ResultMap tableSummary); + default void visit(SQLite3Expression expr) { if (expr instanceof Sqlite3BinaryOperation) { visit((Sqlite3BinaryOperation) expr); @@ -193,6 +215,20 @@ default void visit(SQLite3Expression expr) { visit((SQLite3TableReference) expr); } else if (expr instanceof SQLite3SetClause) { visit((SQLite3SetClause) expr); + } else if (expr instanceof SQLite3Alias) { + visit((SQLite3Alias) expr); + } else if (expr instanceof SQLite3WithClasure) { + visit((SQLite3WithClasure) expr); + } else if (expr instanceof SQLite3TableAndColumnRef) { + visit((SQLite3TableAndColumnRef) expr); + } else if (expr instanceof SQLite3Values) { + visit((SQLite3Values) expr); + } else if (expr instanceof SQLite3ExpressionBag) { + visit((SQLite3ExpressionBag) expr); + } else if (expr instanceof SQLite3Typeof) { + visit((SQLite3Typeof) expr); + } else if (expr instanceof SQLite3ResultMap) { + visit((SQLite3ResultMap) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 7b131b6b5..a14432eec 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -2,6 +2,7 @@ import java.util.List; import java.util.Optional; +import java.util.Map; import sqlancer.IgnoreMeException; import sqlancer.LikeImplementationHelper; @@ -113,9 +114,19 @@ public SQLite3CollateSequence getImplicitCollateSequence() { public static class SQLite3Exist extends SQLite3Expression { private final SQLite3Expression select; + private boolean negated = false; - public SQLite3Exist(SQLite3Expression select) { + public SQLite3Exist(SQLite3Expression select, boolean negated) { this.select = select; + this.negated = negated; + } + + public void setNegated(boolean negated) { + this.negated = negated; + } + + public boolean getNegated() { + return this.negated; } public SQLite3Expression getExpression() { @@ -1552,4 +1563,192 @@ public boolean omitBracketsWhenPrinting() { } } + public static class SQLite3WithClasure extends SQLite3Expression { + + private SQLite3Expression left; + private SQLite3Expression right; + + public SQLite3WithClasure(SQLite3Expression left, SQLite3Expression right) { + this.left = left; + this.right = right; + } + + public SQLite3Expression getLeft() { + return this.left; + } + + public SQLite3Expression getRight() { + return this.right; + } + + public void updateRight(SQLite3Expression right) { + this.right = right; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3Alias extends SQLite3Expression { + + private SQLite3Expression origonalExpression; + private SQLite3Expression aliasExpression; + + public SQLite3Alias(SQLite3Expression origonalExpression, SQLite3Expression aliasExpression) { + this.origonalExpression = origonalExpression; + this.aliasExpression = aliasExpression; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + public SQLite3Expression getOrigonalExpression() { + return origonalExpression; + } + + public SQLite3Expression getAliasExpression() { + return aliasExpression; + } + } + + public static class SQLite3TableAndColumnRef extends SQLite3Expression { + + private final SQLite3Table table; + + public SQLite3TableAndColumnRef(SQLite3Table table) { + this.table = table; + } + + public SQLite3Table getTable() { + return this.table; + } + + public String getString() { + StringBuilder sb = new StringBuilder(); + sb.append(table.getName()); + sb.append("("); + for (SQLite3Column c : this.table.getColumns()) { + sb.append(c.getName()); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3Values extends SQLite3Expression { + + private final Map> values; + private final List columns; + + public SQLite3Values(Map> values, List columns) { + this.values = values; + this.columns = columns; + } + + public Map> getValues() { + return this.values; + } + + public List getColumns() { + return this.columns; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3ExpressionBag extends SQLite3Expression { + private SQLite3Expression innerExpr; + + public SQLite3ExpressionBag(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public void updateInnerExpr(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public SQLite3Expression getInnerExpr() { + return innerExpr; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } + + public static class SQLite3Typeof extends SQLite3Expression { + private SQLite3Expression innerExpr; + + public SQLite3Typeof(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public SQLite3Expression getInnerExpr() { + return innerExpr; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } + + public static class SQLite3ResultMap extends SQLite3Expression { + private final SQLite3Values values; + private final List columns; + private final List summary; + private final SQLite3DataType summaryDataType; + + public SQLite3ResultMap(SQLite3Values values, List columns, List summary, SQLite3DataType summaryDataType) { + this.values = values; + this.columns = columns; + this.summary = summary; + this.summaryDataType = summaryDataType; + + Map> vs = values.getValues(); + if (vs.get(vs.keySet().iterator().next()).size() != summary.size()) { + throw new AssertionError(); + } + } + + public SQLite3Values getValues() { + return this.values; + } + + public List getColumns() { + return this.columns; + } + + public List getSummary() { + return this.summary; + } + + public SQLite3DataType getSummaryDataType() { + return this.summaryDataType; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 57eb91b0b..042aab856 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -4,6 +4,7 @@ import java.util.Collections; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.common.ast.newast.Select; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression.Join; @@ -24,6 +25,7 @@ public class SQLite3Select extends SQLite3Expression private List fetchColumns = Collections.emptyList(); private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; + private SQLite3WithClasure withClause = null; public SQLite3Select() { } @@ -42,6 +44,7 @@ public SQLite3Select(SQLite3Select other) { joinStatements.add(new Join(j)); } havingClause = other.havingClause; + withClause = other.withClause; } public enum SelectType { @@ -161,4 +164,33 @@ public SQLite3Expression getHavingClause() { public String asString() { return SQLite3Visitor.asString(this); } + + public void setWithClasure(SQLite3WithClasure withClasure) { + this.withClause = withClasure; + } + + public void updateWithClasureRight(SQLite3Expression withClasureRight) { + this.withClause.updateRight(withClasureRight); + } + + public SQLite3Expression getWithClasure() { + return this.withClause; + } + + public void replaceFromTable(String tableName, SQLite3Expression newFromExpression) { + int replaceIdx = -1; + for (int i = 0; i < fromList.size(); ++i) { + SQLite3Expression f = fromList.get(i); + if (f instanceof SQLite3TableReference) { + SQLite3TableReference tableRef = (SQLite3TableReference) f; + if (tableRef.getTable().getName() == tableName) { + replaceIdx = i; + } + } + } + if (replaceIdx == -1) { + throw new IgnoreMeException(); + } + fromList.set(replaceIdx, newFromExpression); + } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java new file mode 100644 index 000000000..d20e6e9be --- /dev/null +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -0,0 +1,1002 @@ +package sqlancer.sqlite3.oracle; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.CODDTestBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3Provider; +import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Aggregate; +import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.ast.SQLite3Constant.SQLite3TextConstant; +import sqlancer.sqlite3.ast.SQLite3Expression; +import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; +import sqlancer.sqlite3.ast.SQLite3Select; +import sqlancer.sqlite3.gen.SQLite3Common; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.schema.SQLite3DataType; +import sqlancer.sqlite3.schema.SQLite3Schema; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; + +public class SQLite3CODDTestOracle extends CODDTestBase implements TestOracle { + + private final SQLite3Schema s; + private SQLite3ExpressionGenerator gen; + private Reproducer reproducer; + + private String tempTableName = "temp_table"; + + private SQLite3Expression foldedExpr; + private SQLite3Expression constantResOfFoldedExpr; + + private List tablesFromOuterContext = new ArrayList<>(); + private List joinsInExpr = null; + + Map> auxiliaryQueryResult = new HashMap<>(); + Map> selectResult = new HashMap<>(); + + Boolean useSubqueryAsFoldedExpr; + Boolean useCorrelatedSubqueryAsFoldedExpr; + + public SQLite3CODDTestOracle(SQLite3GlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + SQLite3Errors.addExpectedExpressionErrors(errors); + SQLite3Errors.addMatchQueryErrors(errors); + SQLite3Errors.addQueryErrors(errors); + errors.add("misuse of aggregate"); + errors.add("misuse of window function"); + errors.add("second argument to nth_value must be a positive integer"); + errors.add("no such table"); + errors.add("no query solution"); + errors.add("unable to use function MATCH in the requested context"); + errors.add("[SQLITE_ERROR] SQL error or missing database (unrecognized token:"); + } + + @Override + public void check() throws SQLException { + reproducer = null; + + useSubqueryAsFoldedExpr = useSubquery(); + useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); + + SQLite3Select auxiliaryQuery = null; + if (useSubqueryAsFoldedExpr) { + if (useCorrelatedSubqueryAsFoldedExpr) { + auxiliaryQuery = genSelectWithCorrelatedSubquery(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + + auxiliaryQueryResult.putAll(selectResult); + } else { + auxiliaryQuery = genSelectExpression(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); + } + } else { + auxiliaryQuery = genSimpleSelect(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + + auxiliaryQueryResult.putAll(selectResult); + } + + + SQLite3Select originalQuery = null; + + Map> foldedResult = new HashMap<>(); + Map> originalResult = new HashMap<>(); + + // dependent expression + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + // original query + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(this.foldedExpr); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // independent expression + // empty result, put the inner query in (NOT) EXIST + else if (auxiliaryQueryResult.size() == 0 || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).size() == 0) { + boolean isNegated = Randomly.getBoolean() ? false : true; + // original query + SQLite3Exist existExpr = new SQLite3Exist(new SQLite3Select(auxiliaryQuery), isNegated); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(existExpr); + + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + SQLite3Expression equivalentExpr = isNegated ? SQLite3Constant.createTrue() : SQLite3Constant.createFalse(); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant + else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 && Randomly.getBoolean()) { + // original query + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(auxiliaryQuery); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + SQLite3Expression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); + specificCondition.updateInnerExpr(equivalentExpr);; + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // one column + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability()) { + // else if (auxiliaryQueryResult.size() == 1 && false) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + SQLite3ColumnName selectedColumn = new SQLite3ColumnName(Randomly.fromList(columns), null); + SQLite3Table selectedTable = selectedColumn.getColumn().getTable(); + InOperation INOperation = new InOperation(selectedColumn, new SQLite3Select(auxiliaryQuery)); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(INOperation); + + originalQuery = this.genSelectExpression(selectedTable, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + // folded query + // can not use IN VALUES here, because there is no affinity for the right operand of IN when right operand is a list + try { + SQLite3Table t = this.createTemporaryTable(auxiliaryQuery, "intable"); + SQLite3TableReference equivalentTable = new SQLite3TableReference(t); + INOperation = new InOperation(selectedColumn, equivalentTable); + specificCondition.updateInnerExpr(INOperation); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable("intable"); + } + } + // There is not `ANY` and `ALL` operator in SQLite3 + // Row Subquery + else { + // original query + SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); + originalQuery = this.genSelectExpression(temporaryTable, null); + SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); + SQLite3WithClasure withClasure = new SQLite3WithClasure(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); + originalQuery.setWithClasure(withClasure); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + // folded query + if (Randomly.getBoolean() && this.testCommonTableExpression()) { + // there are too many false positives + // common table expression + // folded query: WITH table AS VALUES () + SQLite3Values values = new SQLite3Values(auxiliaryQueryResult, temporaryTable.getColumns()); + originalQuery.updateWithClasureRight(values); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (Randomly.getBoolean() && this.testDerivedTable()) { + // derived table + // folded query: SELECT FROM () AS table + originalQuery.setWithClasure(null); + SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); + SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); + originalQuery.replaceFromTable(this.tempTableName, alias); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (this.testInsert()){ + // there are too many false positives + // folded query: CREATE the table and INSERT INTO table subquery + try { + this.createTemporaryTable(auxiliaryQuery, this.tempTableName); + originalQuery.setWithClasure(null); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable(this.tempTableName); + } + } else { + throw new IgnoreMeException(); + } + } + if (foldedResult == null || originalResult == null) { + throw new IgnoreMeException(); + } + if (foldedQueryString.equals(originalQueryString)) { + throw new IgnoreMeException(); + } + if (!compareResult(foldedResult, originalResult)) { + reproducer = null; // TODO + state.getState().getLocalState().log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError(auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); + } + } + + private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Expression specificCondition) { + SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); + if (tempTable != null) { + randomTables.addTable(tempTable); + } + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + for (SQLite3Table t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + randomTables.removeTable(t); + } + } + } + + List columns = randomTables.getColumns(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + List joinStatements = new ArrayList<>(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + joinStatements.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } + else if (Randomly.getBoolean()) { + joinStatements = genJoinExpression(gen, tables, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); + } + List tableRefs = SQLite3Common.getTableRefs(tables, s); + SQLite3Select select = new SQLite3Select(); + select.setFromList(tableRefs); + if (joinStatements.size() > 0) { + select.setJoinClauses(joinStatements); + } + + SQLite3Expression randomWhereCondition = gen.generateExpression(); + SQLite3Expression whereCondition = null; + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomWhereCondition, specificCondition, operator); + } else { + whereCondition = randomWhereCondition; + } + select.setWhereClause(whereCondition); + + if (Randomly.getBoolean()) { + select.setOrderByClauses(genOrderBysExpression(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + } + + if (Randomly.getBoolean()) { + List selectedColumns = Randomly.nonEmptySubset(columns); + List selectedAlias = new LinkedList<>(); + for (int i = 0; i < selectedColumns.size(); ++i) { + SQLite3ColumnName originalName = new SQLite3ColumnName(selectedColumns.get(i), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(i)), null); + SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); + selectedAlias.add(columnAlias); + } + select.setFetchColumns(selectedAlias); + } else { + SQLite3ColumnName aggr = new SQLite3ColumnName(Randomly.fromList(columns), null); + SQLite3Provider.mustKnowResult = true; + SQLite3Expression originalName = new SQLite3Aggregate(Arrays.asList(aggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c0"), null); + SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); + select.setFetchColumns(Arrays.asList(columnAlias)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List groupByClause = genGroupByClause(columns, specificCondition); + select.setGroupByClause(groupByClause); + if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + select.setHavingClause(genHavingClause(columns, specificCondition)); + } + } + } + return select; + } + + // For expression test + private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression specificCondition) { + SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); + if (tempTable != null) { + randomTables.addTable(tempTable); + } + if (!useSubqueryAsFoldedExpr) { + for (SQLite3Table t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + randomTables.removeTable(t); + } + } + } + + List columns = randomTables.getColumns(); + if (!useSubqueryAsFoldedExpr && this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + tablesFromOuterContext = randomTables.getTables(); + + if (joinsInExpr == null) { + if (Randomly.getBooleanWithRatherLowProbability()) { + joinsInExpr = genJoinExpression(gen, tables, null, true); + } else { + joinsInExpr = new ArrayList(); + } + } + + List tableRefs = SQLite3Common.getTableRefs(tables, s); + SQLite3Select select = new SQLite3Select(); + select.setFromList(tableRefs); + if (joinsInExpr != null && joinsInExpr.size() > 0) { + select.setJoinClauses(joinsInExpr); + } + + SQLite3Expression whereCondition = gen.generateExpression(); + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(whereCondition, specificCondition, operator); + } + this.foldedExpr = whereCondition; + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (SQLite3Column c : randomTables.getColumns()) { + SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); + fetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(whereCondition, aliasName); + fetchColumns.add(columnAlias); + + select.setFetchColumns(fetchColumns); + + originalQueryString = SQLite3Visitor.asString(select); + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the summary from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + List tempColumnList = new ArrayList<>(); + + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + SQLite3Alias cAlias = (SQLite3Alias) fetchColumns.get(i); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3Column column = cRef.getColumn(); + String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); + SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); + tempColumnList.add(newColumn); + } + List columnRef = new ArrayList<>(); + for (SQLite3Column c : randomTables.getColumns()) { + columnRef.add(new SQLite3ColumnName(c, null)); + } + if (tempColumnList.size() != queryRes.size()) { + throw new AssertionError(); + } + SQLite3Values values = new SQLite3Values(queryRes, tempColumnList); + this.constantResOfFoldedExpr = new SQLite3ResultMap(values, columnRef, summary, null); + + return select; + } + + private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable, SQLite3Expression specificCondition) { + // do not support join now + this.joinsInExpr = new ArrayList(); + + SQLite3Tables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); + SQLite3Tables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); + + if (selectedTable != null) { + innerQueryRandomTables.addTable(selectedTable); + } + + List innerQueryFromTables = new ArrayList<>(); + for (SQLite3Table t : innerQueryRandomTables.getTables()) { + if (!outerQueryRandomTables.isContained(t)) { + innerQueryFromTables.add(new SQLite3TableReference(t)); + } + } + for (SQLite3Table t : outerQueryRandomTables.getTables()) { + if (innerQueryRandomTables.isContained(t)) { + innerQueryRandomTables.removeTable(t); + + List newColumns = new ArrayList<>(); + for (SQLite3Column c : t.getColumns()) { + SQLite3Column newColumn = new SQLite3Column(c.getName(), c.getType(), false, null, false); + newColumns.add(newColumn); + } + SQLite3Table newTable = new SQLite3Table(t.getName() + "a", newColumns, null, true, false, false, false); + for (SQLite3Column c : newColumns) { + c.setTable(newTable); + } + innerQueryRandomTables.addTable(newTable); + + SQLite3Alias alias = new SQLite3Alias(new SQLite3TableReference(t), new SQLite3TableReference(newTable)); + innerQueryFromTables.add(alias); + } + } + + List innerQueryColumns = new ArrayList<>(); + innerQueryColumns.addAll(innerQueryRandomTables.getColumns()); + innerQueryColumns.addAll(outerQueryRandomTables.getColumns()); + gen = new SQLite3ExpressionGenerator(state).setColumns(innerQueryColumns); + + SQLite3Select innerQuery = new SQLite3Select(); + innerQuery.setFromList(innerQueryFromTables); + + SQLite3Expression innerQueryWhereCondition = gen.generateExpression(); + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + innerQueryWhereCondition = new SQLite3Expression.Sqlite3BinaryOperation(innerQueryWhereCondition, specificCondition, operator); + } + innerQuery.setWhereClause(innerQueryWhereCondition); + + // use aggregate function in fetch column + SQLite3ColumnName innerQueryAggr = new SQLite3ColumnName(Randomly.fromList(innerQueryRandomTables.getColumns()), null); + SQLite3Provider.mustKnowResult = true; + SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List groupByClause = genGroupByClause(innerQueryColumns, specificCondition); + innerQuery.setGroupByClause(groupByClause); + if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + innerQuery.setHavingClause(genHavingClause(innerQueryColumns, specificCondition)); + } + } + + this.foldedExpr = innerQuery; + + + // outer query + SQLite3Select outerQuery = new SQLite3Select(); + outerQuery.setFromList(SQLite3Common.getTableRefs(outerQueryRandomTables.getTables(), s)); + tablesFromOuterContext = outerQueryRandomTables.getTables(); + + List outerQueryFetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (SQLite3Column c : outerQueryRandomTables.getColumns()) { + SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); + outerQueryFetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(innerQuery, aliasName); + outerQueryFetchColumns.add(columnAlias); + + outerQuery.setFetchColumns(outerQueryFetchColumns); + + originalQueryString = SQLite3Visitor.asString(outerQuery); + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the summary from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + List tempColumnList = new ArrayList<>(); + + for (int i = 0; i < outerQueryFetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + SQLite3Alias cAlias = (SQLite3Alias) outerQueryFetchColumns.get(i); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3Column column = cRef.getColumn(); + String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); + SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); + tempColumnList.add(newColumn); + } + List columnRef = new ArrayList<>(); + for (SQLite3Column c : outerQueryRandomTables.getColumns()) { + columnRef.add(new SQLite3ColumnName(c, null)); + } + if (tempColumnList.size() != queryRes.size()) { + throw new AssertionError(); + } + SQLite3Values values = new SQLite3Values(queryRes, tempColumnList); + this.constantResOfFoldedExpr = new SQLite3ResultMap(values, columnRef, summary, null); + + return outerQuery; + } + + private List genJoinExpression(SQLite3ExpressionGenerator gen, List tables, SQLite3Expression specificCondition, boolean joinForExperssion) { + List joinStatements = new ArrayList<>(); + if (!state.getDbmsSpecificOptions().testJoins) { + return joinStatements; + } + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + if (Randomly.getBoolean() && tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1 || joinForExperssion) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + SQLite3Expression randomOnCondition = gen.generateExpression(); + SQLite3Expression onCondition = null; + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + onCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomOnCondition, specificCondition, operator); + } else { + onCondition = randomOnCondition; + } + + SQLite3Table table = Randomly.fromList(tables); + tables.remove(table); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + onCondition = null; + } + Join j = new SQLite3Expression.Join(table, onCondition, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } + + private List genOrderBysExpression(SQLite3ExpressionGenerator gen, SQLite3Expression specificCondition) { + List expressions = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + expressions.add(genOrderingTerm(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + } + return expressions; + } + + private SQLite3Expression genOrderingTerm(SQLite3ExpressionGenerator gen, SQLite3Expression specificCondition) { + SQLite3Expression expr = gen.generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + // COLLATE is potentially already generated + if (Randomly.getBoolean()) { + expr = new SQLite3OrderingTerm(expr, Ordering.getRandomValue()); + } + if (state.getDbmsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { + expr = new SQLite3PostfixText(expr, Randomly.fromOptions(" NULLS FIRST", " NULLS LAST"), + null /* expr.getExpectedValue() */) { + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + }; + } + return expr; + } + + private List genGroupByClause(List columns, SQLite3Expression specificCondition) { + errors.add("GROUP BY term out of range"); + if (Randomly.getBoolean()) { + List collect = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + SQLite3Expression expr = new SQLite3ExpressionGenerator(state).setColumns(columns).generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + collect.add(expr); + } + return collect; + } + return Collections.emptyList(); + } + + private SQLite3Expression genHavingClause(List columns, SQLite3Expression specificCondition) { + SQLite3Expression expr = new SQLite3ExpressionGenerator(state).setColumns(columns).generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + return expr; + } + + private Map> getQueryResult(String queryString, SQLite3GlobalState state) throws SQLException { + Map> result = new LinkedHashMap<>(); + if (options.logEachSelect()) { + logger.writeCurrentNoLineBreak(queryString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + stmt.setQueryTimeout(600); + ResultSet rs = null; + try { + rs = stmt.executeQuery(queryString); + ResultSetMetaData metaData = rs.getMetaData(); + Integer columnCount = metaData.getColumnCount(); + Map idxNameMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + result.put("c" + String.valueOf(i-1), new ArrayList<>()); + idxNameMap.put(i, "c" + String.valueOf(i-1)); + } + + int resultRows = 0; + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + try { + Object value = rs.getObject(i); + SQLite3Constant constant; + if (rs.wasNull()) { + constant = SQLite3Constant.createNullConstant(); + } + + else if (value instanceof Integer) { + constant = SQLite3Constant.createIntConstant(Long.valueOf((Integer) value)); + } else if (value instanceof Short) { + constant = SQLite3Constant.createIntConstant(Long.valueOf((Short) value)); + } else if (value instanceof Long) { + constant = SQLite3Constant.createIntConstant((Long) value); + } + + else if (value instanceof Double) { + constant = SQLite3Constant.createRealConstant((double) value); + } else if (value instanceof Float) { + constant = SQLite3Constant.createRealConstant(((Float) value).doubleValue()); + } else if (value instanceof BigDecimal) { + constant = SQLite3Constant.createRealConstant(((BigDecimal) value).doubleValue()); + } + + else if (value instanceof Byte) { + constant = SQLite3Constant.createBinaryConstant((byte[]) value); + } else if (value instanceof byte[]) { + constant = SQLite3Constant.createBinaryConstant((byte[]) value); + } else if (value instanceof Boolean) { + constant = SQLite3Constant.createBoolean((boolean) value); + } else if (value instanceof String) { + constant = SQLite3Constant.createTextConstant((String) value); + } else if (value == null) { + constant = SQLite3Constant.createNullConstant(); + } else { + throw new IgnoreMeException(); + } + List v = result.get(idxNameMap.get(i)); + v.add(constant); + } catch (SQLException e) { + System.out.println(e.getMessage()); + throw new IgnoreMeException(); + } + } + ++resultRows; + if (resultRows > 100) { + throw new IgnoreMeException(); + } + } + Main.nrSuccessfulActions.addAndGet(1); + rs.close(); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + if (rs != null) { + rs.close(); + } + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + return result; + } + + private SQLite3Table genTemporaryTable(SQLite3Select select, String tableName) { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); + databaseColumns.add(column); + } + SQLite3Table table = new SQLite3Table(tableName, databaseColumns, null, false, false, false, false); + for (SQLite3Column c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName) throws SQLException { + String selectString = SQLite3Visitor.asString(select); + Map idxTypeMap = getColumnTypeFromSelect(select); + + Integer columnNumber = idxTypeMap.size(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE " + tableName + " ("); + for (int i = 0; i < columnNumber; ++i) { + String columnTypeName = ""; + if (idxTypeMap.get(i) != null) { + switch (idxTypeMap.get(i)) { + case INT: + case TEXT: + case REAL: + columnTypeName = idxTypeMap.get(i).name(); + break; + case BINARY: + columnTypeName = ""; + break; + default: + columnTypeName = ""; + } + } + sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(");"); + String crateTableString = sb.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(crateTableString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(crateTableString); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + StringBuilder sb2 = new StringBuilder(); + sb2.append("INSERT INTO " + tableName + " "+ selectString); + String insertValueString = sb2.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(insertValueString); + } + stmt = null; + try { + stmt = this.con.createStatement(); + try { + Main.nrSuccessfulActions.addAndGet(1); + stmt.execute(insertValueString); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); + databaseColumns.add(column); + } + SQLite3Table table = new SQLite3Table(tableName, databaseColumns, null, false, false, false, false); + for (SQLite3Column c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private void dropTemporaryTable(String tableName) throws SQLException { + String dropString = "DROP TABLE " + tableName + ";"; + if (options.logEachSelect()) { + logger.writeCurrent(dropString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(dropString); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + } + + private boolean compareResult(Map> r1, Map> r2) { + if (r1.size() != r2.size()) { + return false; + } + for (Map.Entry < String, List > entry: r1.entrySet()) { + String currentKey = entry.getKey(); + if (!r2.containsKey(currentKey)) { + return false; + } + List v1= entry.getValue(); + List v2= r2.get(currentKey); + if (v1.size() != v2.size()) { + return false; + } + List v1Value = new ArrayList<>(v1.stream().map(c -> c.toString()).collect(Collectors.toList())); + List v2Value = new ArrayList<>(v2.stream().map(c -> c.toString()).collect(Collectors.toList())); + Collections.sort(v1Value); + Collections.sort(v2Value); + if (!v1Value.equals(v2Value)) { + return false; + } + } + return true; + } + + private Map getColumnTypeFromSelect(SQLite3Select select) { + List fetchColumns = select.getFetchColumns(); + List newFetchColumns = new ArrayList<>(); + for(SQLite3Expression column : fetchColumns) { + newFetchColumns.add(column); + SQLite3Alias columnAlias = (SQLite3Alias) column; + SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOrigonalExpression()); + newFetchColumns.add(typeofColumn); + } + SQLite3Select newSelect = new SQLite3Select(select); + newSelect.setFetchColumns(newFetchColumns); + Map> typeResult = null; + try { + typeResult = getQueryResult(SQLite3Visitor.asString(newSelect), state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + + if (typeResult == null) { + throw new IgnoreMeException(); + } + Map idxTypeMap = new HashMap<>(); + for (int i = 0; i * 2 < typeResult.size(); ++i) { + String columnName = "c" + String.valueOf(i * 2 + 1); + SQLite3Expression t = typeResult.get(columnName).get(0); + SQLite3TextConstant tString = (SQLite3TextConstant) t; + String typeName = tString.asString(); + SQLite3DataType cType = SQLite3DataType.getTypeFromName(typeName); + idxTypeMap.put(i, cType); + } + + return idxTypeMap; + } + + public boolean useSubquery() { + if (this.state.getDbmsSpecificOptions().coddTestModel.equals("random")) { + return Randomly.getBoolean(); + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("expression")) { + return false; + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("subquery")) { + return true; + } else { + System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); + System.exit(1); + return false; + } + } + + public boolean useCorrelatedSubquery() { + return Randomly.getBoolean(); + } + + public boolean testCommonTableExpression() { + return false; + } + public boolean testDerivedTable() { + return true; + } + public boolean testInsert() { + return false; + } + + @Override + public String getLastQueryString() { + return originalQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } +} diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index 87d91f451..8a343fed2 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -1,6 +1,23 @@ package sqlancer.sqlite3.schema; +import sqlancer.IgnoreMeException; + public enum SQLite3DataType { NULL, INT, TEXT, REAL, NONE, BINARY; + public static SQLite3DataType getTypeFromName(String name) { + if (name.equals("integer")) { + return INT; + } else if (name.equals("real")) { + return REAL; + } else if (name.equals("text")) { + return TEXT; + } else if (name.equals("blob")) { + return NONE; + } else if (name.equals("null")) { + return NULL; + } else { + throw new IgnoreMeException(); + } + } } From 2f0c1eef72f0d4de9661909362eeaa9a5b6057d2 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sun, 5 Jan 2025 21:09:59 +0800 Subject: [PATCH 1077/1284] fix a typo and remove some redundant code of CODDTest for sqlite3 --- .../sqlite3/SQLite3ExpectedValueVisitor.java | 2 +- .../sqlite3/SQLite3ToStringVisitor.java | 2 +- .../sqlite3/ast/SQLite3Expression.java | 10 +-- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 69 ++++--------------- 4 files changed, 22 insertions(+), 61 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 40f8ae20a..246105a5e 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -313,7 +313,7 @@ public void visit(SQLite3SetClause set) { @Override public void visit(SQLite3Alias alias) { print(alias); - print(alias.getOrigonalExpression()); + print(alias.getOriginalExpression()); print(alias.getAliasExpression()); } diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 44aa0d1e0..2ec206e11 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -512,7 +512,7 @@ public void visit(SQLite3SetClause set) { @Override public void visit(SQLite3Alias alias) { sb.append("("); - visit(alias.getOrigonalExpression()); + visit(alias.getOriginalExpression()); sb.append(")"); sb.append(" AS "); visit(alias.getAliasExpression()); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index a14432eec..6c4b2f9ff 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1593,11 +1593,11 @@ public SQLite3CollateSequence getExplicitCollateSequence() { public static class SQLite3Alias extends SQLite3Expression { - private SQLite3Expression origonalExpression; + private SQLite3Expression originalExpression; private SQLite3Expression aliasExpression; - public SQLite3Alias(SQLite3Expression origonalExpression, SQLite3Expression aliasExpression) { - this.origonalExpression = origonalExpression; + public SQLite3Alias(SQLite3Expression originalExpression, SQLite3Expression aliasExpression) { + this.originalExpression = originalExpression; this.aliasExpression = aliasExpression; } @@ -1606,8 +1606,8 @@ public SQLite3CollateSequence getExplicitCollateSequence() { return null; } - public SQLite3Expression getOrigonalExpression() { - return origonalExpression; + public SQLite3Expression getOriginalExpression() { + return originalExpression; } public SQLite3Expression getAliasExpression() { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index d20e6e9be..7118f4081 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -100,7 +100,7 @@ public void check() throws SQLException { SQLite3Select auxiliaryQuery = null; if (useSubqueryAsFoldedExpr) { if (useCorrelatedSubqueryAsFoldedExpr) { - auxiliaryQuery = genSelectWithCorrelatedSubquery(null, null); + auxiliaryQuery = genSelectWithCorrelatedSubquery(); auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); auxiliaryQueryResult.putAll(selectResult); @@ -110,7 +110,7 @@ public void check() throws SQLException { auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); } } else { - auxiliaryQuery = genSimpleSelect(null, null); + auxiliaryQuery = genSimpleSelect(); auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); auxiliaryQueryResult.putAll(selectResult); @@ -334,40 +334,18 @@ else if (Randomly.getBoolean()) { } // For expression test - private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression specificCondition) { + private SQLite3Select genSimpleSelect() { SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); - if (tempTable != null) { - randomTables.addTable(tempTable); - } - if (!useSubqueryAsFoldedExpr) { - for (SQLite3Table t : this.tablesFromOuterContext) { - randomTables.addTable(t); - } - if (this.joinsInExpr != null) { - for (Join j : this.joinsInExpr) { - SQLite3Table t = j.getTable(); - randomTables.removeTable(t); - } - } - } - List columns = randomTables.getColumns(); - if (!useSubqueryAsFoldedExpr && this.joinsInExpr != null) { - for (Join j : this.joinsInExpr) { - SQLite3Table t = j.getTable(); - columns.addAll(t.getColumns()); - } - } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); List tables = randomTables.getTables(); tablesFromOuterContext = randomTables.getTables(); - if (joinsInExpr == null) { - if (Randomly.getBooleanWithRatherLowProbability()) { - joinsInExpr = genJoinExpression(gen, tables, null, true); - } else { - joinsInExpr = new ArrayList(); - } + if (Randomly.getBooleanWithRatherLowProbability()) { + joinsInExpr = genJoinExpression(gen, tables, null, true); + } else { + joinsInExpr = new ArrayList(); } List tableRefs = SQLite3Common.getTableRefs(tables, s); @@ -378,10 +356,6 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression } SQLite3Expression whereCondition = gen.generateExpression(); - if (specificCondition != null) { - BinaryOperator operator = BinaryOperator.getRandomOperator(); - whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(whereCondition, specificCondition, operator); - } this.foldedExpr = whereCondition; List fetchColumns = new ArrayList<>(); @@ -401,11 +375,9 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression select.setFetchColumns(fetchColumns); - originalQueryString = SQLite3Visitor.asString(select); - Map> queryRes = null; try { - queryRes = getQueryResult(originalQueryString, state); + queryRes = getQueryResult(SQLite3Visitor.asString(select), state); } catch (SQLException e) { if (errors.errorIsExpected(e.getMessage())) { throw new IgnoreMeException(); @@ -429,7 +401,7 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression for (int i = 0; i < fetchColumns.size() - 1; ++i) { // do not put the last fetch column to values SQLite3Alias cAlias = (SQLite3Alias) fetchColumns.get(i); - SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOriginalExpression(); SQLite3Column column = cRef.getColumn(); String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); @@ -448,17 +420,10 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression return select; } - private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable, SQLite3Expression specificCondition) { - // do not support join now - this.joinsInExpr = new ArrayList(); - + private SQLite3Select genSelectWithCorrelatedSubquery() { SQLite3Tables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); SQLite3Tables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); - if (selectedTable != null) { - innerQueryRandomTables.addTable(selectedTable); - } - List innerQueryFromTables = new ArrayList<>(); for (SQLite3Table t : innerQueryRandomTables.getTables()) { if (!outerQueryRandomTables.isContained(t)) { @@ -494,10 +459,6 @@ private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable innerQuery.setFromList(innerQueryFromTables); SQLite3Expression innerQueryWhereCondition = gen.generateExpression(); - if (specificCondition != null) { - BinaryOperator operator = BinaryOperator.getRandomOperator(); - innerQueryWhereCondition = new SQLite3Expression.Sqlite3BinaryOperation(innerQueryWhereCondition, specificCondition, operator); - } innerQuery.setWhereClause(innerQueryWhereCondition); // use aggregate function in fetch column @@ -506,10 +467,10 @@ private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); if (Randomly.getBooleanWithRatherLowProbability()) { - List groupByClause = genGroupByClause(innerQueryColumns, specificCondition); + List groupByClause = genGroupByClause(innerQueryColumns, null); innerQuery.setGroupByClause(groupByClause); if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { - innerQuery.setHavingClause(genHavingClause(innerQueryColumns, specificCondition)); + innerQuery.setHavingClause(genHavingClause(innerQueryColumns, null)); } } @@ -566,7 +527,7 @@ private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable for (int i = 0; i < outerQueryFetchColumns.size() - 1; ++i) { // do not put the last fetch column to values SQLite3Alias cAlias = (SQLite3Alias) outerQueryFetchColumns.get(i); - SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOriginalExpression(); SQLite3Column column = cRef.getColumn(); String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); @@ -930,7 +891,7 @@ private Map getColumnTypeFromSelect(SQLite3Select sele for(SQLite3Expression column : fetchColumns) { newFetchColumns.add(column); SQLite3Alias columnAlias = (SQLite3Alias) column; - SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOrigonalExpression()); + SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOriginalExpression()); newFetchColumns.add(typeofColumn); } SQLite3Select newSelect = new SQLite3Select(select); From b8dbd9de91c5b8dbe2e2ca5b248b3bc59393ea84 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Tue, 14 Jan 2025 16:03:00 +0800 Subject: [PATCH 1078/1284] fix the false positive in CODDTest for SQLite --- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 7118f4081..c4e8d892a 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -94,6 +94,9 @@ public SQLite3CODDTestOracle(SQLite3GlobalState globalState) { public void check() throws SQLException { reproducer = null; + joinsInExpr = null; + tablesFromOuterContext.clear(); + useSubqueryAsFoldedExpr = useSubquery(); useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); @@ -168,8 +171,7 @@ else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQ foldedResult = getQueryResult(foldedQueryString, state); } // one column - else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability()) { - // else if (auxiliaryQueryResult.size() == 1 && false) { + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability() && testInOperator()) { // original query List columns = s.getRandomTableNonEmptyTables().getColumns(); SQLite3ColumnName selectedColumn = new SQLite3ColumnName(Randomly.fromList(columns), null); @@ -277,9 +279,11 @@ private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Express gen = new SQLite3ExpressionGenerator(state).setColumns(columns); List tables = randomTables.getTables(); List joinStatements = new ArrayList<>(); - if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { - joinStatements.addAll(this.joinsInExpr); - this.joinsInExpr = null; + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (this.joinsInExpr != null) { + joinStatements.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } } else if (Randomly.getBoolean()) { joinStatements = genJoinExpression(gen, tables, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); @@ -950,6 +954,9 @@ public boolean testDerivedTable() { public boolean testInsert() { return false; } + public boolean testInOperator() { + return false; + } @Override public String getLastQueryString() { From d9579b845b0469b8644b445c7b9fed27053f65eb Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sat, 22 Mar 2025 15:14:35 +0800 Subject: [PATCH 1079/1284] fix a typo --- .settings/org.eclipse.jdt.core.prefs | 9 ++++++--- .../sqlite3/SQLite3ExpectedValueVisitor.java | 10 +++++----- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 12 ++++++------ src/sqlancer/sqlite3/SQLite3Visitor.java | 8 ++++---- src/sqlancer/sqlite3/ast/SQLite3Expression.java | 4 ++-- src/sqlancer/sqlite3/ast/SQLite3Select.java | 12 ++++++------ .../sqlite3/oracle/SQLite3CODDTestOracle.java | 12 ++++++------ 7 files changed, 35 insertions(+), 32 deletions(-) diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 0a659c1a7..3af1c5ad8 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.compliance=11 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,6 +28,7 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -73,6 +74,7 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -110,5 +112,6 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=1.8 +org.eclipse.jdt.core.compiler.source=11 diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 246105a5e..594fe5c92 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -28,7 +28,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -318,10 +318,10 @@ public void visit(SQLite3Alias alias) { } @Override - public void visit(SQLite3WithClasure withClasure) { - print(withClasure); - print(withClasure.getLeft()); - print(withClasure.getRight()); + public void visit(SQLite3WithClause withClause) { + print(withClause); + print(withClause.getLeft()); + print(withClause.getRight()); } @Override diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 2ec206e11..202540eb6 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -36,7 +36,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; @@ -112,8 +112,8 @@ public void visit(SQLite3Select s, boolean inner) { if (inner) { sb.append("("); } - if (s.getWithClasure() != null) { - visit(s.getWithClasure()); + if (s.getWithClause() != null) { + visit(s.getWithClause()); sb.append(" "); } sb.append("SELECT "); @@ -519,11 +519,11 @@ public void visit(SQLite3Alias alias) { } @Override - public void visit(SQLite3WithClasure withClasure) { + public void visit(SQLite3WithClause withClause) { sb.append("WITH "); - visit(withClasure.getLeft()); + visit(withClause.getLeft()); sb.append(" AS "); - visit(withClasure.getRight()); + visit(withClause.getRight()); } @Override diff --git a/src/sqlancer/sqlite3/SQLite3Visitor.java b/src/sqlancer/sqlite3/SQLite3Visitor.java index d5ebce096..5e1e5dbf6 100644 --- a/src/sqlancer/sqlite3/SQLite3Visitor.java +++ b/src/sqlancer/sqlite3/SQLite3Visitor.java @@ -27,7 +27,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -140,7 +140,7 @@ default void visit(SQLite3PostfixUnaryOperation exp) { void visit(SQLite3Alias alias); - void visit(SQLite3WithClasure withClasure); + void visit(SQLite3WithClause withClause); void visit(SQLite3TableAndColumnRef tableAndColumnRef); @@ -217,8 +217,8 @@ default void visit(SQLite3Expression expr) { visit((SQLite3SetClause) expr); } else if (expr instanceof SQLite3Alias) { visit((SQLite3Alias) expr); - } else if (expr instanceof SQLite3WithClasure) { - visit((SQLite3WithClasure) expr); + } else if (expr instanceof SQLite3WithClause) { + visit((SQLite3WithClause) expr); } else if (expr instanceof SQLite3TableAndColumnRef) { visit((SQLite3TableAndColumnRef) expr); } else if (expr instanceof SQLite3Values) { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 6c4b2f9ff..12e5c71f1 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1563,12 +1563,12 @@ public boolean omitBracketsWhenPrinting() { } } - public static class SQLite3WithClasure extends SQLite3Expression { + public static class SQLite3WithClause extends SQLite3Expression { private SQLite3Expression left; private SQLite3Expression right; - public SQLite3WithClasure(SQLite3Expression left, SQLite3Expression right) { + public SQLite3WithClause(SQLite3Expression left, SQLite3Expression right) { this.left = left; this.right = right; } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 042aab856..6f1571df0 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -25,7 +25,7 @@ public class SQLite3Select extends SQLite3Expression private List fetchColumns = Collections.emptyList(); private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; - private SQLite3WithClasure withClause = null; + private SQLite3WithClause withClause = null; public SQLite3Select() { } @@ -165,15 +165,15 @@ public String asString() { return SQLite3Visitor.asString(this); } - public void setWithClasure(SQLite3WithClasure withClasure) { - this.withClause = withClasure; + public void setWithClause(SQLite3WithClause withClause) { + this.withClause = withClause; } - public void updateWithClasureRight(SQLite3Expression withClasureRight) { - this.withClause.updateRight(withClasureRight); + public void updateWithClauseRight(SQLite3Expression withClauseRight) { + this.withClause.updateRight(withClauseRight); } - public SQLite3Expression getWithClasure() { + public SQLite3Expression getWithClause() { return this.withClause; } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index c4e8d892a..17444026a 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -41,7 +41,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; @@ -202,8 +202,8 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); originalQuery = this.genSelectExpression(temporaryTable, null); SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); - SQLite3WithClasure withClasure = new SQLite3WithClasure(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); - originalQuery.setWithClasure(withClasure); + SQLite3WithClause withClause = new SQLite3WithClause(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); + originalQuery.setWithClause(withClause); originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); // folded query @@ -212,13 +212,13 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro // common table expression // folded query: WITH table AS VALUES () SQLite3Values values = new SQLite3Values(auxiliaryQueryResult, temporaryTable.getColumns()); - originalQuery.updateWithClasureRight(values); + originalQuery.updateWithClauseRight(values); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } else if (Randomly.getBoolean() && this.testDerivedTable()) { // derived table // folded query: SELECT FROM () AS table - originalQuery.setWithClasure(null); + originalQuery.setWithClause(null); SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); originalQuery.replaceFromTable(this.tempTableName, alias); @@ -229,7 +229,7 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro // folded query: CREATE the table and INSERT INTO table subquery try { this.createTemporaryTable(auxiliaryQuery, this.tempTableName); - originalQuery.setWithClasure(null); + originalQuery.setWithClause(null); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } finally { From 9a3a2e9bd8077451df53849d85217be65da44b3f Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sat, 22 Mar 2025 15:16:11 +0800 Subject: [PATCH 1080/1284] add comments for SQLite3ExpressionBag --- src/sqlancer/sqlite3/ast/SQLite3Expression.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 12e5c71f1..e5f1e2064 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1671,6 +1671,9 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } } + // The ExpressionBag is not a built-in SQL feature, + // but rather a utility class used in CODDTest's oracle construction + // to substitute expressions with their corresponding constant values. public static class SQLite3ExpressionBag extends SQLite3Expression { private SQLite3Expression innerExpr; From 9ead27ac349f81df77351fe78c371f1e253e0f30 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sat, 22 Mar 2025 15:19:40 +0800 Subject: [PATCH 1081/1284] remove deleteCharAt to simplify code logic --- .../sqlite3/SQLite3ToStringVisitor.java | 38 +++++++++---------- .../sqlite3/ast/SQLite3Expression.java | 11 ++++-- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 9 +++-- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 202540eb6..987d293e8 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -156,7 +156,7 @@ public void visit(SQLite3Select s, boolean inner) { visit(whereClause); sb.append(")"); } - if (!s.getGroupByClause().isEmpty()) { + if (s.getGroupByClause().size() > 0) { sb.append(" "); sb.append("GROUP BY "); visit(s.getGroupByClause()); @@ -539,7 +539,11 @@ public void visit(SQLite3Values values) { sb.append("(VALUES "); for (int i = 0; i < size; i++) { sb.append("("); + Boolean isFirstColumn = true; for (String name : columnNames) { + if (!isFirstColumn) { + sb.append(", "); + } if (vs.get(name).get(i).getDataType() == SQLite3DataType.NULL) { visit(vs.get(name).get(i)); } else { @@ -563,15 +567,13 @@ public void visit(SQLite3Values values) { throw new IgnoreMeException(); } } - + isFirstColumn = true; + } + sb.append(")"); + if (i < size - 1) { sb.append(", "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("), "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); } @@ -600,13 +602,11 @@ public void visit(SQLite3ResultMap tableSummary) { sb.append("("); for (int j = 0; j < columnRefs.size(); ++j) { visit(columnRefs.get(j)); - sb.append(" IS NULL AND "); + sb.append(" IS NULL"); + if (j < columnRefs.size() - 1) { + sb.append(" AND "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); return; } @@ -622,16 +622,14 @@ public void visit(SQLite3ResultMap tableSummary) { sb.append(" = "); sb.append(vs.get(columnNames.get(j)).get(i).toString()); } - sb.append(" AND "); + if (j < columnNames.size() - 1) { + sb.append(" AND "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("THEN "); + sb.append(" THEN "); visit(summary.get(i)); sb.append(" "); } sb.append("END "); } -} +} \ No newline at end of file diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index e5f1e2064..681566c03 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1631,12 +1631,14 @@ public String getString() { StringBuilder sb = new StringBuilder(); sb.append(table.getName()); sb.append("("); + Boolean isFirstColumn = true; for (SQLite3Column c : this.table.getColumns()) { + if (!isFirstColumn) { + sb.append(", "); + } sb.append(c.getName()); - sb.append(", "); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); return sb.toString(); } @@ -1671,6 +1673,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } } + // The ExpressionBag is not a built-in SQL feature, // but rather a utility class used in CODDTest's oracle construction // to substitute expressions with their corresponding constant values. @@ -1754,4 +1757,4 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } } -} +} \ No newline at end of file diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 17444026a..72d1b6072 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -781,10 +781,11 @@ private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName columnTypeName = ""; } } - sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + sb.append("c" + String.valueOf(i) + " " + columnTypeName); + if (i < columnNumber - 1) { + sb.append(", "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(");"); String crateTableString = sb.toString(); if (options.logEachSelect()) { @@ -967,4 +968,4 @@ public String getLastQueryString() { public Reproducer getLastReproducer() { return reproducer; } -} +} \ No newline at end of file From 967852ad31afcb697deb9808fe405960a49d540c Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Sun, 23 Mar 2025 07:03:25 +0000 Subject: [PATCH 1082/1284] Improve tests for schema classes --- test/sqlancer/TestCommonSchema.java | 34 +++++++++++++---------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/test/sqlancer/TestCommonSchema.java b/test/sqlancer/TestCommonSchema.java index 764e94ea4..80a57dd8c 100644 --- a/test/sqlancer/TestCommonSchema.java +++ b/test/sqlancer/TestCommonSchema.java @@ -42,8 +42,7 @@ static class TestIndex extends TableIndex { } private TestTable createTestTable(String name, List indexes, boolean isView, String... columns) { - List cols = Arrays.stream(columns) - .map(col -> new TestTableColumn(col, null, "VARCHAR")) + List cols = Arrays.stream(columns).map(col -> new TestTableColumn(col, null, "VARCHAR")) .collect(Collectors.toList()); return new TestTable(name, cols, indexes, isView); } @@ -59,11 +58,14 @@ private TestSchema createTestSchema(TestTable... tables) { @Test void testColumnManagement() { TestTable table = createTestTable("products", Collections.emptyList(), false, "sku", "price"); + List columnNames = table.getColumns().stream().map(TestTableColumn::getName) + .collect(Collectors.toList()); + List columnTypes = table.getColumns().stream().map(TestTableColumn::getType) + .collect(Collectors.toList()); TestTableColumn randomCol = table.getRandomColumn(); - assertEquals("sku", table.getColumns().get(0).getName()); - assertEquals("VARCHAR", table.getColumns().get(1).getType()); - + assertTrue(columnNames.containsAll(Set.of("sku", "price"))); + assertTrue(columnTypes.containsAll(Set.of("VARCHAR"))); assertTrue(table.getColumns().contains(randomCol)); } @@ -81,16 +83,14 @@ void testIndexManagement() { @Test void testViewManagement() { - TestTable view1 = createTestTable("v1", Collections.emptyList(),true, "col1"); - TestTable view2 = createTestTable("v2", Collections.emptyList(),true, "col2"); + TestTable view1 = createTestTable("v1", Collections.emptyList(), true, "col1"); + TestTable view2 = createTestTable("v2", Collections.emptyList(), true, "col2"); TestTable table = createTestTable("t1", Collections.emptyList(), false, "col3"); TestSchema schema = createTestSchema(view1, view2, table); - assertAll( - () -> assertEquals(2, schema.getViews().size(), "Should detect 2 views"), + assertAll(() -> assertEquals(2, schema.getViews().size(), "Should detect 2 views"), () -> assertEquals(1, schema.getDatabaseTablesWithoutViews().size(), "Should detect 1 normal table"), - () -> assertEquals("t1", schema.getDatabaseTablesWithoutViews().get(0).getName()) - ); + () -> assertEquals("t1", schema.getDatabaseTablesWithoutViews().get(0).getName())); } @Test @@ -117,12 +117,10 @@ void testObjectComparison() { TestTableColumn colA2 = new TestTableColumn("y", tableA, "INT"); TestTableColumn colB1 = new TestTableColumn("b", tableB, "TEXT"); - assertAll( - () -> assertTrue(colA1.compareTo(colA2) < 0, "Columns should be ordered by name"), + assertAll(() -> assertTrue(colA1.compareTo(colA2) < 0, "Columns should be ordered by name"), () -> assertTrue(colA1.compareTo(colB1) < 0, "Columns should be ordered by name"), () -> assertTrue(tableA.compareTo(tableB) > 0, "Tables should be ordered reverse-alphabetically"), - () -> assertEquals(0, tableA.compareTo(tableA), "Same table should be equal") - ); + () -> assertEquals(0, tableA.compareTo(tableA), "Same table should be equal")); } @Test @@ -135,12 +133,10 @@ void testEquality() { TestTableColumn col3 = new TestTableColumn("id", table2, "INT"); TestTableColumn col4 = new TestTableColumn("name", table1, "TEXT"); - assertAll( - () -> assertEquals(col1, col2, "Same table/column should be equal"), + assertAll(() -> assertEquals(col1, col2, "Same table/column should be equal"), () -> assertNotEquals(col1, col3, "Different tables should not be equal"), () -> assertNotEquals(col1, col4, "Different columns should not be equal"), - () -> assertNotEquals(col1, "invalid_object", "Different types should not be equal") - ); + () -> assertNotEquals(col1, "invalid_object", "Different types should not be equal")); } @Test From 9a4999434d805a3102917dcdf2a24a68d7fa21fc Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sat, 5 Apr 2025 17:08:17 +0800 Subject: [PATCH 1083/1284] [MySQL] Support generating CASE operators --- .../mysql/MySQLExpectedValueVisitor.java | 23 +++++++++++++ src/sqlancer/mysql/MySQLToStringVisitor.java | 27 +++++++++++++++ src/sqlancer/mysql/MySQLVisitor.java | 5 +++ src/sqlancer/mysql/ast/MySQLCaseOperator.java | 13 ++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 7 +++- .../mysql/MySQLExpectedValueVisitorTest.java | 33 +++++++++++++++++++ .../mysql/MySQLToStringVisitorTest.java | 27 +++++++++++++++ 7 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/mysql/ast/MySQLCaseOperator.java create mode 100644 test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index c9db08306..b0159fb27 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -1,11 +1,14 @@ package sqlancer.mysql; +import java.util.List; + import sqlancer.IgnoreMeException; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLCollate; import sqlancer.mysql.ast.MySQLColumnReference; @@ -172,4 +175,24 @@ public void visit(MySQLAggregate aggr) { // do nothing } + @Override + public void visit(MySQLCaseOperator caseOp) { + print(caseOp); + + if (caseOp.getSwitchCondition() != null) { + visit(caseOp.getSwitchCondition()); + } + + List whenConditions = caseOp.getConditions(); + List thenExpressions = caseOp.getExpressions(); + + for (int i = 0; i < whenConditions.size(); i++) { + visit(whenConditions.get(i)); + visit(thenExpressions.get(i)); + } + + if (caseOp.getElseExpr() != null) { + visit(caseOp.getElseExpr()); + } + } } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index b459fe81e..be82d45b5 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -11,6 +11,7 @@ import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLCollate; import sqlancer.mysql.ast.MySQLColumnReference; @@ -345,4 +346,30 @@ public void visit(MySQLAggregate aggr) { } sb.append(")"); } + + @Override + public void visit(MySQLCaseOperator caseOp) { + sb.append("(CASE "); + + if (caseOp.getSwitchCondition() != null) { + visit(caseOp.getSwitchCondition()); + sb.append(" "); + } + + for (int i = 0; i < caseOp.getConditions().size(); i++) { + if (i > 0) { + sb.append(" "); + } + sb.append("WHEN "); + visit(caseOp.getConditions().get(i)); + sb.append(" THEN "); + visit(caseOp.getExpressions().get(i)); + } + + if (caseOp.getElseExpr() != null) { + sb.append(" ELSE "); + visit(caseOp.getElseExpr()); + } + sb.append(" END)"); + } } diff --git a/src/sqlancer/mysql/MySQLVisitor.java b/src/sqlancer/mysql/MySQLVisitor.java index e413f7668..12c93ecfc 100644 --- a/src/sqlancer/mysql/MySQLVisitor.java +++ b/src/sqlancer/mysql/MySQLVisitor.java @@ -5,6 +5,7 @@ import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLCollate; import sqlancer.mysql.ast.MySQLColumnReference; @@ -61,6 +62,8 @@ public interface MySQLVisitor { void visit(MySQLAggregate aggregate); + void visit(MySQLCaseOperator caseOp); + default void visit(MySQLExpression expr) { if (expr instanceof MySQLConstant) { visit((MySQLConstant) expr); @@ -100,6 +103,8 @@ default void visit(MySQLExpression expr) { visit((MySQLText) expr); } else if (expr instanceof MySQLAggregate) { visit((MySQLAggregate) expr); + } else if (expr instanceof MySQLCaseOperator) { + visit((MySQLCaseOperator) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/mysql/ast/MySQLCaseOperator.java b/src/sqlancer/mysql/ast/MySQLCaseOperator.java new file mode 100644 index 000000000..c648be5ec --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLCaseOperator.java @@ -0,0 +1,13 @@ +package sqlancer.mysql.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class MySQLCaseOperator extends NewCaseOperatorNode implements MySQLExpression { + + public MySQLCaseOperator(MySQLExpression switchCondition, List whenExprs, + List thenExprs, MySQLExpression elseExpr) { + super(switchCondition, whenExprs, thenExprs, elseExpr); + } +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index f23c48c29..1ce413db4 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -25,6 +25,7 @@ import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLBinaryOperation.MySQLBinaryOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLColumnReference; @@ -64,7 +65,7 @@ public MySQLExpressionGenerator setRowVal(MySQLRowValue rowVal) { private enum Actions { COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, - BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, BINARY_OPERATION, EXISTS, BETWEEN_OPERATOR; + BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, BINARY_OPERATION, EXISTS, BETWEEN_OPERATOR, CASE_OPERATOR; } @Override @@ -117,6 +118,10 @@ public MySQLExpression generateExpression(int depth) { } return new MySQLBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1)); + case CASE_OPERATOR: + int nr = Randomly.smallNumber() + 1; + return new MySQLCaseOperator(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); default: throw new AssertionError(); } diff --git a/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java b/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java new file mode 100644 index 000000000..05cd273b6 --- /dev/null +++ b/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java @@ -0,0 +1,33 @@ +package sqlancer.mysql; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; +import sqlancer.mysql.ast.MySQLExpression; + +public class MySQLExpectedValueVisitorTest { + + @Test + void testCaseOp() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, null); + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1)); + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11)); + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + MySQLExpectedValueVisitor visitor = new MySQLExpectedValueVisitor(); + + AssertionError err = assertThrows(AssertionError.class, + () -> visitor.visit(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr))); + + assertTrue(err.getMessage().contains("PQS not supported")); + } +} diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 5aa9b4e14..169aaa57b 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -7,7 +7,11 @@ import org.junit.jupiter.api.Test; import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; public class MySQLToStringVisitorTest { @@ -50,4 +54,27 @@ void visitAggregateWithDistinctToString() { MySQLAggregate.MySQLAggregateFunction.MAX_DISTINCT); assertEquals("MAX(DISTINCT a)", MySQLVisitor.asString(aggrMaxDistinct)); } + + @Test + void visitCaseWhenToString() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, null); + + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + assertEquals("(CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 0 END)", + MySQLVisitor.asString(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr))); + assertEquals("(CASE WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 0 END)", + MySQLVisitor.asString(new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr))); + assertEquals("(CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 END)", + MySQLVisitor.asString(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null))); + assertEquals("(CASE WHEN 1 THEN 11 WHEN 2 THEN 22 END)", + MySQLVisitor.asString(new MySQLCaseOperator(null, whenExprs, thenExprs, null))); + } } From 72311cbbb3be765829d98a7b766b11c74d8ae5f6 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sat, 5 Apr 2025 17:57:45 +0800 Subject: [PATCH 1084/1284] Format files --- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 1ce413db4..98641ab26 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -25,8 +25,8 @@ import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; import sqlancer.mysql.ast.MySQLBinaryOperation; -import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLBinaryOperation.MySQLBinaryOperator; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLComputableFunction; From 5fa0d837a09cb8c90ee6a021a40756090399a7f0 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sun, 6 Apr 2025 11:23:56 +0800 Subject: [PATCH 1085/1284] [MySQL] Add PQS for CASE expressions --- .../mysql/MySQLExpectedValueVisitor.java | 17 ++++++--- src/sqlancer/mysql/ast/MySQLCaseOperator.java | 35 +++++++++++++++++++ .../mysql/MySQLExpectedValueVisitorTest.java | 33 ----------------- 3 files changed, 47 insertions(+), 38 deletions(-) delete mode 100644 test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index b0159fb27..8ad2d8b2b 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -172,27 +172,34 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { - // do nothing + // PQS is currently unsupported for aggregates. + throw new IgnoreMeException(); } @Override public void visit(MySQLCaseOperator caseOp) { print(caseOp); - if (caseOp.getSwitchCondition() != null) { - visit(caseOp.getSwitchCondition()); + MySQLExpression switchCondition = caseOp.getSwitchCondition(); + if (switchCondition != null) { + print(switchCondition); + visit(switchCondition); } List whenConditions = caseOp.getConditions(); List thenExpressions = caseOp.getExpressions(); for (int i = 0; i < whenConditions.size(); i++) { + print(whenConditions.get(i)); visit(whenConditions.get(i)); + print(thenExpressions.get(i)); visit(thenExpressions.get(i)); } - if (caseOp.getElseExpr() != null) { - visit(caseOp.getElseExpr()); + MySQLExpression elseExpr = caseOp.getElseExpr(); + if (elseExpr != null) { + print(elseExpr); + visit(elseExpr); } } } diff --git a/src/sqlancer/mysql/ast/MySQLCaseOperator.java b/src/sqlancer/mysql/ast/MySQLCaseOperator.java index c648be5ec..c91e33d7f 100644 --- a/src/sqlancer/mysql/ast/MySQLCaseOperator.java +++ b/src/sqlancer/mysql/ast/MySQLCaseOperator.java @@ -10,4 +10,39 @@ public MySQLCaseOperator(MySQLExpression switchCondition, List List thenExprs, MySQLExpression elseExpr) { super(switchCondition, whenExprs, thenExprs, elseExpr); } + + @Override + public MySQLConstant getExpectedValue() { + int nrConditions = getConditions().size(); + + MySQLExpression switchCondition = getSwitchCondition(); + List whenExprs = getConditions(); + List thenExprs = getExpressions(); + MySQLExpression elseExpr = getElseExpr(); + + if (switchCondition != null) { + MySQLConstant switchValue = switchCondition.getExpectedValue(); + + for (int i = 0; i < nrConditions; i++) { + MySQLConstant whenValue = whenExprs.get(i).getExpectedValue(); + MySQLConstant isConditionMatched = switchValue.isEquals(whenValue); + if (!isConditionMatched.isNull() && isConditionMatched.asBooleanNotNull()) { + return thenExprs.get(i).getExpectedValue(); + } + } + } else { + for (int i = 0; i < nrConditions; i++) { + MySQLConstant whenValue = whenExprs.get(i).getExpectedValue(); + if (!whenValue.isNull() && whenValue.asBooleanNotNull()) { + return thenExprs.get(i).getExpectedValue(); + } + } + } + + if (elseExpr != null) { + return elseExpr.getExpectedValue(); + } + + return null; + } } diff --git a/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java b/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java deleted file mode 100644 index 05cd273b6..000000000 --- a/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package sqlancer.mysql; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; - -import org.junit.jupiter.api.Test; - -import sqlancer.mysql.ast.MySQLCaseOperator; -import sqlancer.mysql.ast.MySQLColumnReference; -import sqlancer.mysql.ast.MySQLConstant; -import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; -import sqlancer.mysql.ast.MySQLExpression; - -public class MySQLExpectedValueVisitorTest { - - @Test - void testCaseOp() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); - MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, null); - List whenExprs = List.of(MySQLIntConstant.createIntConstant(1)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); - - MySQLExpectedValueVisitor visitor = new MySQLExpectedValueVisitor(); - - AssertionError err = assertThrows(AssertionError.class, - () -> visitor.visit(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr))); - - assertTrue(err.getMessage().contains("PQS not supported")); - } -} From c6c8576a97e952f9421264287fdfca88d8a93743 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Tue, 8 Apr 2025 11:40:34 +0930 Subject: [PATCH 1086/1284] Add an expected error around ALTER TABLE ALTER COLUMN --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index ac2964661..f12927e90 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -206,6 +206,7 @@ public SQLQueryAdapter generate() { sb.append("DROP NOT NULL"); errors.add("is in a primary key"); errors.add("is an identity column"); + errors.add("is in index used as replica identity"); } break; case ALTER_COLUMN_SET_STATISTICS: From 089d2d00583c115ee5e3170d9f14bdb5cad27532 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 10 Apr 2025 16:58:50 +0800 Subject: [PATCH 1087/1284] Address the issues mentioned in the PR comments. --- .../common/schema/AbstractTables.java | 2 +- src/sqlancer/sqlite3/SQLite3Options.java | 14 +++++++- .../sqlite3/SQLite3ToStringVisitor.java | 34 +++---------------- src/sqlancer/sqlite3/ast/SQLite3Select.java | 2 ++ .../sqlite3/oracle/SQLite3CODDTestOracle.java | 6 ++-- .../sqlite3/schema/SQLite3DataType.java | 17 ++++++++++ 6 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java index c75cb30e1..67da6364e 100644 --- a/src/sqlancer/common/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -51,7 +51,7 @@ public void removeTable(T table) { } } - public Boolean isContained(T table) { + public boolean isContained(T table) { return this.tables.contains(table); } diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 47a80e832..6a5188337 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -79,8 +79,20 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") public int maxNumIndexes = 20; + public enum coddtest_model { + random, expression, subquery; + public boolean isRandom() { + return this == random; + } + public boolean isExpression() { + return this == expression; + } + public boolean isSubquery() { + return this == subquery; + } + } @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") - public String coddTestModel = "random"; + public coddtest_model coddTestModel = coddtest_model.random; @Override public List getTestOracleFactory() { diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 987d293e8..4b504a4be 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -5,7 +5,6 @@ import java.util.Map; import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; @@ -156,7 +155,7 @@ public void visit(SQLite3Select s, boolean inner) { visit(whereClause); sb.append(")"); } - if (s.getGroupByClause().size() > 0) { + if (!s.getGroupByClause().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); visit(s.getGroupByClause()); @@ -550,22 +549,8 @@ public void visit(SQLite3Values values) { sb.append("(CAST("); visit(vs.get(name).get(i)); sb.append(" AS "); - switch(vs.get(name).get(i).getDataType()) { - case BINARY: - sb.append("BLOB))"); - break; - case INT: - sb.append("INT))"); - break; - case TEXT: - sb.append("TEXT))"); - break; - case REAL: - sb.append("REAL))"); - break; - default: - throw new IgnoreMeException(); - } + sb.append(vs.get(name).get(i).getDataType().toString()); + sb.append("))"); } isFirstColumn = true; } @@ -591,7 +576,7 @@ public void visit(SQLite3Typeof expr) { @Override public void visit(SQLite3ResultMap tableSummary) { - // we utlize CASE WHEN THEN END here + // We use the CASE WHEN THEN END expression to represent the result of an expression for each row in the table. SQLite3Values values = tableSummary.getValues(); List columnRefs = tableSummary.getColumns(); List summary = tableSummary.getSummary(); @@ -599,16 +584,7 @@ public void visit(SQLite3ResultMap tableSummary) { Map> vs = values.getValues(); int size = vs.get(vs.keySet().iterator().next()).size(); if (size == 0) { - sb.append("("); - for (int j = 0; j < columnRefs.size(); ++j) { - visit(columnRefs.get(j)); - sb.append(" IS NULL"); - if (j < columnRefs.size() - 1) { - sb.append(" AND "); - } - } - sb.append(")"); - return; + throw new AssertionError("The result of the expression must not be empty."); } List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); sb.append(" CASE "); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 6f1571df0..cf35b124a 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -177,6 +177,8 @@ public SQLite3Expression getWithClause() { return this.withClause; } + // This method is used in CODDTest to test subquery by replacing a table name + // in the SELECT clause with a derived table expression. public void replaceFromTable(String tableName, SQLite3Expression newFromExpression) { int replaceIdx = -1; for (int i = 0; i < fromList.size(); ++i) { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 72d1b6072..049ca7fe4 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -929,11 +929,11 @@ private Map getColumnTypeFromSelect(SQLite3Select sele } public boolean useSubquery() { - if (this.state.getDbmsSpecificOptions().coddTestModel.equals("random")) { + if (this.state.getDbmsSpecificOptions().coddTestModel.isRandom()) { return Randomly.getBoolean(); - } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("expression")) { + } else if (this.state.getDbmsSpecificOptions().coddTestModel.isExpression()) { return false; - } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("subquery")) { + } else if (this.state.getDbmsSpecificOptions().coddTestModel.isSubquery()) { return true; } else { System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index 8a343fed2..556976056 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -20,4 +20,21 @@ public static SQLite3DataType getTypeFromName(String name) { throw new IgnoreMeException(); } } + + public String toString() { + switch(this) { + case BINARY: + return "BLOB"; + case INT: + return "INT"; + case TEXT: + return "TEXT"; + case REAL: + return "REAL"; + case NULL: + return "NULL"; + default: + throw new IgnoreMeException(); + } + } } From 8e607cc46235142ee586adc0ad77b84feaae014e Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 10 Apr 2025 16:58:50 +0800 Subject: [PATCH 1088/1284] Address the issues mentioned in the PR comments. --- .settings/org.eclipse.jdt.core.prefs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 3af1c5ad8..0a659c1a7 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=11 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,7 +28,6 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore -org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -74,7 +73,6 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore -org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -112,6 +110,5 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=11 +org.eclipse.jdt.core.compiler.source=1.8 From a956eaaec4e2002cb4d21e01fbe8190d4058690d Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sun, 13 Apr 2025 10:48:56 +0800 Subject: [PATCH 1089/1284] [MySQL] Add tests for CaseOperator --- src/sqlancer/mysql/ast/MySQLCaseOperator.java | 2 +- .../mysql/MySQLToStringVisitorTest.java | 6 +- .../mysql/ast/MySQLCaseOperatorTest.java | 78 +++++++++++++++++++ 3 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java diff --git a/src/sqlancer/mysql/ast/MySQLCaseOperator.java b/src/sqlancer/mysql/ast/MySQLCaseOperator.java index c91e33d7f..ceb7fbbaa 100644 --- a/src/sqlancer/mysql/ast/MySQLCaseOperator.java +++ b/src/sqlancer/mysql/ast/MySQLCaseOperator.java @@ -43,6 +43,6 @@ public MySQLConstant getExpectedValue() { return elseExpr.getExpectedValue(); } - return null; + return MySQLConstant.createNullConstant(); } } diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 169aaa57b..7b30a2345 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -18,7 +18,7 @@ public class MySQLToStringVisitorTest { @Test void visitAggregateToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); - MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCount = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT); assertEquals("COUNT(a)", MySQLVisitor.asString(aggrCount)); @@ -36,7 +36,7 @@ void visitAggregateToString() { @Test void visitAggregateWithDistinctToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); - MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCountDistinct = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT_DISTINCT); @@ -58,7 +58,7 @@ void visitAggregateWithDistinctToString() { @Test void visitCaseWhenToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); - MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, null); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); diff --git a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java new file mode 100644 index 000000000..6df384b50 --- /dev/null +++ b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java @@ -0,0 +1,78 @@ +package sqlancer.mysql.ast; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import sqlancer.mysql.MySQLSchema; +import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; + +public class MySQLCaseOperatorTest { + + @Test + void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createIntConstant(1)); + + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + MySQLCaseOperator caseOperator = new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr); + + assertEquals("11", caseOperator.getExpectedValue().toString()); + } + + @Test + void getExpectedValue_switchConditionHasNoMatches_ReturnsElse() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createNullConstant()); + + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + assertEquals("0", + new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr).getExpectedValue().toString()); + assertEquals("NULL", + new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null).getExpectedValue().toString()); + } + + @Test + void getExpectedValue_whenTrue_ReturnsThen() { + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + MySQLCaseOperator caseOperator = new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr); + + assertEquals("11", caseOperator.getExpectedValue().toString()); + } + + @Test + void getExpectedValue_whenAllFalse_ReturnsElse() { + List whenExprs = List.of(MySQLIntConstant.createBoolean(false), + MySQLIntConstant.createBoolean(false)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + assertEquals("0", new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr).getExpectedValue().toString()); + assertEquals("NULL", new MySQLCaseOperator(null, whenExprs, thenExprs, null).getExpectedValue().toString()); + } +} From 49f36ed7fdf0c5f06406a7b646be9f3c73961e31 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sun, 13 Apr 2025 11:39:18 +0800 Subject: [PATCH 1090/1284] Remove unneeded blank lines --- .../mysql/MySQLToStringVisitorTest.java | 3 --- .../mysql/ast/MySQLCaseOperatorTest.java | 27 ++++++------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 7b30a2345..3f991e3f1 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -59,13 +59,10 @@ void visitAggregateWithDistinctToString() { void visitCaseWhenToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); - List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); assertEquals("(CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 0 END)", diff --git a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java index 6df384b50..674ab027e 100644 --- a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java +++ b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java @@ -1,6 +1,8 @@ package sqlancer.mysql.ast; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.List; import org.junit.jupiter.api.Test; @@ -14,65 +16,52 @@ public class MySQLCaseOperatorTest { void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createIntConstant(1)); - List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); MySQLCaseOperator caseOperator = new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr); - assertEquals("11", caseOperator.getExpectedValue().toString()); + assertEquals(11, caseOperator.getExpectedValue().getInt()); } @Test void getExpectedValue_switchConditionHasNoMatches_ReturnsElse() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createNullConstant()); - List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); - assertEquals("0", - new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr).getExpectedValue().toString()); - assertEquals("NULL", - new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null).getExpectedValue().toString()); + assertEquals(0, new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr).getExpectedValue().getInt()); + assertTrue(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null).getExpectedValue().isNull()); } @Test void getExpectedValue_whenTrue_ReturnsThen() { List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); - MySQLCaseOperator caseOperator = new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr); - assertEquals("11", caseOperator.getExpectedValue().toString()); + assertEquals(11, caseOperator.getExpectedValue().getInt()); } @Test void getExpectedValue_whenAllFalse_ReturnsElse() { List whenExprs = List.of(MySQLIntConstant.createBoolean(false), MySQLIntConstant.createBoolean(false)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); - assertEquals("0", new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr).getExpectedValue().toString()); - assertEquals("NULL", new MySQLCaseOperator(null, whenExprs, thenExprs, null).getExpectedValue().toString()); + assertEquals(0, new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr).getExpectedValue().getInt()); + assertTrue(new MySQLCaseOperator(null, whenExprs, thenExprs, null).getExpectedValue().isNull()); } } From 4faa1c9741cf186e5b09d032e6491d3d8bcc4c19 Mon Sep 17 00:00:00 2001 From: liyxiris Date: Sun, 14 Jan 2024 20:21:16 +0800 Subject: [PATCH 1091/1284] [Hive] Support TLP oracle --- .github/workflows/main.yml | 36 ++ .gitignore | 4 +- pom.xml | 10 + src/sqlancer/Main.java | 2 + src/sqlancer/hive/HiveErrors.java | 40 +++ src/sqlancer/hive/HiveGlobalState.java | 11 + src/sqlancer/hive/HiveOptions.java | 35 ++ src/sqlancer/hive/HiveProvider.java | 121 +++++++ src/sqlancer/hive/HiveSchema.java | 114 ++++++ src/sqlancer/hive/HiveToStringVisitor.java | 115 ++++++ .../hive/ast/HiveBetweenOperation.java | 12 + .../hive/ast/HiveBinaryOperation.java | 12 + src/sqlancer/hive/ast/HiveCaseOperation.java | 13 + src/sqlancer/hive/ast/HiveCastOperation.java | 25 ++ .../hive/ast/HiveColumnReference.java | 11 + src/sqlancer/hive/ast/HiveConstant.java | 215 +++++++++++ src/sqlancer/hive/ast/HiveExpression.java | 7 + src/sqlancer/hive/ast/HiveFunction.java | 13 + src/sqlancer/hive/ast/HiveInOperation.java | 12 + src/sqlancer/hive/ast/HiveJoin.java | 48 +++ src/sqlancer/hive/ast/HiveOrderingTerm.java | 10 + src/sqlancer/hive/ast/HiveSelect.java | 41 +++ src/sqlancer/hive/ast/HiveTableReference.java | 13 + .../hive/ast/HiveUnaryPostfixOperation.java | 13 + .../hive/ast/HiveUnaryPrefixOperation.java | 13 + .../hive/gen/HiveExpressionGenerator.java | 334 ++++++++++++++++++ .../hive/gen/HiveInsertGenerator.java | 52 +++ src/sqlancer/hive/gen/HiveTableGenerator.java | 125 +++++++ src/sqlancer/hive/oracle/HiveTLPBase.java | 68 ++++ .../hive/oracle/HiveTLPWhereOracle.java | 42 +++ test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestHiveTLP.java | 20 ++ 32 files changed, 1585 insertions(+), 3 deletions(-) create mode 100644 src/sqlancer/hive/HiveErrors.java create mode 100644 src/sqlancer/hive/HiveGlobalState.java create mode 100644 src/sqlancer/hive/HiveOptions.java create mode 100644 src/sqlancer/hive/HiveProvider.java create mode 100644 src/sqlancer/hive/HiveSchema.java create mode 100644 src/sqlancer/hive/HiveToStringVisitor.java create mode 100644 src/sqlancer/hive/ast/HiveBetweenOperation.java create mode 100644 src/sqlancer/hive/ast/HiveBinaryOperation.java create mode 100644 src/sqlancer/hive/ast/HiveCaseOperation.java create mode 100644 src/sqlancer/hive/ast/HiveCastOperation.java create mode 100644 src/sqlancer/hive/ast/HiveColumnReference.java create mode 100644 src/sqlancer/hive/ast/HiveConstant.java create mode 100644 src/sqlancer/hive/ast/HiveExpression.java create mode 100644 src/sqlancer/hive/ast/HiveFunction.java create mode 100644 src/sqlancer/hive/ast/HiveInOperation.java create mode 100644 src/sqlancer/hive/ast/HiveJoin.java create mode 100644 src/sqlancer/hive/ast/HiveOrderingTerm.java create mode 100644 src/sqlancer/hive/ast/HiveSelect.java create mode 100644 src/sqlancer/hive/ast/HiveTableReference.java create mode 100644 src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java create mode 100644 src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java create mode 100644 src/sqlancer/hive/gen/HiveExpressionGenerator.java create mode 100644 src/sqlancer/hive/gen/HiveInsertGenerator.java create mode 100644 src/sqlancer/hive/gen/HiveTableGenerator.java create mode 100644 src/sqlancer/hive/oracle/HiveTLPBase.java create mode 100644 src/sqlancer/hive/oracle/HiveTLPWhereOracle.java create mode 100644 test/sqlancer/dbms/TestHiveTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1744f6a3c..8ea1059a6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -297,6 +297,42 @@ jobs: - name: Run Tests run: mvn -Dtest=TestH2 test + hive: + name: DBMS Tests (Hive) + runs-on: ubuntu-latest + services: + metastore: + image: apache/hive:4.0.1 + env: + SERVICE_NAME: 'metastore' + ports: + - 9083:9083 + volumes: + - warehouse:/opt/hive/data/warehouse + hiveserver2: + image: apache/hive:4.0.1 + env: + SERVICE_NAME: 'hiveserver2' + ports: + - 10000:10000 + - 10002:10002 + volumes: + - warehouse:/opt/hive/data/warehouse + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: HIVE_AVAILABLE=true mvn -Dtest=TestHiveTLP test + hsqldb: name: DBMS Tests (HSQLB) runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index ff30cb6de..d7cbeb55f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ target/ .classpath -.settings/org.eclipse.core.resources.prefs -.settings/org.eclipse.m2e.core.prefs -.settings/org.eclipse.jdt.core.prefs +.settings/ .vscode .project .checkstyle diff --git a/pom.xml b/pom.xml index 836d84fa5..1894cae67 100644 --- a/pom.xml +++ b/pom.xml @@ -377,6 +377,16 @@ org.apache.arrow flight-sql-jdbc-driver 16.1.0 + + + org.apache.hive + hive-jdbc + 3.1.3 + + + org.apache.hive + hive-serde + 3.1.3 diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 7df750bf1..eb3ad7f0c 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -34,6 +34,7 @@ import sqlancer.doris.DorisProvider; import sqlancer.duckdb.DuckDBProvider; import sqlancer.h2.H2Provider; +import sqlancer.hive.HiveProvider; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.mariadb.MariaDBProvider; import sqlancer.materialize.MaterializeProvider; @@ -735,6 +736,7 @@ private static void checkForIssue799(List> providers) providers.add(new DorisProvider()); providers.add(new DuckDBProvider()); providers.add(new H2Provider()); + providers.add(new HiveProvider()); providers.add(new HSQLDBProvider()); providers.add(new MariaDBProvider()); providers.add(new MaterializeProvider()); diff --git a/src/sqlancer/hive/HiveErrors.java b/src/sqlancer/hive/HiveErrors.java new file mode 100644 index 000000000..b061c1071 --- /dev/null +++ b/src/sqlancer/hive/HiveErrors.java @@ -0,0 +1,40 @@ +package sqlancer.hive; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public class HiveErrors { + + private HiveErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("cannot recognize input near"); + errors.add("Argument type mismatch"); + errors.add("Error while compiling statement"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Either CHECK or NOT NULL constraint violated!"); + errors.add("Error running query"); + errors.add("is different from preceding arguments"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } +} diff --git a/src/sqlancer/hive/HiveGlobalState.java b/src/sqlancer/hive/HiveGlobalState.java new file mode 100644 index 000000000..a362c1f88 --- /dev/null +++ b/src/sqlancer/hive/HiveGlobalState.java @@ -0,0 +1,11 @@ +package sqlancer.hive; + +import sqlancer.SQLGlobalState; + +public class HiveGlobalState extends SQLGlobalState { + + @Override + protected HiveSchema readSchema() throws Exception { + return HiveSchema.fromConnection(getConnection(), getDatabaseName()); + } +} diff --git a/src/sqlancer/hive/HiveOptions.java b/src/sqlancer/hive/HiveOptions.java new file mode 100644 index 000000000..fff2d74b1 --- /dev/null +++ b/src/sqlancer/hive/HiveOptions.java @@ -0,0 +1,35 @@ +package sqlancer.hive; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import sqlancer.common.oracle.TestOracle; +import sqlancer.DBMSSpecificOptions; +import sqlancer.hive.oracle.HiveTLPWhereOracle; +import sqlancer.OracleFactory; + +import java.util.Arrays; +import java.util.List; + +@Parameters(separators = "=", commandDescription = "Hive (default port: " + HiveOptions.DEFAULT_PORT + + ", default host: " + HiveOptions.DEFAULT_HOST + ")") +public class HiveOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 10000; + + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(HiveOracleFactory.TLPWhere); + + public enum HiveOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(HiveGlobalState globalState) { + return new HiveTLPWhereOracle(globalState); + } + }; + } + + @Override + public List getTestOracleFactory() { + return oracle; + } +} diff --git a/src/sqlancer/hive/HiveProvider.java b/src/sqlancer/hive/HiveProvider.java new file mode 100644 index 000000000..cf1df5410 --- /dev/null +++ b/src/sqlancer/hive/HiveProvider.java @@ -0,0 +1,121 @@ +package sqlancer.hive; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.hive.gen.HiveInsertGenerator; +import sqlancer.hive.gen.HiveTableGenerator; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +@AutoService(DatabaseProvider.class) +public class HiveProvider extends SQLProviderAdapter { + + public HiveProvider() { + super(HiveGlobalState.class, HiveOptions.class); + } + + public enum Action implements AbstractAction { + + INSERT(HiveInsertGenerator::getQuery); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(HiveGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(HiveGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + default: + throw new AssertionError(a); + } + } + + @Override + public void generateDatabase(HiveGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + String tableName = globalState.getSchema().getFreeTableName(); + SQLQueryAdapter qt = HiveTableGenerator.generate(globalState, tableName); + success = globalState.executeStatement(qt); + } while(!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + + StatementExecutor se = new StatementExecutor( + globalState, Action.values(), + HiveProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(HiveGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = HiveOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = HiveOptions.DEFAULT_PORT; + } + + String databaseName = globalState.getDatabaseName(); + + String url = String.format("jdbc:hive2://%s:%d/%s", host, port, "default"); + Connection con = DriverManager.getConnection(url, username, password); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + con.close(); + con = DriverManager.getConnection( + String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName, + username, password)); + + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "hive"; + } +} diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java new file mode 100644 index 000000000..018293093 --- /dev/null +++ b/src/sqlancer/hive/HiveSchema.java @@ -0,0 +1,114 @@ +package sqlancer.hive; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.hive.HiveSchema.HiveTable; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class HiveSchema extends AbstractSchema { + + public enum HiveDataType { + + // TODO: support more types, e.g. TIMESTAMP, DATE, VARCHAR, CHAR, BINARY, ARRAY, MAP, STRUCT, UNIONTYPE... + INT, FLOAT, DECIMAL, STRING, BOOLEAN; + + public static HiveDataType getRandomType() { + return Randomly.fromList(Arrays.asList(values())); + } + } + + public static class HiveColumn extends AbstractTableColumn { + + public HiveColumn(String name, HiveTable table, HiveDataType type) { + super(name, table, type); + } + } + + public static class HiveTables extends AbstractTables { + + public HiveTables(List tables) { + super(tables); + } + } + + public static class HiveTable extends AbstractRelationalTable { + + public HiveTable(String name, List columns, boolean isView) { + super(name, columns, Collections.emptyList(), isView); + } + } + + public HiveSchema(List databaseTables) { + super(databaseTables); + } + + public static HiveSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + // TODO: discover built-in tables + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + HiveTable t = new HiveTable(tableName, databaseColumns, isView); + for (HiveColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new HiveSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet tableRs = s.executeQuery("SHOW TABLES"); + while (tableRs.next()) { + String tableName = tableRs.getString(1); + tableNames.add(tableName); + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("DESCRIBE %s", tableName))) { + while (rs.next()) { + String columnName = rs.getString("col_name"); + String dataType = rs.getString("data_type"); + HiveColumn c = new HiveColumn(columnName, null, getColumnType(dataType.toUpperCase())); + columns.add(c); + } + } + } + return columns; + } + + private static HiveDataType getColumnType(String typeString) { + if (typeString.startsWith("DECIMAL")) { + return HiveDataType.DECIMAL; + } + return HiveDataType.valueOf(typeString.toUpperCase()); + } + + public HiveTables getRandomTableNonEmptyTables() { + return new HiveTables(Randomly.nonEmptySubset(getDatabaseTables())); + } +} diff --git a/src/sqlancer/hive/HiveToStringVisitor.java b/src/sqlancer/hive/HiveToStringVisitor.java new file mode 100644 index 000000000..3815561d8 --- /dev/null +++ b/src/sqlancer/hive/HiveToStringVisitor.java @@ -0,0 +1,115 @@ +package sqlancer.hive; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hive.ast.HiveConstant; +import sqlancer.hive.ast.HiveExpression; +import sqlancer.hive.ast.HiveJoin; +import sqlancer.hive.ast.HiveSelect; +import sqlancer.hive.ast.HiveCastOperation; + +public class HiveToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(HiveExpression expr) { + if (expr instanceof HiveConstant) { + visit((HiveConstant) expr); + } else if (expr instanceof HiveSelect) { + visit((HiveSelect) expr); + } else if (expr instanceof HiveJoin) { + visit((HiveJoin) expr); + } else if (expr instanceof HiveCastOperation) { + visit((HiveCastOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(HiveConstant constant) { + sb.append(constant.toString()); + } + + private void visit(HiveSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + private void visit(HiveJoin join) { + switch (join.getJoinType()) { + case INNER: + sb.append(" INNER JOIN "); + break; + case LEFT_OUTER: + sb.append(" LEFT JOIN "); + break; + case RIGHT_OUTER: + sb.append(" RIGHT JOIN "); + break; + case FULL_OUTER: + sb.append(" FULL JOIN "); + break; + case LEFT_SEMI: + sb.append(" LEFT SEMI JOIN "); + break; + case CROSS: + sb.append(" CROSS JOIN "); + break; + default: + throw new UnsupportedOperationException(); + } + visit((TableReferenceNode) join.getRightTable()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + private void visit(HiveCastOperation cast) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + sb.append(cast.getType()); + sb.append(")"); + } + + public static String asString(HiveExpression expr) { + HiveToStringVisitor visitor = new HiveToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/hive/ast/HiveBetweenOperation.java b/src/sqlancer/hive/ast/HiveBetweenOperation.java new file mode 100644 index 000000000..a65ef70d2 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveBetweenOperation.java @@ -0,0 +1,12 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class HiveBetweenOperation extends NewBetweenOperatorNode + implements HiveExpression { + + public HiveBetweenOperation(HiveExpression left, HiveExpression middle, HiveExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/hive/ast/HiveBinaryOperation.java b/src/sqlancer/hive/ast/HiveBinaryOperation.java new file mode 100644 index 000000000..61d9cf104 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveBinaryOperation.java @@ -0,0 +1,12 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class HiveBinaryOperation extends NewBinaryOperatorNode + implements HiveExpression { + + public HiveBinaryOperation(HiveExpression left, HiveExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/hive/ast/HiveCaseOperation.java b/src/sqlancer/hive/ast/HiveCaseOperation.java new file mode 100644 index 000000000..2a52a5c45 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveCaseOperation.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class HiveCaseOperation extends NewCaseOperatorNode implements HiveExpression { + + public HiveCaseOperation(HiveExpression switchCondition, List conditions, + List expressions, HiveExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/hive/ast/HiveCastOperation.java b/src/sqlancer/hive/ast/HiveCastOperation.java new file mode 100644 index 000000000..ae2fbe435 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveCastOperation.java @@ -0,0 +1,25 @@ +package sqlancer.hive.ast; + +import sqlancer.hive.HiveSchema.HiveDataType; + +public class HiveCastOperation implements HiveExpression { + + private final HiveExpression expression; + private final HiveDataType type; + + public HiveCastOperation(HiveExpression expression, HiveDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + public HiveExpression getExpression() { + return expression; + } + + public HiveDataType getType() { + return type; + } +} diff --git a/src/sqlancer/hive/ast/HiveColumnReference.java b/src/sqlancer/hive/ast/HiveColumnReference.java new file mode 100644 index 000000000..c3237955f --- /dev/null +++ b/src/sqlancer/hive/ast/HiveColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.hive.HiveSchema.HiveColumn; + +public class HiveColumnReference extends ColumnReferenceNode implements HiveExpression { + + public HiveColumnReference(HiveColumn column) { + super(column); + } +} diff --git a/src/sqlancer/hive/ast/HiveConstant.java b/src/sqlancer/hive/ast/HiveConstant.java new file mode 100644 index 000000000..ed3ed591d --- /dev/null +++ b/src/sqlancer/hive/ast/HiveConstant.java @@ -0,0 +1,215 @@ +package sqlancer.hive.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public abstract class HiveConstant implements HiveExpression { + + public boolean isNull() { + return false; + } + + public static class HiveNullConstant extends HiveConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public String toString() { + return "NULL"; + } + } + + public static class HiveIntConstant extends HiveConstant { + + private final long value; + + public HiveIntConstant(long value) { + this.value = value; + } + + public long getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // public static class HiveFloatConstant extends HiveConstant { + + // private final float value; + + // public HiveFloatConstant(float value) { + // this.value = value; + // } + + // public float getValue() { + // return value; + // } + + // @Override + // public String toString() { + // if (value == Float.POSITIVE_INFINITY) { + // return "'+Inf'"; + // } else if (value == Float.NEGATIVE_INFINITY) { + // return "'-Inf'"; + // } + // return String.valueOf(value); + // } + // } + + public static class HiveDoubleConstant extends HiveConstant { + + private final double value; + + public HiveDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + } + + public static class HiveDecimalConstant extends HiveConstant { + + private final BigDecimal value; + + public HiveDecimalConstant(BigDecimal value) { + this.value = value; + } + + public BigDecimal getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class HiveTimestampConstant extends HiveConstant { + + private final String textRepr; + + public HiveTimestampConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + } + + public static class HiveDateConstant extends HiveConstant { + + private final String textRepr; + + public HiveDateConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + } + + public static class StringConstant extends HiveConstant { + + private final String value; + + public StringConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + } + + public static class HiveBooleanConstant extends HiveConstant { + + private final boolean value; + + public HiveBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static HiveConstant createNullConstant() { + return new HiveNullConstant(); + } + + public static HiveConstant createIntConstant(long value) { + return new HiveIntConstant(value); + } + + public static HiveConstant createDoubleConstant(double value) { + return new HiveDoubleConstant(value); + } + + public static HiveConstant createDecimalConstant(BigDecimal value) { + return new HiveDecimalConstant(value); + } + + public static HiveConstant createTimestampConstant(long value) { + return new HiveTimestampConstant(value); + } + + public static HiveConstant createDateConstant(long value) { + return new HiveDateConstant(value); + } + + public static HiveConstant createStringConstant(String value) { + return new StringConstant(value); + } + + public static HiveConstant createBooleanConstant(boolean value) { + return new HiveBooleanConstant(value); + } +} diff --git a/src/sqlancer/hive/ast/HiveExpression.java b/src/sqlancer/hive/ast/HiveExpression.java new file mode 100644 index 000000000..40842e181 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveExpression.java @@ -0,0 +1,7 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.hive.HiveSchema.HiveColumn; + +public interface HiveExpression extends Expression { +} diff --git a/src/sqlancer/hive/ast/HiveFunction.java b/src/sqlancer/hive/ast/HiveFunction.java new file mode 100644 index 000000000..507015259 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveFunction.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class HiveFunction extends NewFunctionNode implements HiveExpression { + + public HiveFunction(List args, F func) { + super(args, func); + } + +} diff --git a/src/sqlancer/hive/ast/HiveInOperation.java b/src/sqlancer/hive/ast/HiveInOperation.java new file mode 100644 index 000000000..ccffde58c --- /dev/null +++ b/src/sqlancer/hive/ast/HiveInOperation.java @@ -0,0 +1,12 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class HiveInOperation extends NewInOperatorNode implements HiveExpression { + + public HiveInOperation(HiveExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/hive/ast/HiveJoin.java b/src/sqlancer/hive/ast/HiveJoin.java new file mode 100644 index 000000000..fc6b7a200 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveJoin.java @@ -0,0 +1,48 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.Join; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; + +public class HiveJoin implements HiveExpression, Join { + + private final HiveTableReference leftTable; + private final HiveTableReference rightTable; + private final JoinType joinType; + private HiveExpression onClause; + + // TODO: test map-join optimization + + public enum JoinType { + INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, CROSS; + } + + public HiveJoin(HiveTableReference leftTable, HiveTableReference rightTable, JoinType joinType, + HiveExpression onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onClause = onClause; + } + + public HiveTableReference getLeftTable() { + return leftTable; + } + + public HiveTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public HiveExpression getOnClause() { + return onClause; + } + + @Override + public void setOnClause(HiveExpression onClause) { + this.onClause = onClause; + } +} diff --git a/src/sqlancer/hive/ast/HiveOrderingTerm.java b/src/sqlancer/hive/ast/HiveOrderingTerm.java new file mode 100644 index 000000000..70fef52ad --- /dev/null +++ b/src/sqlancer/hive/ast/HiveOrderingTerm.java @@ -0,0 +1,10 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class HiveOrderingTerm extends NewOrderingTerm implements HiveExpression { + + public HiveOrderingTerm(HiveExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/hive/ast/HiveSelect.java b/src/sqlancer/hive/ast/HiveSelect.java new file mode 100644 index 000000000..ffd5e8fbd --- /dev/null +++ b/src/sqlancer/hive/ast/HiveSelect.java @@ -0,0 +1,41 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; + +public class HiveSelect extends SelectBase + implements Select, HiveExpression { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'setJoinClauses'"); + } + + @Override + public List getJoinClauses() { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'getJoinClauses'"); + } + + @Override + public String asString() { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'asString'"); + } + +} diff --git a/src/sqlancer/hive/ast/HiveTableReference.java b/src/sqlancer/hive/ast/HiveTableReference.java new file mode 100644 index 000000000..a0955f86c --- /dev/null +++ b/src/sqlancer/hive/ast/HiveTableReference.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hive.HiveSchema; + +public class HiveTableReference extends TableReferenceNode + implements HiveExpression { + + public HiveTableReference(HiveSchema.HiveTable table) { + super(table); + } + +} \ No newline at end of file diff --git a/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java new file mode 100644 index 000000000..c32a40847 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class HiveUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements HiveExpression { + + public HiveUnaryPostfixOperation(HiveExpression expr, Operator op) { + super(expr, op); + } + +} diff --git a/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java new file mode 100644 index 000000000..9312c4c26 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class HiveUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements HiveExpression { + + public HiveUnaryPrefixOperation(HiveExpression expr, Operator op) { + super(expr, op); + } + +} diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java new file mode 100644 index 000000000..d8949a72c --- /dev/null +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -0,0 +1,334 @@ +package sqlancer.hive.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema.*; +import sqlancer.hive.ast.HiveBetweenOperation; +import sqlancer.hive.ast.HiveBinaryOperation; +import sqlancer.hive.ast.HiveCaseOperation; +import sqlancer.hive.ast.HiveCastOperation; +import sqlancer.hive.ast.HiveColumnReference; +import sqlancer.hive.ast.HiveConstant; +import sqlancer.hive.ast.HiveExpression; +import sqlancer.hive.ast.HiveFunction; +import sqlancer.hive.ast.HiveInOperation; +import sqlancer.hive.ast.HiveOrderingTerm; +import sqlancer.hive.ast.HiveUnaryPrefixOperation; +import sqlancer.hive.ast.HiveUnaryPostfixOperation; +import sqlancer.common.ast.BinaryOperatorNode.Operator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class HiveExpressionGenerator extends UntypedExpressionGenerator { + + private final HiveGlobalState globalState; + + private enum Expression { + // TODO: add or delete expressions. + UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, + CAST, FUNC, BETWEEN, IN, CASE; + } + + private enum ConstantType { + // TODO: add DECIMAL, DATE, TIMESTAMP, BINARY,... + STRING, INT, DOUBLE, BOOLEAN, NULL + } + + public HiveExpressionGenerator(HiveGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public HiveExpression negatePredicate(HiveExpression predicate) { + return new HiveUnaryPrefixOperation(predicate, HiveUnaryPrefixOperator.NOT); + } + + @Override + public HiveExpression isNull(HiveExpression expr) { + return new HiveUnaryPostfixOperation(expr, HiveUnaryPostfixOperator.IS_NULL); + } + + @Override + protected HiveExpression generateExpression(int depth) { + // TODO: randomly cast some types like what PostgresExpressionGenerator does? + return generateExpressionInternal(depth); + } + + private HiveExpression generateExpressionInternal(int depth) throws AssertionError { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBooleanWithRatherLowProbability()) { + allowAggregates= false; // aggregate function calls cannot be nested + HiveAggregateFunction aggregate = HiveAggregateFunction.getRandom(); + return new HiveFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + // TODO: remove some of the possible expression types according to options. + + Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case UNARY_PREFIX: + return new HiveUnaryPrefixOperation(generateExpression(depth + 1), + HiveUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new HiveUnaryPostfixOperation(generateExpression(depth + 1), + HiveUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = HiveBinaryComparisonOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = HiveExpressionGenerator.HiveBinaryLogicalOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new HiveBinaryOperation(generateExpression(depth + 1), + generateExpression(depth + 1), + HiveExpressionGenerator.HiveBinaryArithmeticOperator.getRandom()); + case CAST: + // return new HiveCastOperation(generateExpression(depth + 1), + // HiveSchema.HiveCompositeDataType.getRandomWithoutNull()); + return new HiveCastOperation(generateExpression(depth + 1), HiveDataType.getRandomType()); + case FUNC: + HiveFunc func = HiveFunc.getRandom(); + return new HiveFunction<>(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new HiveBetweenOperation(generateExpression(depth + 1), + generateExpression(depth + 1), + generateExpression(depth + 1), + Randomly.getBoolean()); + case IN: + return new HiveInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), + Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new HiveCaseOperation(generateExpression(depth + 1), + generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), + generateExpression(depth + 1)); + default: + throw new AssertionError(expr); + } + } + + @Override + public HiveExpression generateConstant() { + ConstantType[] values = ConstantType.values(); + ConstantType constantType = Randomly.fromOptions(values); + switch (constantType) { + case STRING: + return HiveConstant.createStringConstant(globalState.getRandomly().getString()); + case INT: + return HiveConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DOUBLE: + return HiveConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BOOLEAN: + return HiveConstant.createBooleanConstant(Randomly.getBoolean()); + case NULL: + return HiveConstant.createNullConstant(); + default: + throw new AssertionError(constantType); + } + } + + @Override + protected HiveExpression generateColumn() { + HiveColumn column = Randomly.fromList(columns); + return new HiveColumnReference(column); + } + + @Override + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (HiveExpression curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new HiveOrderingTerm(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + } + + public enum HiveUnaryPrefixOperator implements Operator { + + // TODO: ~A (bitwise NOT) + NOT("NOT"), PLUS("+"), MINUS("-"); + + private String textRepr; + + HiveUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveUnaryPostfixOperator implements Operator { + + // TODO: A IS [NOT] (NULL|TRUE|FALSE)... + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + HiveUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveBinaryComparisonOperator implements Operator { + + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), + SMALLER_EQUALS("<="), NOT_EQUALS("!="), LIKE("LIKE"), + NOT_LIKE("NOT LIKE"), REGEXP("RLIKE"); + + private String textRepr; + + HiveBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveBinaryLogicalOperator implements Operator { + + AND("AND"), OR("OR"); + + private String textRepr; + + HiveBinaryLogicalOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveBinaryArithmeticOperator implements Operator { + + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), BITWISE_AND("&"), BITWISE_OR("|"), + BITWISE_XOR("^"); + + private String textRepr; + + HiveBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveAggregateFunction { + COUNT(1), + SUM(1), + AVG(1), + MIN(1), + MAX(1), + VARIANCE(1), + VAR_SAMP(1), + STDDEV_POP(1), + STDDEV_SAMP(1), + COVAR_POP(2), + COVAR_SAMP(2), + CORR(2); + + private int nrArgs; + + HiveAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static HiveAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + // TODO: test all Hive default functions... + public enum HiveFunc { + + // mathematical functions + ROUND(2), + FLOOR(1); + + // collection functions + + // date functions + + // string functions + + + private int nrArgs; + private boolean isVariadic; + + HiveFunc(int nrArgs) { + this(nrArgs, false); + } + + HiveFunc(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static HiveFunc getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + + } +} diff --git a/src/sqlancer/hive/gen/HiveInsertGenerator.java b/src/sqlancer/hive/gen/HiveInsertGenerator.java new file mode 100644 index 000000000..963fafbce --- /dev/null +++ b/src/sqlancer/hive/gen/HiveInsertGenerator.java @@ -0,0 +1,52 @@ +package sqlancer.hive.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.HiveToStringVisitor; + +public class HiveInsertGenerator extends AbstractInsertGenerator { + + private final HiveGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + private final HiveExpressionGenerator gen; + + public HiveInsertGenerator(HiveGlobalState globalState) { + this.globalState = globalState; + this.gen = new HiveExpressionGenerator(globalState); + } + + public static SQLQueryAdapter getQuery(HiveGlobalState globalState) { + return new HiveInsertGenerator(globalState).generate(); + } + + @Override + protected void insertValue(HiveColumn column) { + sb.append(HiveToStringVisitor.asString(gen.generateConstant())); + } + + private SQLQueryAdapter generate() { + // Inserting values into tables from SQL. + sb.append("INSERT INTO "); + HiveTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + + // TODO: specify the inserted partition + + sb.append(" VALUES "); + + // Values must be provided by every column in the Hive table. + // A value is either null or any valid SQL literal. + List columns = table.getColumns(); + insertColumns(columns); + + HiveErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } +} diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java new file mode 100644 index 000000000..6f1a2121b --- /dev/null +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -0,0 +1,125 @@ +package sqlancer.hive.gen; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema; +import sqlancer.hive.HiveToStringVisitor; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveDataType; +import sqlancer.hive.HiveSchema.HiveTable; + +import java.util.ArrayList; +import java.util.List; + +public class HiveTableGenerator { + + // TODO: support various file formats? e.g. JSONFILE, SEQUENCEFILE, TEXTFILE, RCFILE, ORC, PARQUET, AVRO. + + private enum ColumnConstraints { + PRIMARY_KEY_DISABLE, UNIQUE_DISABLE, NOT_NULL, DEFAULT, CHECK + // ENABLE_OR_DISABLE, NOVALIDATE, RELY_OR_NORELY + } + + private final HiveGlobalState globalState; + private final String tableName; + private final boolean allowPrimaryKey = Randomly.getBoolean(); + private final StringBuilder sb = new StringBuilder(); + private final HiveExpressionGenerator gen; + private final HiveTable table; + private final List columnsToBeAdded = new ArrayList<>(); + private boolean setPrimaryKey = false; + + public HiveTableGenerator(HiveGlobalState globalState, String tableName) { + this.tableName = tableName; + this.globalState = globalState; + this.table = new HiveTable(tableName, columnsToBeAdded, false); + this.gen = new HiveExpressionGenerator(globalState).setColumns(columnsToBeAdded); + } + + public static SQLQueryAdapter generate(HiveGlobalState globalState, String tableName) { + HiveTableGenerator generator = new HiveTableGenerator(globalState, tableName); + return generator.create(); + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + + sb.append("CREATE "); + sb.append("TABLE "); + sb.append(globalState.getDatabaseName()); + sb.append("."); + sb.append(tableName); + sb.append(" ("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(i, errors); + } + sb.append(")"); + + // TODO: implement PARTITION BY clause + // TODO: implement CLUSTERED BY, SKEWED BY clauses + // TODO: implement ROW FORMAT and STORED AS clauses + // TODO: randomly add some predefined TABLEPROPERTIES + // TODO: implement CTAS (AS clause) + + HiveErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + + private void appendColumn(int columnId, ExpectedErrors errors) { + String columnName = DBMSCommon.createColumnName(columnId); + sb.append(columnName); + sb.append(" "); + HiveDataType randType = HiveSchema.HiveDataType.getRandomType(); + sb.append(randType); + columnsToBeAdded.add(new HiveColumn(columnName, table, randType)); + appendColumnConstraint(randType, errors); + } + + private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { + /* column_constraint_specification: + : [ PRIMARY KEY|UNIQUE|NOT NULL|DEFAULT [default_value]|CHECK [check_expression] ENABLE|DISABLE NOVALIDATE RELY/NORELY ] + */ + if (Randomly.getBoolean()) { + // no column constraint + return; + } + + ColumnConstraints constraint = Randomly.fromOptions(ColumnConstraints.values()); + switch (constraint) { + case PRIMARY_KEY_DISABLE: + if (allowPrimaryKey && !setPrimaryKey) { + sb.append(" PRIMARY KEY DISABLE"); + setPrimaryKey = true; + } + break; + case UNIQUE_DISABLE: + sb.append(" UNIQUE DISABLE"); + break; + case NOT_NULL: + sb.append(" NOT NULL"); + break; + case DEFAULT: + // TODO: add errors + sb.append(" DEFAULT ("); + sb.append(HiveToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + case CHECK: + errors.add("Invalid Constraint syntax"); + sb.append(" CHECK ("); + sb.append(HiveToStringVisitor.asString(gen.generateExpression())); + sb.append(")"); + break; + default: + throw new AssertionError(constraint); + } + } + + +} diff --git a/src/sqlancer/hive/oracle/HiveTLPBase.java b/src/sqlancer/hive/oracle/HiveTLPBase.java new file mode 100644 index 000000000..cff69d06b --- /dev/null +++ b/src/sqlancer/hive/oracle/HiveTLPBase.java @@ -0,0 +1,68 @@ +package sqlancer.hive.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.HiveSchema.HiveTables; +import sqlancer.hive.ast.HiveColumnReference; +import sqlancer.hive.ast.HiveExpression; +import sqlancer.hive.ast.HiveSelect; +import sqlancer.hive.ast.HiveTableReference; +import sqlancer.hive.gen.HiveExpressionGenerator; + +public class HiveTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + HiveSchema schema; + HiveTables targetTables; + HiveExpressionGenerator gen; + HiveSelect select; + + public HiveTLPBase(HiveGlobalState state) { + super(state); + HiveErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws Exception { + schema = state.getSchema(); + targetTables = schema.getRandomTableNonEmptyTables(); + gen = new HiveExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new HiveSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List tableList = tables.stream().map(t -> new HiveTableReference(t)) + .collect(Collectors.toList()); + // List joins = HiveJoin.getJoins(tableList, state); + // select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + + List generateFetchColumns() { + List columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new HiveColumnReference(new HiveColumn("*", null, null))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new HiveColumnReference(c)) + .collect(ArrayList::new, ArrayList::add, ArrayList::addAll); + } + return columns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } +} diff --git a/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java b/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java new file mode 100644 index 000000000..d59867058 --- /dev/null +++ b/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java @@ -0,0 +1,42 @@ +package sqlancer.hive.oracle; + +import java.util.List; +import java.util.ArrayList; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveToStringVisitor; + +public class HiveTLPWhereOracle extends HiveTLPBase { + + public HiveTLPWhereOracle(HiveGlobalState state) { + super(state); + HiveErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws Exception { + super.check(); + String originalQueryString = HiveToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByClauses(gen.generateOrderBys()); + } + + select.setWhereClause(predicate); + String firstQueryString = HiveToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = HiveToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = HiveToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } +} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 92de68fe9..f5aeefa12 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -10,6 +10,7 @@ public class TestConfig { public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; + public static final String HIVE_ENV = "HIVE_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; public static final String MYSQL_ENV = "MYSQL_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestHiveTLP.java b/test/sqlancer/dbms/TestHiveTLP.java new file mode 100644 index 000000000..05a4bc20d --- /dev/null +++ b/test/sqlancer/dbms/TestHiveTLP.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestHiveTLP { + + @Test + public void testHiveTLPWhere() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.HIVE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", + "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", + "--num-queries", TestConfig.NUM_QUERIES, "hive", "--oracle", "TLPWhere" })); + } +} \ No newline at end of file From 6e8669857eff0f2d592a831801ad4e1fcbe68c1f Mon Sep 17 00:00:00 2001 From: liyxiris Date: Wed, 9 Apr 2025 19:15:21 +0800 Subject: [PATCH 1092/1284] [Hive] Remove redundant lines --- src/sqlancer/hive/HiveSchema.java | 9 +------- src/sqlancer/hive/ast/HiveConstant.java | 23 ------------------- .../hive/gen/HiveExpressionGenerator.java | 16 ++++--------- src/sqlancer/hive/gen/HiveTableGenerator.java | 4 +--- 4 files changed, 7 insertions(+), 45 deletions(-) diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java index 018293093..8ae33bf99 100644 --- a/src/sqlancer/hive/HiveSchema.java +++ b/src/sqlancer/hive/HiveSchema.java @@ -23,7 +23,7 @@ public class HiveSchema extends AbstractSchema { public enum HiveDataType { // TODO: support more types, e.g. TIMESTAMP, DATE, VARCHAR, CHAR, BINARY, ARRAY, MAP, STRUCT, UNIONTYPE... - INT, FLOAT, DECIMAL, STRING, BOOLEAN; + STRING, INT, DOUBLE, BOOLEAN; public static HiveDataType getRandomType() { return Randomly.fromList(Arrays.asList(values())); @@ -56,13 +56,9 @@ public HiveSchema(List databaseTables) { } public static HiveSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { - // TODO: discover built-in tables List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { - if (DBMSCommon.matchesIndexName(tableName)) { - continue; // TODO: unexpected? - } List databaseColumns = getTableColumns(con, tableName); boolean isView = tableName.startsWith("v"); HiveTable t = new HiveTable(tableName, databaseColumns, isView); @@ -102,9 +98,6 @@ private static List getTableColumns(SQLConnection con, String tableN } private static HiveDataType getColumnType(String typeString) { - if (typeString.startsWith("DECIMAL")) { - return HiveDataType.DECIMAL; - } return HiveDataType.valueOf(typeString.toUpperCase()); } diff --git a/src/sqlancer/hive/ast/HiveConstant.java b/src/sqlancer/hive/ast/HiveConstant.java index ed3ed591d..7f89997d2 100644 --- a/src/sqlancer/hive/ast/HiveConstant.java +++ b/src/sqlancer/hive/ast/HiveConstant.java @@ -41,29 +41,6 @@ public String toString() { } } - // public static class HiveFloatConstant extends HiveConstant { - - // private final float value; - - // public HiveFloatConstant(float value) { - // this.value = value; - // } - - // public float getValue() { - // return value; - // } - - // @Override - // public String toString() { - // if (value == Float.POSITIVE_INFINITY) { - // return "'+Inf'"; - // } else if (value == Float.NEGATIVE_INFINITY) { - // return "'-Inf'"; - // } - // return String.valueOf(value); - // } - // } - public static class HiveDoubleConstant extends HiveConstant { private final double value; diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index d8949a72c..ca2e53929 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -33,11 +33,6 @@ private enum Expression { CAST, FUNC, BETWEEN, IN, CASE; } - private enum ConstantType { - // TODO: add DECIMAL, DATE, TIMESTAMP, BINARY,... - STRING, INT, DOUBLE, BOOLEAN, NULL - } - public HiveExpressionGenerator(HiveGlobalState globalState) { this.globalState = globalState; } @@ -93,8 +88,6 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr generateExpression(depth + 1), HiveExpressionGenerator.HiveBinaryArithmeticOperator.getRandom()); case CAST: - // return new HiveCastOperation(generateExpression(depth + 1), - // HiveSchema.HiveCompositeDataType.getRandomWithoutNull()); return new HiveCastOperation(generateExpression(depth + 1), HiveDataType.getRandomType()); case FUNC: HiveFunc func = HiveFunc.getRandom(); @@ -121,8 +114,11 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr @Override public HiveExpression generateConstant() { - ConstantType[] values = ConstantType.values(); - ConstantType constantType = Randomly.fromOptions(values); + if (Randomly.getBooleanWithRatherLowProbability()) { + return HiveConstant.createNullConstant(); + } + HiveDataType[] values = HiveDataType.values(); + HiveDataType constantType = Randomly.fromOptions(values); switch (constantType) { case STRING: return HiveConstant.createStringConstant(globalState.getRandomly().getString()); @@ -132,8 +128,6 @@ public HiveExpression generateConstant() { return HiveConstant.createDoubleConstant(globalState.getRandomly().getDouble()); case BOOLEAN: return HiveConstant.createBooleanConstant(Randomly.getBoolean()); - case NULL: - return HiveConstant.createNullConstant(); default: throw new AssertionError(constantType); } diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java index 6f1a2121b..2d91325bf 100644 --- a/src/sqlancer/hive/gen/HiveTableGenerator.java +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -48,8 +48,7 @@ public static SQLQueryAdapter generate(HiveGlobalState globalState, String table private SQLQueryAdapter create() { ExpectedErrors errors = new ExpectedErrors(); - sb.append("CREATE "); - sb.append("TABLE "); + sb.append("CREATE TABLE "); sb.append(globalState.getDatabaseName()); sb.append("."); sb.append(tableName); @@ -106,7 +105,6 @@ private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { sb.append(" NOT NULL"); break; case DEFAULT: - // TODO: add errors sb.append(" DEFAULT ("); sb.append(HiveToStringVisitor.asString(gen.generateConstant())); sb.append(")"); From ab731ec7f06554bfc365197bae3ac0b156196f18 Mon Sep 17 00:00:00 2001 From: liyxiris Date: Sun, 13 Apr 2025 20:37:29 +0800 Subject: [PATCH 1093/1284] [Hive] refactor Hive TLP oracle using generic approaches --- src/sqlancer/hive/HiveOptions.java | 14 +++- src/sqlancer/hive/HiveSchema.java | 1 - src/sqlancer/hive/ast/HiveSelect.java | 12 ++-- .../hive/gen/HiveExpressionGenerator.java | 49 ++++++++++++- src/sqlancer/hive/oracle/HiveTLPBase.java | 68 ------------------- .../hive/oracle/HiveTLPWhereOracle.java | 42 ------------ 6 files changed, 65 insertions(+), 121 deletions(-) delete mode 100644 src/sqlancer/hive/oracle/HiveTLPBase.java delete mode 100644 src/sqlancer/hive/oracle/HiveTLPWhereOracle.java diff --git a/src/sqlancer/hive/HiveOptions.java b/src/sqlancer/hive/HiveOptions.java index fff2d74b1..a8c049756 100644 --- a/src/sqlancer/hive/HiveOptions.java +++ b/src/sqlancer/hive/HiveOptions.java @@ -2,11 +2,15 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; + +import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.DBMSSpecificOptions; -import sqlancer.hive.oracle.HiveTLPWhereOracle; +import sqlancer.hive.gen.HiveExpressionGenerator; import sqlancer.OracleFactory; +import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -22,8 +26,12 @@ public class HiveOptions implements DBMSSpecificOptions { TLPWhere { @Override - public TestOracle create(HiveGlobalState globalState) { - return new HiveTLPWhereOracle(globalState); + public TestOracle create(HiveGlobalState globalState) throws SQLException { + HiveExpressionGenerator gen = new HiveExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(HiveErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); } }; } diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java index 8ae33bf99..fcc17c77c 100644 --- a/src/sqlancer/hive/HiveSchema.java +++ b/src/sqlancer/hive/HiveSchema.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.DBMSCommon; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; diff --git a/src/sqlancer/hive/ast/HiveSelect.java b/src/sqlancer/hive/ast/HiveSelect.java index ffd5e8fbd..4853ef892 100644 --- a/src/sqlancer/hive/ast/HiveSelect.java +++ b/src/sqlancer/hive/ast/HiveSelect.java @@ -4,6 +4,7 @@ import sqlancer.common.ast.SelectBase; import sqlancer.common.ast.newast.Select; +import sqlancer.hive.HiveToStringVisitor; import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveTable; @@ -22,20 +23,19 @@ public boolean isDistinct() { @Override public void setJoinClauses(List joinStatements) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'setJoinClauses'"); + List expressions = joinStatements.stream().map(e -> (HiveExpression) e) + .collect(java.util.stream.Collectors.toList()); + setJoinList(expressions); } @Override public List getJoinClauses() { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'getJoinClauses'"); + return getJoinList().stream().map(e -> (HiveJoin) e).collect(java.util.stream.Collectors.toList()); } @Override public String asString() { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'asString'"); + return HiveToStringVisitor.asString(this); } } diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index ca2e53929..3e35544c5 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -2,7 +2,9 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.hive.HiveGlobalState; import sqlancer.hive.HiveSchema.*; import sqlancer.hive.ast.HiveBetweenOperation; @@ -14,7 +16,10 @@ import sqlancer.hive.ast.HiveExpression; import sqlancer.hive.ast.HiveFunction; import sqlancer.hive.ast.HiveInOperation; +import sqlancer.hive.ast.HiveJoin; import sqlancer.hive.ast.HiveOrderingTerm; +import sqlancer.hive.ast.HiveSelect; +import sqlancer.hive.ast.HiveTableReference; import sqlancer.hive.ast.HiveUnaryPrefixOperation; import sqlancer.hive.ast.HiveUnaryPostfixOperation; import sqlancer.common.ast.BinaryOperatorNode.Operator; @@ -22,10 +27,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; -public class HiveExpressionGenerator extends UntypedExpressionGenerator { +public class HiveExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { private final HiveGlobalState globalState; + private List tables; private enum Expression { // TODO: add or delete expressions. @@ -152,6 +160,45 @@ public List generateOrderBys() { return newExpr; } + + @Override + public HiveExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public HiveExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public HiveSelect generateSelect() { + return new HiveSelect(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new HiveTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean allowAggregates) { + if (Randomly.getBoolean()) { + return List.of(new HiveColumnReference(new HiveColumn("*", null, null))); + } + return Randomly.nonEmptySubset(columns).stream() + .map(c -> new HiveColumnReference(c)) + .collect(Collectors.toList()); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + public enum HiveUnaryPrefixOperator implements Operator { // TODO: ~A (bitwise NOT) diff --git a/src/sqlancer/hive/oracle/HiveTLPBase.java b/src/sqlancer/hive/oracle/HiveTLPBase.java deleted file mode 100644 index cff69d06b..000000000 --- a/src/sqlancer/hive/oracle/HiveTLPBase.java +++ /dev/null @@ -1,68 +0,0 @@ -package sqlancer.hive.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.hive.HiveErrors; -import sqlancer.hive.HiveGlobalState; -import sqlancer.hive.HiveSchema; -import sqlancer.hive.HiveSchema.HiveColumn; -import sqlancer.hive.HiveSchema.HiveTable; -import sqlancer.hive.HiveSchema.HiveTables; -import sqlancer.hive.ast.HiveColumnReference; -import sqlancer.hive.ast.HiveExpression; -import sqlancer.hive.ast.HiveSelect; -import sqlancer.hive.ast.HiveTableReference; -import sqlancer.hive.gen.HiveExpressionGenerator; - -public class HiveTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - HiveSchema schema; - HiveTables targetTables; - HiveExpressionGenerator gen; - HiveSelect select; - - public HiveTLPBase(HiveGlobalState state) { - super(state); - HiveErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getRandomTableNonEmptyTables(); - gen = new HiveExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new HiveSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List tableList = tables.stream().map(t -> new HiveTableReference(t)) - .collect(Collectors.toList()); - // List joins = HiveJoin.getJoins(tableList, state); - // select.setJoinList(joins.stream().collect(Collectors.toList())); - select.setFromList(tableList.stream().collect(Collectors.toList())); - select.setWhereClause(null); - } - - List generateFetchColumns() { - List columns = new ArrayList<>(); - if (Randomly.getBoolean()) { - columns.add(new HiveColumnReference(new HiveColumn("*", null, null))); - } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new HiveColumnReference(c)) - .collect(ArrayList::new, ArrayList::add, ArrayList::addAll); - } - return columns; - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } -} diff --git a/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java b/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java deleted file mode 100644 index d59867058..000000000 --- a/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.hive.oracle; - -import java.util.List; -import java.util.ArrayList; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.hive.HiveErrors; -import sqlancer.hive.HiveGlobalState; -import sqlancer.hive.HiveToStringVisitor; - -public class HiveTLPWhereOracle extends HiveTLPBase { - - public HiveTLPWhereOracle(HiveGlobalState state) { - super(state); - HiveErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws Exception { - super.check(); - String originalQueryString = HiveToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - - select.setWhereClause(predicate); - String firstQueryString = HiveToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = HiveToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = HiveToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } -} From b9c13c35ff4a29a78bfd7dfdd8c75e4cc8cde833 Mon Sep 17 00:00:00 2001 From: liyxiris Date: Wed, 16 Apr 2025 18:14:55 +0800 Subject: [PATCH 1094/1284] [Hive] upgrade Hive dependencies & remove expected 'Invalid Constraint syntax' error --- pom.xml | 9 +++++++-- src/sqlancer/hive/gen/HiveTableGenerator.java | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 1894cae67..bf30df4ce 100644 --- a/pom.xml +++ b/pom.xml @@ -381,12 +381,17 @@ org.apache.hive hive-jdbc - 3.1.3 + 4.0.1 org.apache.hive hive-serde - 3.1.3 + 4.0.1 + + + org.apache.hive + hive-cli + 4.0.1 diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java index 2d91325bf..43192a5be 100644 --- a/src/sqlancer/hive/gen/HiveTableGenerator.java +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -109,7 +109,6 @@ private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { sb.append(HiveToStringVisitor.asString(gen.generateConstant())); sb.append(")"); case CHECK: - errors.add("Invalid Constraint syntax"); sb.append(" CHECK ("); sb.append(HiveToStringVisitor.asString(gen.generateExpression())); sb.append(")"); From d45cd022e8f710ace2105aa79bc4eb949774bfc0 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 17 Apr 2025 13:20:02 +0800 Subject: [PATCH 1095/1284] modify variabels name to follow Java naming convertions, fix an error in SQLite3ToStringVisitor, add unit tests for AbstractTables and CODDTest for SQLite --- src/sqlancer/sqlite3/SQLite3Options.java | 14 ++++---- .../sqlite3/SQLite3ToStringVisitor.java | 2 +- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 2 +- test/sqlancer/TestCommonSchema.java | 34 +++++++++++++++++++ test/sqlancer/dbms/TestSQLiteCODDTest.java | 18 ++++++++++ 5 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 test/sqlancer/dbms/TestSQLiteCODDTest.java diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 6a5188337..fce9f04ba 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -79,20 +79,20 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") public int maxNumIndexes = 20; - public enum coddtest_model { - random, expression, subquery; + public enum CODDTestModel { + RANDOM, EXPRESSION, SUBQUERY; public boolean isRandom() { - return this == random; + return this == RANDOM; } public boolean isExpression() { - return this == expression; + return this == EXPRESSION; } public boolean isSubquery() { - return this == subquery; + return this == SUBQUERY; } } - @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") - public coddtest_model coddTestModel = coddtest_model.random; + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on EXPRESSION, SUBQUERY, or RANDOM") + public CODDTestModel coddTestModel = CODDTestModel.RANDOM; @Override public List getTestOracleFactory() { diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 4b504a4be..dd51baafd 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -552,7 +552,7 @@ public void visit(SQLite3Values values) { sb.append(vs.get(name).get(i).getDataType().toString()); sb.append("))"); } - isFirstColumn = true; + isFirstColumn = false; } sb.append(")"); if (i < size - 1) { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 049ca7fe4..8198eadb6 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -936,7 +936,7 @@ public boolean useSubquery() { } else if (this.state.getDbmsSpecificOptions().coddTestModel.isSubquery()) { return true; } else { - System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); + System.out.printf("Wrong option of --coddtest-model, should be one of: RANDOM, EXPRESSION, SUBQUERY"); System.exit(1); return false; } diff --git a/test/sqlancer/TestCommonSchema.java b/test/sqlancer/TestCommonSchema.java index 80a57dd8c..c77892124 100644 --- a/test/sqlancer/TestCommonSchema.java +++ b/test/sqlancer/TestCommonSchema.java @@ -4,6 +4,7 @@ import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; import java.util.*; @@ -35,6 +36,12 @@ static class TestSchema extends AbstractSchema, TestTable> } } + static class TestTables extends AbstractTables { + TestTables(List tables) { + super(tables); + } + } + static class TestIndex extends TableIndex { TestIndex(String name) { super(name); @@ -55,6 +62,10 @@ private TestSchema createTestSchema(TestTable... tables) { return new TestSchema(Arrays.asList(tables)); } + private TestTables createTestTables(TestTable... tables) { + return new TestTables(new ArrayList(Arrays.asList(tables))); + } + @Test void testColumnManagement() { TestTable table = createTestTable("products", Collections.emptyList(), false, "sku", "price"); @@ -149,4 +160,27 @@ void testBoundaryConditions() { assertEquals("orphan", col2.getFullQualifiedName()); assertNull(col2.getTable()); } + + @Test + void testTablesManagement() { + TestTable table1 = createTestTable("t1", Collections.emptyList(), false, "col1"); + TestTable table2 = createTestTable("t2", Collections.emptyList(), false, "col2"); + TestTable table3 = createTestTable("t3", Collections.emptyList(), false, "col3"); + + TestTables tables = createTestTables(table1, table2, table3); + assertEquals(3, tables.getSize(), "Should detect 3 tables"); + assertEquals(3, tables.getColumns().size(), "Should detect 3 columns"); + assertTrue(tables.isContained(table3), "Table3 shoule be contained"); + + TestTable table4 = createTestTable("t4", Collections.emptyList(), false, "col4"); + tables.addTable(table4); + assertEquals(4, tables.getSize(), "Should detect 4 tables"); + assertEquals(4, tables.getColumns().size(), "Should detect 4 columns"); + assertTrue(tables.isContained(table4), "Table4 should be contained"); + + tables.removeTable(table4); + assertEquals(3, tables.getSize(), "Should detect 3 tables"); + assertEquals(3, tables.getColumns().size(), "Should detect 3 columns"); + assertTrue(!tables.isContained(table4), "Table4 should not be contained"); + } } diff --git a/test/sqlancer/dbms/TestSQLiteCODDTest.java b/test/sqlancer/dbms/TestSQLiteCODDTest.java new file mode 100644 index 000000000..6697c09c5 --- /dev/null +++ b/test/sqlancer/dbms/TestSQLiteCODDTest.java @@ -0,0 +1,18 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLiteCODDTest { + + @Test + public void testSqliteCODDTest() { + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", + "CODDTest" })); + } +} From c00a19bb52b21247ae2ec862b3421195f87a0306 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Tue, 22 Apr 2025 14:20:26 +0800 Subject: [PATCH 1096/1284] format the CODDTest for SQLite3 code --- src/sqlancer/common/oracle/CODDTestBase.java | 2 +- .../common/schema/AbstractTables.java | 1 - .../sqlite3/SQLite3ExpectedValueVisitor.java | 4 +- src/sqlancer/sqlite3/SQLite3Options.java | 4 + .../sqlite3/SQLite3ToStringVisitor.java | 10 +- src/sqlancer/sqlite3/SQLite3Visitor.java | 3 +- .../sqlite3/ast/SQLite3Expression.java | 32 +-- src/sqlancer/sqlite3/ast/SQLite3Select.java | 6 +- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 248 +++++++++--------- .../sqlite3/schema/SQLite3DataType.java | 27 +- test/sqlancer/TestCommonSchema.java | 2 +- test/sqlancer/dbms/TestSQLiteCODDTest.java | 6 +- 12 files changed, 178 insertions(+), 167 deletions(-) diff --git a/src/sqlancer/common/oracle/CODDTestBase.java b/src/sqlancer/common/oracle/CODDTestBase.java index 162a8eaed..639a4f077 100644 --- a/src/sqlancer/common/oracle/CODDTestBase.java +++ b/src/sqlancer/common/oracle/CODDTestBase.java @@ -22,4 +22,4 @@ public CODDTestBase(S state) { this.logger = state.getLogger(); this.options = state.getOptions(); } -} \ No newline at end of file +} diff --git a/src/sqlancer/common/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java index 67da6364e..ff20fde20 100644 --- a/src/sqlancer/common/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -34,7 +34,6 @@ public String columnNamesAsString(Function function) { return getColumns().stream().map(function).collect(Collectors.joining(", ")); } - public void addTable(T table) { if (!this.tables.contains(table)) { this.tables.add(table); diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 594fe5c92..7ca82ec02 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -22,9 +22,9 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; @@ -310,7 +310,7 @@ public void visit(SQLite3SetClause set) { visit(set.getRight()); } - @Override + @Override public void visit(SQLite3Alias alias) { print(alias); print(alias.getOriginalExpression()); diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index fce9f04ba..e9e34892e 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -81,16 +81,20 @@ public class SQLite3Options implements DBMSSpecificOptions public enum CODDTestModel { RANDOM, EXPRESSION, SUBQUERY; + public boolean isRandom() { return this == RANDOM; } + public boolean isExpression() { return this == EXPRESSION; } + public boolean isSubquery() { return this == SUBQUERY; } } + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on EXPRESSION, SUBQUERY, or RANDOM") public CODDTestModel coddTestModel = CODDTestModel.RANDOM; diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index dd51baafd..8fdbf4438 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -29,9 +29,9 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; @@ -317,7 +317,7 @@ public void visit(InOperation op) { sb.append(")"); } } - + sb.append(")"); } @@ -534,7 +534,7 @@ public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { public void visit(SQLite3Values values) { Map> vs = values.getValues(); int size = vs.get(vs.keySet().iterator().next()).size(); - List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + List columnNames = values.getColumns().stream().map(c -> c.getName()).collect(Collectors.toList()); sb.append("(VALUES "); for (int i = 0; i < size; i++) { sb.append("("); @@ -586,7 +586,7 @@ public void visit(SQLite3ResultMap tableSummary) { if (size == 0) { throw new AssertionError("The result of the expression must not be empty."); } - List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + List columnNames = values.getColumns().stream().map(c -> c.getName()).collect(Collectors.toList()); sb.append(" CASE "); for (int i = 0; i < size; i++) { sb.append("WHEN "); @@ -608,4 +608,4 @@ public void visit(SQLite3ResultMap tableSummary) { } sb.append("END "); } -} \ No newline at end of file +} diff --git a/src/sqlancer/sqlite3/SQLite3Visitor.java b/src/sqlancer/sqlite3/SQLite3Visitor.java index 5e1e5dbf6..02ac2b4d7 100644 --- a/src/sqlancer/sqlite3/SQLite3Visitor.java +++ b/src/sqlancer/sqlite3/SQLite3Visitor.java @@ -21,9 +21,9 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; @@ -137,7 +137,6 @@ default void visit(SQLite3PostfixUnaryOperation exp) { void visit(SQLite3WindowFunctionFrameSpecBetween between); - void visit(SQLite3Alias alias); void visit(SQLite3WithClause withClause); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 681566c03..ad0f64ad6 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1,8 +1,8 @@ package sqlancer.sqlite3.ast; import java.util.List; -import java.util.Optional; import java.util.Map; +import java.util.Optional; import sqlancer.IgnoreMeException; import sqlancer.LikeImplementationHelper; @@ -114,7 +114,7 @@ public SQLite3CollateSequence getImplicitCollateSequence() { public static class SQLite3Exist extends SQLite3Expression { private final SQLite3Expression select; - private boolean negated = false; + private boolean negated; public SQLite3Exist(SQLite3Expression select, boolean negated) { this.select = select; @@ -1565,7 +1565,7 @@ public boolean omitBracketsWhenPrinting() { public static class SQLite3WithClause extends SQLite3Expression { - private SQLite3Expression left; + private final SQLite3Expression left; private SQLite3Expression right; public SQLite3WithClause(SQLite3Expression left, SQLite3Expression right) { @@ -1593,23 +1593,23 @@ public SQLite3CollateSequence getExplicitCollateSequence() { public static class SQLite3Alias extends SQLite3Expression { - private SQLite3Expression originalExpression; - private SQLite3Expression aliasExpression; - + private final SQLite3Expression originalExpression; + private final SQLite3Expression aliasExpression; + public SQLite3Alias(SQLite3Expression originalExpression, SQLite3Expression aliasExpression) { this.originalExpression = originalExpression; this.aliasExpression = aliasExpression; } - + @Override public SQLite3CollateSequence getExplicitCollateSequence() { return null; } - + public SQLite3Expression getOriginalExpression() { return originalExpression; } - + public SQLite3Expression getAliasExpression() { return aliasExpression; } @@ -1673,8 +1673,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } } - - // The ExpressionBag is not a built-in SQL feature, + // The ExpressionBag is not a built-in SQL feature, // but rather a utility class used in CODDTest's oracle construction // to substitute expressions with their corresponding constant values. public static class SQLite3ExpressionBag extends SQLite3Expression { @@ -1700,7 +1699,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } public static class SQLite3Typeof extends SQLite3Expression { - private SQLite3Expression innerExpr; + private final SQLite3Expression innerExpr; public SQLite3Typeof(SQLite3Expression innerExpr) { this.innerExpr = innerExpr; @@ -1714,7 +1713,7 @@ public SQLite3Expression getInnerExpr() { public SQLite3CollateSequence getExplicitCollateSequence() { return null; } - + } public static class SQLite3ResultMap extends SQLite3Expression { @@ -1723,7 +1722,8 @@ public static class SQLite3ResultMap extends SQLite3Expression { private final List summary; private final SQLite3DataType summaryDataType; - public SQLite3ResultMap(SQLite3Values values, List columns, List summary, SQLite3DataType summaryDataType) { + public SQLite3ResultMap(SQLite3Values values, List columns, List summary, + SQLite3DataType summaryDataType) { this.values = values; this.columns = columns; this.summary = summary; @@ -1755,6 +1755,6 @@ public SQLite3DataType getSummaryDataType() { public SQLite3CollateSequence getExplicitCollateSequence() { return null; } - + } -} \ No newline at end of file +} diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index cf35b124a..176057e8a 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -25,7 +25,7 @@ public class SQLite3Select extends SQLite3Expression private List fetchColumns = Collections.emptyList(); private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; - private SQLite3WithClause withClause = null; + private SQLite3WithClause withClause; public SQLite3Select() { } @@ -178,14 +178,14 @@ public SQLite3Expression getWithClause() { } // This method is used in CODDTest to test subquery by replacing a table name - // in the SELECT clause with a derived table expression. + // in the SELECT clause with a derived table expression. public void replaceFromTable(String tableName, SQLite3Expression newFromExpression) { int replaceIdx = -1; for (int i = 0; i < fromList.size(); ++i) { SQLite3Expression f = fromList.get(i); if (f instanceof SQLite3TableReference) { SQLite3TableReference tableRef = (SQLite3TableReference) f; - if (tableRef.getTable().getName() == tableName) { + if (tableRef.getTable().getName().equals(tableName)) { replaceIdx = i; } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 8198eadb6..c262243a9 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -31,20 +31,20 @@ import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; -import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; import sqlancer.sqlite3.ast.SQLite3Select; import sqlancer.sqlite3.gen.SQLite3Common; @@ -61,13 +61,13 @@ public class SQLite3CODDTestOracle extends CODDTestBase impl private SQLite3ExpressionGenerator gen; private Reproducer reproducer; - private String tempTableName = "temp_table"; + private static final String TEMP_TABLE_NAME = "temp_table"; private SQLite3Expression foldedExpr; private SQLite3Expression constantResOfFoldedExpr; private List tablesFromOuterContext = new ArrayList<>(); - private List joinsInExpr = null; + private List joinsInExpr; Map> auxiliaryQueryResult = new HashMap<>(); Map> selectResult = new HashMap<>(); @@ -118,15 +118,14 @@ public void check() throws SQLException { auxiliaryQueryResult.putAll(selectResult); } - SQLite3Select originalQuery = null; - + Map> foldedResult = new HashMap<>(); Map> originalResult = new HashMap<>(); // dependent expression - if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) { // original query SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(this.foldedExpr); originalQuery = this.genSelectExpression(null, specificCondition); @@ -137,69 +136,71 @@ public void check() throws SQLException { specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } - // independent expression - // empty result, put the inner query in (NOT) EXIST - else if (auxiliaryQueryResult.size() == 0 || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).size() == 0) { - boolean isNegated = Randomly.getBoolean() ? false : true; + } else if (auxiliaryQueryResult.isEmpty() + || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).isEmpty()) { + // independent expression + // empty result, put the inner query in (NOT) EXIST + boolean isNegated = !Randomly.getBoolean(); // original query SQLite3Exist existExpr = new SQLite3Exist(new SQLite3Select(auxiliaryQuery), isNegated); SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(existExpr); - + originalQuery = this.genSelectExpression(null, specificCondition); originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); - + // folded query SQLite3Expression equivalentExpr = isNegated ? SQLite3Constant.createTrue() : SQLite3Constant.createFalse(); specificCondition.updateInnerExpr(equivalentExpr); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } - // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant - else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 && Randomly.getBoolean()) { + } else if (auxiliaryQueryResult.size() == 1 + && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 + && Randomly.getBoolean()) { + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant // original query SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(auxiliaryQuery); originalQuery = this.genSelectExpression(null, specificCondition); originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); - + // folded query - SQLite3Expression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); - specificCondition.updateInnerExpr(equivalentExpr);; + SQLite3Expression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]) + .get(0); + specificCondition.updateInnerExpr(equivalentExpr); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } - // one column - else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability() && testInOperator()) { + } else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability() + && enableInOperator()) { + // one column // original query List columns = s.getRandomTableNonEmptyTables().getColumns(); SQLite3ColumnName selectedColumn = new SQLite3ColumnName(Randomly.fromList(columns), null); SQLite3Table selectedTable = selectedColumn.getColumn().getTable(); - InOperation INOperation = new InOperation(selectedColumn, new SQLite3Select(auxiliaryQuery)); - SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(INOperation); + InOperation inOperation = new InOperation(selectedColumn, new SQLite3Select(auxiliaryQuery)); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(inOperation); originalQuery = this.genSelectExpression(selectedTable, specificCondition); originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); // folded query - // can not use IN VALUES here, because there is no affinity for the right operand of IN when right operand is a list + // can not use IN VALUES here, because there is no affinity for the right operand of IN when right operand + // is a list try { SQLite3Table t = this.createTemporaryTable(auxiliaryQuery, "intable"); SQLite3TableReference equivalentTable = new SQLite3TableReference(t); - INOperation = new InOperation(selectedColumn, equivalentTable); - specificCondition.updateInnerExpr(INOperation); + inOperation = new InOperation(selectedColumn, equivalentTable); + specificCondition.updateInnerExpr(inOperation); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } finally { dropTemporaryTable("intable"); } - } - // There is not `ANY` and `ALL` operator in SQLite3 - // Row Subquery - else { + } else { + // There is not `ANY` and `ALL` operator in SQLite3 + // Row Subquery // original query - SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); + SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.TEMP_TABLE_NAME); originalQuery = this.genSelectExpression(temporaryTable, null); SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); SQLite3WithClause withClause = new SQLite3WithClause(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); @@ -207,7 +208,7 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); // folded query - if (Randomly.getBoolean() && this.testCommonTableExpression()) { + if (Randomly.getBoolean() && this.enableCommonTableExpression()) { // there are too many false positives // common table expression // folded query: WITH table AS VALUES () @@ -215,25 +216,25 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro originalQuery.updateWithClauseRight(values); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } else if (Randomly.getBoolean() && this.testDerivedTable()) { + } else if (Randomly.getBoolean() && this.enableDerivedTable()) { // derived table // folded query: SELECT FROM () AS table originalQuery.setWithClause(null); SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); - originalQuery.replaceFromTable(this.tempTableName, alias); + originalQuery.replaceFromTable(this.TEMP_TABLE_NAME, alias); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } else if (this.testInsert()){ + } else if (this.enableInsert()) { // there are too many false positives // folded query: CREATE the table and INSERT INTO table subquery try { - this.createTemporaryTable(auxiliaryQuery, this.tempTableName); + this.createTemporaryTable(auxiliaryQuery, this.TEMP_TABLE_NAME); originalQuery.setWithClause(null); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } finally { - dropTemporaryTable(this.tempTableName); + dropTemporaryTable(this.TEMP_TABLE_NAME); } } else { throw new IgnoreMeException(); @@ -247,8 +248,10 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro } if (!compareResult(foldedResult, originalResult)) { reproducer = null; // TODO - state.getState().getLocalState().log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); - throw new AssertionError(auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); + state.getState().getLocalState() + .log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError( + auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); } } @@ -257,7 +260,7 @@ private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Express if (tempTable != null) { randomTables.addTable(tempTable); } - if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) { for (SQLite3Table t : this.tablesFromOuterContext) { randomTables.addTable(t); } @@ -270,7 +273,8 @@ private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Express } List columns = randomTables.getColumns(); - if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + if ((!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) + && this.joinsInExpr != null) { for (Join j : this.joinsInExpr) { SQLite3Table t = j.getTable(); columns.addAll(t.getColumns()); @@ -279,34 +283,36 @@ private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Express gen = new SQLite3ExpressionGenerator(state).setColumns(columns); List tables = randomTables.getTables(); List joinStatements = new ArrayList<>(); - if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) { if (this.joinsInExpr != null) { joinStatements.addAll(this.joinsInExpr); this.joinsInExpr = null; } - } - else if (Randomly.getBoolean()) { - joinStatements = genJoinExpression(gen, tables, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); + } else if (Randomly.getBoolean()) { + joinStatements = genJoinExpression(gen, tables, + Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); } List tableRefs = SQLite3Common.getTableRefs(tables, s); SQLite3Select select = new SQLite3Select(); select.setFromList(tableRefs); - if (joinStatements.size() > 0) { + if (!joinStatements.isEmpty()) { select.setJoinClauses(joinStatements); } - + SQLite3Expression randomWhereCondition = gen.generateExpression(); SQLite3Expression whereCondition = null; if (specificCondition != null) { BinaryOperator operator = BinaryOperator.getRandomOperator(); - whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomWhereCondition, specificCondition, operator); + whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomWhereCondition, specificCondition, + operator); } else { whereCondition = randomWhereCondition; } select.setWhereClause(whereCondition); - + if (Randomly.getBoolean()) { - select.setOrderByClauses(genOrderBysExpression(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + select.setOrderByClauses(genOrderBysExpression(gen, + Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); } if (Randomly.getBoolean()) { @@ -314,7 +320,7 @@ else if (Randomly.getBoolean()) { List selectedAlias = new LinkedList<>(); for (int i = 0; i < selectedColumns.size(); ++i) { SQLite3ColumnName originalName = new SQLite3ColumnName(selectedColumns.get(i), null); - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(i)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + i), null); SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); selectedAlias.add(columnAlias); } @@ -322,14 +328,15 @@ else if (Randomly.getBoolean()) { } else { SQLite3ColumnName aggr = new SQLite3ColumnName(Randomly.fromList(columns), null); SQLite3Provider.mustKnowResult = true; - SQLite3Expression originalName = new SQLite3Aggregate(Arrays.asList(aggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + SQLite3Expression originalName = new SQLite3Aggregate(Arrays.asList(aggr), + SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c0"), null); SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); select.setFetchColumns(Arrays.asList(columnAlias)); if (Randomly.getBooleanWithRatherLowProbability()) { List groupByClause = genGroupByClause(columns, specificCondition); select.setGroupByClause(groupByClause); - if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + if (!groupByClause.isEmpty() && Randomly.getBooleanWithRatherLowProbability()) { select.setHavingClause(genHavingClause(columns, specificCondition)); } } @@ -349,13 +356,13 @@ private SQLite3Select genSimpleSelect() { if (Randomly.getBooleanWithRatherLowProbability()) { joinsInExpr = genJoinExpression(gen, tables, null, true); } else { - joinsInExpr = new ArrayList(); + joinsInExpr = new ArrayList<>(); } List tableRefs = SQLite3Common.getTableRefs(tables, s); SQLite3Select select = new SQLite3Select(); select.setFromList(tableRefs); - if (joinsInExpr != null && joinsInExpr.size() > 0) { + if (joinsInExpr != null && !joinsInExpr.isEmpty()) { select.setJoinClauses(joinsInExpr); } @@ -366,14 +373,14 @@ private SQLite3Select genSimpleSelect() { int columnIdx = 0; for (SQLite3Column c : randomTables.getColumns()) { SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); fetchColumns.add(columnAlias); columnIdx++; } // add the expression to fetch clause - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); SQLite3Alias columnAlias = new SQLite3Alias(whereCondition, aliasName); fetchColumns.add(columnAlias); @@ -388,7 +395,7 @@ private SQLite3Select genSimpleSelect() { } else { throw new AssertionError(e.getMessage()); } - } + } if (queryRes.get("c0").size() == 0) { throw new IgnoreMeException(); } @@ -398,7 +405,7 @@ private SQLite3Select genSimpleSelect() { selectResult.putAll(queryRes); // get the summary from results - List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + List summary = queryRes.remove("c" + columnIdx); List tempColumnList = new ArrayList<>(); @@ -443,13 +450,15 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { SQLite3Column newColumn = new SQLite3Column(c.getName(), c.getType(), false, null, false); newColumns.add(newColumn); } - SQLite3Table newTable = new SQLite3Table(t.getName() + "a", newColumns, null, true, false, false, false); + SQLite3Table newTable = new SQLite3Table(t.getName() + "a", newColumns, null, true, false, false, + false); for (SQLite3Column c : newColumns) { c.setTable(newTable); } innerQueryRandomTables.addTable(newTable); - - SQLite3Alias alias = new SQLite3Alias(new SQLite3TableReference(t), new SQLite3TableReference(newTable)); + + SQLite3Alias alias = new SQLite3Alias(new SQLite3TableReference(t), + new SQLite3TableReference(newTable)); innerQueryFromTables.add(alias); } } @@ -466,20 +475,21 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { innerQuery.setWhereClause(innerQueryWhereCondition); // use aggregate function in fetch column - SQLite3ColumnName innerQueryAggr = new SQLite3ColumnName(Randomly.fromList(innerQueryRandomTables.getColumns()), null); + SQLite3ColumnName innerQueryAggr = new SQLite3ColumnName(Randomly.fromList(innerQueryRandomTables.getColumns()), + null); SQLite3Provider.mustKnowResult = true; - SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), + SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); if (Randomly.getBooleanWithRatherLowProbability()) { List groupByClause = genGroupByClause(innerQueryColumns, null); innerQuery.setGroupByClause(groupByClause); - if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + if (!groupByClause.isEmpty() && Randomly.getBooleanWithRatherLowProbability()) { innerQuery.setHavingClause(genHavingClause(innerQueryColumns, null)); } } - this.foldedExpr = innerQuery; - + this.foldedExpr = innerQuery; // outer query SQLite3Select outerQuery = new SQLite3Select(); @@ -490,14 +500,14 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { int columnIdx = 0; for (SQLite3Column c : outerQueryRandomTables.getColumns()) { SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); outerQueryFetchColumns.add(columnAlias); columnIdx++; } // add the expression to fetch clause - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); SQLite3Alias columnAlias = new SQLite3Alias(innerQuery, aliasName); outerQueryFetchColumns.add(columnAlias); @@ -514,7 +524,7 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { } else { throw new AssertionError(e.getMessage()); } - } + } if (queryRes.get("c0").size() == 0) { throw new IgnoreMeException(); } @@ -524,7 +534,7 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { selectResult.putAll(queryRes); // get the summary from results - List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + List summary = queryRes.remove("c" + columnIdx); List tempColumnList = new ArrayList<>(); @@ -550,7 +560,8 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { return outerQuery; } - private List genJoinExpression(SQLite3ExpressionGenerator gen, List tables, SQLite3Expression specificCondition, boolean joinForExperssion) { + private List genJoinExpression(SQLite3ExpressionGenerator gen, List tables, + SQLite3Expression specificCondition, boolean joinForExperssion) { List joinStatements = new ArrayList<>(); if (!state.getDbmsSpecificOptions().testJoins) { return joinStatements; @@ -569,7 +580,8 @@ private List genJoinExpression(SQLite3ExpressionGenerator gen, List genJoinExpression(SQLite3ExpressionGenerator gen, List genOrderBysExpression(SQLite3ExpressionGenerator gen, SQLite3Expression specificCondition) { + private List genOrderBysExpression(SQLite3ExpressionGenerator gen, + SQLite3Expression specificCondition) { List expressions = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - expressions.add(genOrderingTerm(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + expressions.add( + genOrderingTerm(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); } return expressions; } @@ -645,7 +659,8 @@ private SQLite3Expression genHavingClause(List columns, SQLite3Ex return expr; } - private Map> getQueryResult(String queryString, SQLite3GlobalState state) throws SQLException { + private Map> getQueryResult(String queryString, SQLite3GlobalState state) + throws SQLException { Map> result = new LinkedHashMap<>(); if (options.logEachSelect()) { logger.writeCurrentNoLineBreak(queryString); @@ -661,8 +676,8 @@ private Map> getQueryResult(String queryString, SQ Integer columnCount = metaData.getColumnCount(); Map idxNameMap = new HashMap<>(); for (int i = 1; i <= columnCount; i++) { - result.put("c" + String.valueOf(i-1), new ArrayList<>()); - idxNameMap.put(i, "c" + String.valueOf(i-1)); + result.put("c" + (i - 1), new ArrayList<>()); + idxNameMap.put(i, "c" + (i - 1)); } int resultRows = 0; @@ -673,26 +688,18 @@ private Map> getQueryResult(String queryString, SQ SQLite3Constant constant; if (rs.wasNull()) { constant = SQLite3Constant.createNullConstant(); - } - - else if (value instanceof Integer) { + } else if (value instanceof Integer) { constant = SQLite3Constant.createIntConstant(Long.valueOf((Integer) value)); } else if (value instanceof Short) { constant = SQLite3Constant.createIntConstant(Long.valueOf((Short) value)); } else if (value instanceof Long) { constant = SQLite3Constant.createIntConstant((Long) value); - } - - else if (value instanceof Double) { + } else if (value instanceof Double) { constant = SQLite3Constant.createRealConstant((double) value); } else if (value instanceof Float) { constant = SQLite3Constant.createRealConstant(((Float) value).doubleValue()); } else if (value instanceof BigDecimal) { constant = SQLite3Constant.createRealConstant(((BigDecimal) value).doubleValue()); - } - - else if (value instanceof Byte) { - constant = SQLite3Constant.createBinaryConstant((byte[]) value); } else if (value instanceof byte[]) { constant = SQLite3Constant.createBinaryConstant((byte[]) value); } else if (value instanceof Boolean) { @@ -746,7 +753,7 @@ private SQLite3Table genTemporaryTable(SQLite3Select select, String tableName) { List databaseColumns = new ArrayList<>(); for (int i = 0; i < columnNumber; ++i) { - String columnName = "c" + String.valueOf(i); + String columnName = "c" + i; SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); databaseColumns.add(column); } @@ -769,19 +776,19 @@ private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName String columnTypeName = ""; if (idxTypeMap.get(i) != null) { switch (idxTypeMap.get(i)) { - case INT: - case TEXT: - case REAL: - columnTypeName = idxTypeMap.get(i).name(); - break; - case BINARY: - columnTypeName = ""; - break; - default: - columnTypeName = ""; + case INT: + case TEXT: + case REAL: + columnTypeName = idxTypeMap.get(i).name(); + break; + case BINARY: + columnTypeName = ""; + break; + default: + columnTypeName = ""; } } - sb.append("c" + String.valueOf(i) + " " + columnTypeName); + sb.append("c" + i + " " + columnTypeName); if (i < columnNumber - 1) { sb.append(", "); } @@ -808,7 +815,7 @@ private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName } StringBuilder sb2 = new StringBuilder(); - sb2.append("INSERT INTO " + tableName + " "+ selectString); + sb2.append("INSERT INTO " + tableName + " " + selectString); String insertValueString = sb2.toString(); if (options.logEachSelect()) { logger.writeCurrent(insertValueString); @@ -831,7 +838,7 @@ private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName List databaseColumns = new ArrayList<>(); for (int i = 0; i < columnNumber; ++i) { - String columnName = "c" + String.valueOf(i); + String columnName = "c" + i; SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); databaseColumns.add(column); } @@ -869,20 +876,20 @@ private boolean compareResult(Map> r1, Map > entry: r1.entrySet()) { + for (Map.Entry> entry : r1.entrySet()) { String currentKey = entry.getKey(); if (!r2.containsKey(currentKey)) { return false; - } - List v1= entry.getValue(); - List v2= r2.get(currentKey); + } + List v1 = entry.getValue(); + List v2 = r2.get(currentKey); if (v1.size() != v2.size()) { return false; } List v1Value = new ArrayList<>(v1.stream().map(c -> c.toString()).collect(Collectors.toList())); List v2Value = new ArrayList<>(v2.stream().map(c -> c.toString()).collect(Collectors.toList())); Collections.sort(v1Value); - Collections.sort(v2Value); + Collections.sort(v2Value); if (!v1Value.equals(v2Value)) { return false; } @@ -893,7 +900,7 @@ private boolean compareResult(Map> r1, Map getColumnTypeFromSelect(SQLite3Select select) { List fetchColumns = select.getFetchColumns(); List newFetchColumns = new ArrayList<>(); - for(SQLite3Expression column : fetchColumns) { + for (SQLite3Expression column : fetchColumns) { newFetchColumns.add(column); SQLite3Alias columnAlias = (SQLite3Alias) column; SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOriginalExpression()); @@ -910,14 +917,14 @@ private Map getColumnTypeFromSelect(SQLite3Select sele } else { throw new AssertionError(e.getMessage()); } - } + } if (typeResult == null) { throw new IgnoreMeException(); } Map idxTypeMap = new HashMap<>(); for (int i = 0; i * 2 < typeResult.size(); ++i) { - String columnName = "c" + String.valueOf(i * 2 + 1); + String columnName = "c" + (i * 2 + 1); SQLite3Expression t = typeResult.get(columnName).get(0); SQLite3TextConstant tString = (SQLite3TextConstant) t; String typeName = tString.asString(); @@ -946,16 +953,19 @@ public boolean useCorrelatedSubquery() { return Randomly.getBoolean(); } - public boolean testCommonTableExpression() { + public boolean enableCommonTableExpression() { return false; } - public boolean testDerivedTable() { + + public boolean enableDerivedTable() { return true; } - public boolean testInsert() { + + public boolean enableInsert() { return false; } - public boolean testInOperator() { + + public boolean enableInOperator() { return false; } @@ -968,4 +978,4 @@ public String getLastQueryString() { public Reproducer getLastReproducer() { return reproducer; } -} \ No newline at end of file +} diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index 556976056..b7145b98c 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -21,20 +21,21 @@ public static SQLite3DataType getTypeFromName(String name) { } } + @Override public String toString() { - switch(this) { - case BINARY: - return "BLOB"; - case INT: - return "INT"; - case TEXT: - return "TEXT"; - case REAL: - return "REAL"; - case NULL: - return "NULL"; - default: - throw new IgnoreMeException(); + switch (this) { + case BINARY: + return "BLOB"; + case INT: + return "INT"; + case TEXT: + return "TEXT"; + case REAL: + return "REAL"; + case NULL: + return "NULL"; + default: + throw new IgnoreMeException(); } } } diff --git a/test/sqlancer/TestCommonSchema.java b/test/sqlancer/TestCommonSchema.java index c77892124..f3a2ee2ba 100644 --- a/test/sqlancer/TestCommonSchema.java +++ b/test/sqlancer/TestCommonSchema.java @@ -63,7 +63,7 @@ private TestSchema createTestSchema(TestTable... tables) { } private TestTables createTestTables(TestTable... tables) { - return new TestTables(new ArrayList(Arrays.asList(tables))); + return new TestTables(new ArrayList(Arrays.asList(tables))); } @Test diff --git a/test/sqlancer/dbms/TestSQLiteCODDTest.java b/test/sqlancer/dbms/TestSQLiteCODDTest.java index 6697c09c5..c1948a9b8 100644 --- a/test/sqlancer/dbms/TestSQLiteCODDTest.java +++ b/test/sqlancer/dbms/TestSQLiteCODDTest.java @@ -10,9 +10,7 @@ public class TestSQLiteCODDTest { @Test public void testSqliteCODDTest() { - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", - "CODDTest" })); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", "CODDTest" })); } } From dcdf7d641903051a65b8153f01c7c7cf44a0254d Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Wed, 23 Apr 2025 19:11:09 +0800 Subject: [PATCH 1097/1284] remove three errors, and fix a warning about the using of static variable --- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index c262243a9..532709676 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -81,13 +81,13 @@ public SQLite3CODDTestOracle(SQLite3GlobalState globalState) { SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addMatchQueryErrors(errors); SQLite3Errors.addQueryErrors(errors); - errors.add("misuse of aggregate"); - errors.add("misuse of window function"); - errors.add("second argument to nth_value must be a positive integer"); - errors.add("no such table"); - errors.add("no query solution"); - errors.add("unable to use function MATCH in the requested context"); - errors.add("[SQLITE_ERROR] SQL error or missing database (unrecognized token:"); + // errors.add("misuse of aggregate"); + // errors.add("misuse of window function"); + // errors.add("second argument to nth_value must be a positive integer"); + // errors.add("no such table"); + // errors.add("no query solution"); + // errors.add("unable to use function MATCH in the requested context"); + // errors.add("[SQLITE_ERROR] SQL error or missing database (unrecognized token:"); } @Override @@ -200,7 +200,7 @@ && enableInOperator()) { // There is not `ANY` and `ALL` operator in SQLite3 // Row Subquery // original query - SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.TEMP_TABLE_NAME); + SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, SQLite3CODDTestOracle.TEMP_TABLE_NAME); originalQuery = this.genSelectExpression(temporaryTable, null); SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); SQLite3WithClause withClause = new SQLite3WithClause(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); @@ -222,19 +222,19 @@ && enableInOperator()) { originalQuery.setWithClause(null); SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); - originalQuery.replaceFromTable(this.TEMP_TABLE_NAME, alias); + originalQuery.replaceFromTable(SQLite3CODDTestOracle.TEMP_TABLE_NAME, alias); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } else if (this.enableInsert()) { // there are too many false positives // folded query: CREATE the table and INSERT INTO table subquery try { - this.createTemporaryTable(auxiliaryQuery, this.TEMP_TABLE_NAME); + this.createTemporaryTable(auxiliaryQuery, SQLite3CODDTestOracle.TEMP_TABLE_NAME); originalQuery.setWithClause(null); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } finally { - dropTemporaryTable(this.TEMP_TABLE_NAME); + dropTemporaryTable(SQLite3CODDTestOracle.TEMP_TABLE_NAME); } } else { throw new IgnoreMeException(); From 597b43ab3f94fd36b71e19f256edc5850a34587d Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Wed, 23 Apr 2025 23:09:46 +0800 Subject: [PATCH 1098/1284] remove the redundant code --- .../sqlite3/schema/SQLite3DataType.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index b7145b98c..8a343fed2 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -20,22 +20,4 @@ public static SQLite3DataType getTypeFromName(String name) { throw new IgnoreMeException(); } } - - @Override - public String toString() { - switch (this) { - case BINARY: - return "BLOB"; - case INT: - return "INT"; - case TEXT: - return "TEXT"; - case REAL: - return "REAL"; - case NULL: - return "NULL"; - default: - throw new IgnoreMeException(); - } - } } From b2d028434f63b10ab9c82757bd6c989a41a484d8 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 24 Apr 2025 00:16:45 +0800 Subject: [PATCH 1099/1284] format hive code --- src/sqlancer/hive/HiveOptions.java | 4 +- src/sqlancer/hive/HiveProvider.java | 12 +- src/sqlancer/hive/HiveToStringVisitor.java | 40 +++--- .../hive/ast/HiveBetweenOperation.java | 8 +- .../hive/ast/HiveBinaryOperation.java | 5 +- src/sqlancer/hive/ast/HiveCaseOperation.java | 2 +- src/sqlancer/hive/ast/HiveCastOperation.java | 8 +- src/sqlancer/hive/ast/HiveFunction.java | 2 +- src/sqlancer/hive/ast/HiveInOperation.java | 2 +- src/sqlancer/hive/ast/HiveJoin.java | 2 +- src/sqlancer/hive/ast/HiveSelect.java | 2 +- src/sqlancer/hive/ast/HiveTableReference.java | 2 +- .../hive/ast/HiveUnaryPostfixOperation.java | 3 +- .../hive/ast/HiveUnaryPrefixOperation.java | 5 +- .../hive/gen/HiveExpressionGenerator.java | 129 +++++++----------- src/sqlancer/hive/gen/HiveTableGenerator.java | 52 +++---- test/sqlancer/dbms/TestHiveTLP.java | 8 +- 17 files changed, 127 insertions(+), 159 deletions(-) diff --git a/src/sqlancer/hive/HiveOptions.java b/src/sqlancer/hive/HiveOptions.java index a8c049756..5d2ea47e6 100644 --- a/src/sqlancer/hive/HiveOptions.java +++ b/src/sqlancer/hive/HiveOptions.java @@ -28,8 +28,8 @@ public enum HiveOracleFactory implements OracleFactory { @Override public TestOracle create(HiveGlobalState globalState) throws SQLException { HiveExpressionGenerator gen = new HiveExpressionGenerator(globalState); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors() - .with(HiveErrors.getExpressionErrors()).build(); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HiveErrors.getExpressionErrors()) + .build(); return new TLPWhereOracle<>(globalState, gen, expectedErrors); } diff --git a/src/sqlancer/hive/HiveProvider.java b/src/sqlancer/hive/HiveProvider.java index cf1df5410..723e3fd6f 100644 --- a/src/sqlancer/hive/HiveProvider.java +++ b/src/sqlancer/hive/HiveProvider.java @@ -61,15 +61,14 @@ public void generateDatabase(HiveGlobalState globalState) throws Exception { String tableName = globalState.getSchema().getFreeTableName(); SQLQueryAdapter qt = HiveTableGenerator.generate(globalState, tableName); success = globalState.executeStatement(qt); - } while(!success); + } while (!success); } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); // TODO } - StatementExecutor se = new StatementExecutor( - globalState, Action.values(), - HiveProvider::mapActions, (q) -> { + StatementExecutor se = new StatementExecutor(globalState, + Action.values(), HiveProvider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } @@ -107,9 +106,8 @@ public SQLConnection createDatabase(HiveGlobalState globalState) throws SQLExcep s.execute("USE " + databaseName); } con.close(); - con = DriverManager.getConnection( - String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName, - username, password)); + con = DriverManager + .getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName, username, password)); return new SQLConnection(con); } diff --git a/src/sqlancer/hive/HiveToStringVisitor.java b/src/sqlancer/hive/HiveToStringVisitor.java index 3815561d8..d23818916 100644 --- a/src/sqlancer/hive/HiveToStringVisitor.java +++ b/src/sqlancer/hive/HiveToStringVisitor.java @@ -71,26 +71,26 @@ private void visit(HiveSelect select) { private void visit(HiveJoin join) { switch (join.getJoinType()) { - case INNER: - sb.append(" INNER JOIN "); - break; - case LEFT_OUTER: - sb.append(" LEFT JOIN "); - break; - case RIGHT_OUTER: - sb.append(" RIGHT JOIN "); - break; - case FULL_OUTER: - sb.append(" FULL JOIN "); - break; - case LEFT_SEMI: - sb.append(" LEFT SEMI JOIN "); - break; - case CROSS: - sb.append(" CROSS JOIN "); - break; - default: - throw new UnsupportedOperationException(); + case INNER: + sb.append(" INNER JOIN "); + break; + case LEFT_OUTER: + sb.append(" LEFT JOIN "); + break; + case RIGHT_OUTER: + sb.append(" RIGHT JOIN "); + break; + case FULL_OUTER: + sb.append(" FULL JOIN "); + break; + case LEFT_SEMI: + sb.append(" LEFT SEMI JOIN "); + break; + case CROSS: + sb.append(" CROSS JOIN "); + break; + default: + throw new UnsupportedOperationException(); } visit((TableReferenceNode) join.getRightTable()); if (join.getOnClause() != null) { diff --git a/src/sqlancer/hive/ast/HiveBetweenOperation.java b/src/sqlancer/hive/ast/HiveBetweenOperation.java index a65ef70d2..26ec1d940 100644 --- a/src/sqlancer/hive/ast/HiveBetweenOperation.java +++ b/src/sqlancer/hive/ast/HiveBetweenOperation.java @@ -2,11 +2,9 @@ import sqlancer.common.ast.newast.NewBetweenOperatorNode; -public class HiveBetweenOperation extends NewBetweenOperatorNode - implements HiveExpression { - - public HiveBetweenOperation(HiveExpression left, HiveExpression middle, HiveExpression right, - boolean isTrue) { +public class HiveBetweenOperation extends NewBetweenOperatorNode implements HiveExpression { + + public HiveBetweenOperation(HiveExpression left, HiveExpression middle, HiveExpression right, boolean isTrue) { super(left, middle, right, isTrue); } } diff --git a/src/sqlancer/hive/ast/HiveBinaryOperation.java b/src/sqlancer/hive/ast/HiveBinaryOperation.java index 61d9cf104..f74d117d4 100644 --- a/src/sqlancer/hive/ast/HiveBinaryOperation.java +++ b/src/sqlancer/hive/ast/HiveBinaryOperation.java @@ -3,9 +3,8 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -public class HiveBinaryOperation extends NewBinaryOperatorNode - implements HiveExpression { - +public class HiveBinaryOperation extends NewBinaryOperatorNode implements HiveExpression { + public HiveBinaryOperation(HiveExpression left, HiveExpression right, Operator op) { super(left, right, op); } diff --git a/src/sqlancer/hive/ast/HiveCaseOperation.java b/src/sqlancer/hive/ast/HiveCaseOperation.java index 2a52a5c45..666070667 100644 --- a/src/sqlancer/hive/ast/HiveCaseOperation.java +++ b/src/sqlancer/hive/ast/HiveCaseOperation.java @@ -5,7 +5,7 @@ import sqlancer.common.ast.newast.NewCaseOperatorNode; public class HiveCaseOperation extends NewCaseOperatorNode implements HiveExpression { - + public HiveCaseOperation(HiveExpression switchCondition, List conditions, List expressions, HiveExpression elseExpr) { super(switchCondition, conditions, expressions, elseExpr); diff --git a/src/sqlancer/hive/ast/HiveCastOperation.java b/src/sqlancer/hive/ast/HiveCastOperation.java index ae2fbe435..2d76ab4f2 100644 --- a/src/sqlancer/hive/ast/HiveCastOperation.java +++ b/src/sqlancer/hive/ast/HiveCastOperation.java @@ -3,10 +3,10 @@ import sqlancer.hive.HiveSchema.HiveDataType; public class HiveCastOperation implements HiveExpression { - + private final HiveExpression expression; private final HiveDataType type; - + public HiveCastOperation(HiveExpression expression, HiveDataType type) { if (expression == null) { throw new AssertionError(); @@ -14,11 +14,11 @@ public HiveCastOperation(HiveExpression expression, HiveDataType type) { this.expression = expression; this.type = type; } - + public HiveExpression getExpression() { return expression; } - + public HiveDataType getType() { return type; } diff --git a/src/sqlancer/hive/ast/HiveFunction.java b/src/sqlancer/hive/ast/HiveFunction.java index 507015259..b3a34ae25 100644 --- a/src/sqlancer/hive/ast/HiveFunction.java +++ b/src/sqlancer/hive/ast/HiveFunction.java @@ -9,5 +9,5 @@ public class HiveFunction extends NewFunctionNode implemen public HiveFunction(List args, F func) { super(args, func); } - + } diff --git a/src/sqlancer/hive/ast/HiveInOperation.java b/src/sqlancer/hive/ast/HiveInOperation.java index ccffde58c..601bf5e19 100644 --- a/src/sqlancer/hive/ast/HiveInOperation.java +++ b/src/sqlancer/hive/ast/HiveInOperation.java @@ -5,7 +5,7 @@ import sqlancer.common.ast.newast.NewInOperatorNode; public class HiveInOperation extends NewInOperatorNode implements HiveExpression { - + public HiveInOperation(HiveExpression left, List right, boolean isNegated) { super(left, right, isNegated); } diff --git a/src/sqlancer/hive/ast/HiveJoin.java b/src/sqlancer/hive/ast/HiveJoin.java index fc6b7a200..932ed9afa 100644 --- a/src/sqlancer/hive/ast/HiveJoin.java +++ b/src/sqlancer/hive/ast/HiveJoin.java @@ -17,7 +17,7 @@ public enum JoinType { INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, CROSS; } - public HiveJoin(HiveTableReference leftTable, HiveTableReference rightTable, JoinType joinType, + public HiveJoin(HiveTableReference leftTable, HiveTableReference rightTable, JoinType joinType, HiveExpression onClause) { this.leftTable = leftTable; this.rightTable = rightTable; diff --git a/src/sqlancer/hive/ast/HiveSelect.java b/src/sqlancer/hive/ast/HiveSelect.java index 4853ef892..b099ade02 100644 --- a/src/sqlancer/hive/ast/HiveSelect.java +++ b/src/sqlancer/hive/ast/HiveSelect.java @@ -8,7 +8,7 @@ import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveTable; -public class HiveSelect extends SelectBase +public class HiveSelect extends SelectBase implements Select, HiveExpression { private boolean isDistinct; diff --git a/src/sqlancer/hive/ast/HiveTableReference.java b/src/sqlancer/hive/ast/HiveTableReference.java index a0955f86c..a1b586825 100644 --- a/src/sqlancer/hive/ast/HiveTableReference.java +++ b/src/sqlancer/hive/ast/HiveTableReference.java @@ -3,7 +3,7 @@ import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.hive.HiveSchema; -public class HiveTableReference extends TableReferenceNode +public class HiveTableReference extends TableReferenceNode implements HiveExpression { public HiveTableReference(HiveSchema.HiveTable table) { diff --git a/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java index c32a40847..0461c5c73 100644 --- a/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java +++ b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java @@ -3,8 +3,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -public class HiveUnaryPostfixOperation extends NewUnaryPostfixOperatorNode - implements HiveExpression { +public class HiveUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements HiveExpression { public HiveUnaryPostfixOperation(HiveExpression expr, Operator op) { super(expr, op); diff --git a/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java index 9312c4c26..9fe286f82 100644 --- a/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java +++ b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java @@ -3,11 +3,10 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -public class HiveUnaryPrefixOperation extends NewUnaryPrefixOperatorNode - implements HiveExpression { +public class HiveUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements HiveExpression { public HiveUnaryPrefixOperation(HiveExpression expr, Operator op) { super(expr, op); } - + } diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index 3e35544c5..bd9134de6 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.stream.Collectors; -public class HiveExpressionGenerator extends UntypedExpressionGenerator +public class HiveExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator { private final HiveGlobalState globalState; @@ -37,8 +37,8 @@ public class HiveExpressionGenerator extends UntypedExpressionGenerator= globalState.getOptions().getMaxExpressionDepth() + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBooleanWithRatherLowProbability()) { return generateLeafNode(); } if (allowAggregates && Randomly.getBooleanWithRatherLowProbability()) { - allowAggregates= false; // aggregate function calls cannot be nested + allowAggregates = false; // aggregate function calls cannot be nested HiveAggregateFunction aggregate = HiveAggregateFunction.getRandom(); return new HiveFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); } @@ -77,46 +77,36 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr Expression expr = Randomly.fromList(possibleOptions); switch (expr) { - case UNARY_PREFIX: - return new HiveUnaryPrefixOperation(generateExpression(depth + 1), - HiveUnaryPrefixOperator.getRandom()); - case UNARY_POSTFIX: - return new HiveUnaryPostfixOperation(generateExpression(depth + 1), - HiveUnaryPostfixOperator.getRandom()); - case BINARY_COMPARISON: - Operator op = HiveBinaryComparisonOperator.getRandom(); - return new HiveBinaryOperation(generateExpression(depth + 1), - generateExpression(depth + 1), op); - case BINARY_LOGICAL: - op = HiveExpressionGenerator.HiveBinaryLogicalOperator.getRandom(); - return new HiveBinaryOperation(generateExpression(depth + 1), - generateExpression(depth + 1), op); - case BINARY_ARITHMETIC: - return new HiveBinaryOperation(generateExpression(depth + 1), - generateExpression(depth + 1), - HiveExpressionGenerator.HiveBinaryArithmeticOperator.getRandom()); - case CAST: - return new HiveCastOperation(generateExpression(depth + 1), HiveDataType.getRandomType()); - case FUNC: - HiveFunc func = HiveFunc.getRandom(); - return new HiveFunction<>(generateExpressions(func.getNrArgs()), func); - case BETWEEN: - return new HiveBetweenOperation(generateExpression(depth + 1), - generateExpression(depth + 1), - generateExpression(depth + 1), - Randomly.getBoolean()); - case IN: - return new HiveInOperation(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), - Randomly.getBoolean()); - case CASE: - int nr = Randomly.smallNumber() + 1; - return new HiveCaseOperation(generateExpression(depth + 1), - generateExpressions(nr, depth + 1), - generateExpressions(nr, depth + 1), - generateExpression(depth + 1)); - default: - throw new AssertionError(expr); + case UNARY_PREFIX: + return new HiveUnaryPrefixOperation(generateExpression(depth + 1), HiveUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new HiveUnaryPostfixOperation(generateExpression(depth + 1), HiveUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = HiveBinaryComparisonOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = HiveExpressionGenerator.HiveBinaryLogicalOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new HiveBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + HiveExpressionGenerator.HiveBinaryArithmeticOperator.getRandom()); + case CAST: + return new HiveCastOperation(generateExpression(depth + 1), HiveDataType.getRandomType()); + case FUNC: + HiveFunc func = HiveFunc.getRandom(); + return new HiveFunction<>(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new HiveBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new HiveInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new HiveCaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(expr); } } @@ -128,16 +118,16 @@ public HiveExpression generateConstant() { HiveDataType[] values = HiveDataType.values(); HiveDataType constantType = Randomly.fromOptions(values); switch (constantType) { - case STRING: - return HiveConstant.createStringConstant(globalState.getRandomly().getString()); - case INT: - return HiveConstant.createIntConstant(globalState.getRandomly().getInteger()); - case DOUBLE: - return HiveConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - case BOOLEAN: - return HiveConstant.createBooleanConstant(Randomly.getBoolean()); - default: - throw new AssertionError(constantType); + case STRING: + return HiveConstant.createStringConstant(globalState.getRandomly().getString()); + case INT: + return HiveConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DOUBLE: + return HiveConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BOOLEAN: + return HiveConstant.createBooleanConstant(Randomly.getBoolean()); + default: + throw new AssertionError(constantType); } } @@ -160,7 +150,6 @@ public List generateOrderBys() { return newExpr; } - @Override public HiveExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); @@ -189,8 +178,7 @@ public List generateFetchColumns(boolean allowAggregates) { if (Randomly.getBoolean()) { return List.of(new HiveColumnReference(new HiveColumn("*", null, null))); } - return Randomly.nonEmptySubset(columns).stream() - .map(c -> new HiveColumnReference(c)) + return Randomly.nonEmptySubset(columns).stream().map(c -> new HiveColumnReference(c)) .collect(Collectors.toList()); } @@ -243,9 +231,8 @@ public String getTextRepresentation() { public enum HiveBinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), - SMALLER_EQUALS("<="), NOT_EQUALS("!="), LIKE("LIKE"), - NOT_LIKE("NOT LIKE"), REGEXP("RLIKE"); + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), REGEXP("RLIKE"); private String textRepr; @@ -305,18 +292,8 @@ public String getTextRepresentation() { } public enum HiveAggregateFunction { - COUNT(1), - SUM(1), - AVG(1), - MIN(1), - MAX(1), - VARIANCE(1), - VAR_SAMP(1), - STDDEV_POP(1), - STDDEV_SAMP(1), - COVAR_POP(2), - COVAR_SAMP(2), - CORR(2); + COUNT(1), SUM(1), AVG(1), MIN(1), MAX(1), VARIANCE(1), VAR_SAMP(1), STDDEV_POP(1), STDDEV_SAMP(1), COVAR_POP(2), + COVAR_SAMP(2), CORR(2); private int nrArgs; @@ -335,17 +312,15 @@ public int getNrArgs() { // TODO: test all Hive default functions... public enum HiveFunc { - + // mathematical functions - ROUND(2), - FLOOR(1); + ROUND(2), FLOOR(1); // collection functions // date functions // string functions - private int nrArgs; private boolean isVariadic; diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java index 43192a5be..6f57818fe 100644 --- a/src/sqlancer/hive/gen/HiveTableGenerator.java +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -82,8 +82,9 @@ private void appendColumn(int columnId, ExpectedErrors errors) { } private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { - /* column_constraint_specification: - : [ PRIMARY KEY|UNIQUE|NOT NULL|DEFAULT [default_value]|CHECK [check_expression] ENABLE|DISABLE NOVALIDATE RELY/NORELY ] + /* + * column_constraint_specification: : [ PRIMARY KEY|UNIQUE|NOT NULL|DEFAULT [default_value]|CHECK + * [check_expression] ENABLE|DISABLE NOVALIDATE RELY/NORELY ] */ if (Randomly.getBoolean()) { // no column constraint @@ -92,31 +93,30 @@ private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { ColumnConstraints constraint = Randomly.fromOptions(ColumnConstraints.values()); switch (constraint) { - case PRIMARY_KEY_DISABLE: - if (allowPrimaryKey && !setPrimaryKey) { - sb.append(" PRIMARY KEY DISABLE"); - setPrimaryKey = true; - } - break; - case UNIQUE_DISABLE: - sb.append(" UNIQUE DISABLE"); - break; - case NOT_NULL: - sb.append(" NOT NULL"); - break; - case DEFAULT: - sb.append(" DEFAULT ("); - sb.append(HiveToStringVisitor.asString(gen.generateConstant())); - sb.append(")"); - case CHECK: - sb.append(" CHECK ("); - sb.append(HiveToStringVisitor.asString(gen.generateExpression())); - sb.append(")"); - break; - default: - throw new AssertionError(constraint); + case PRIMARY_KEY_DISABLE: + if (allowPrimaryKey && !setPrimaryKey) { + sb.append(" PRIMARY KEY DISABLE"); + setPrimaryKey = true; + } + break; + case UNIQUE_DISABLE: + sb.append(" UNIQUE DISABLE"); + break; + case NOT_NULL: + sb.append(" NOT NULL"); + break; + case DEFAULT: + sb.append(" DEFAULT ("); + sb.append(HiveToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + case CHECK: + sb.append(" CHECK ("); + sb.append(HiveToStringVisitor.asString(gen.generateExpression())); + sb.append(")"); + break; + default: + throw new AssertionError(constraint); } } - } diff --git a/test/sqlancer/dbms/TestHiveTLP.java b/test/sqlancer/dbms/TestHiveTLP.java index 05a4bc20d..5f8692f3f 100644 --- a/test/sqlancer/dbms/TestHiveTLP.java +++ b/test/sqlancer/dbms/TestHiveTLP.java @@ -8,13 +8,13 @@ import sqlancer.Main; public class TestHiveTLP { - + @Test public void testHiveTLPWhere() { assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.HIVE_ENV)); assertEquals(0, - Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", - "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", - "--num-queries", TestConfig.NUM_QUERIES, "hive", "--oracle", "TLPWhere" })); + Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", + "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "hive", "--oracle", "TLPWhere" })); } } \ No newline at end of file From f52353050457570da5f5eea02bb11948b350c190 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 24 Apr 2025 00:18:44 +0800 Subject: [PATCH 1100/1284] add test of CODDTest for SQLite3 in github action --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8ea1059a6..888c04d06 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -569,6 +569,7 @@ jobs: mvn -Dtest=TestSQLitePQS test mvn -Dtest=TestSQLiteTLP test mvn -Dtest=TestSQLiteNoREC test + mvn -Dtest=TestSQLiteCODDTest test sqlite-qpg: name: QPG Tests (SQLite) From a9f7dce592ab056f8cf4a28a1da79a3ead589958 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 24 Apr 2025 09:30:09 +0800 Subject: [PATCH 1101/1284] format hive for mvn verify --- pom.xml | 2 +- src/sqlancer/hive/HiveErrors.java | 2 +- src/sqlancer/hive/HiveOptions.java | 12 ++++++------ src/sqlancer/hive/HiveProvider.java | 18 +++++++++--------- src/sqlancer/hive/HiveSchema.java | 16 ++++++++-------- src/sqlancer/hive/HiveToStringVisitor.java | 2 +- src/sqlancer/hive/ast/HiveSelect.java | 2 +- src/sqlancer/hive/ast/HiveTableReference.java | 2 +- .../hive/gen/HiveExpressionGenerator.java | 18 ++++++++++-------- src/sqlancer/hive/gen/HiveTableGenerator.java | 18 +++++++++--------- 10 files changed, 47 insertions(+), 45 deletions(-) diff --git a/pom.xml b/pom.xml index bf30df4ce..83068f498 100644 --- a/pom.xml +++ b/pom.xml @@ -378,7 +378,7 @@ flight-sql-jdbc-driver 16.1.0 - + org.apache.hive hive-jdbc 4.0.1 diff --git a/src/sqlancer/hive/HiveErrors.java b/src/sqlancer/hive/HiveErrors.java index b061c1071..81b0be668 100644 --- a/src/sqlancer/hive/HiveErrors.java +++ b/src/sqlancer/hive/HiveErrors.java @@ -5,7 +5,7 @@ import sqlancer.common.query.ExpectedErrors; -public class HiveErrors { +public final class HiveErrors { private HiveErrors() { } diff --git a/src/sqlancer/hive/HiveOptions.java b/src/sqlancer/hive/HiveOptions.java index 5d2ea47e6..ea55ac676 100644 --- a/src/sqlancer/hive/HiveOptions.java +++ b/src/sqlancer/hive/HiveOptions.java @@ -1,18 +1,18 @@ package sqlancer.hive; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.DBMSSpecificOptions; import sqlancer.hive.gen.HiveExpressionGenerator; -import sqlancer.OracleFactory; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; @Parameters(separators = "=", commandDescription = "Hive (default port: " + HiveOptions.DEFAULT_PORT + ", default host: " + HiveOptions.DEFAULT_HOST + ")") diff --git a/src/sqlancer/hive/HiveProvider.java b/src/sqlancer/hive/HiveProvider.java index 723e3fd6f..12798df93 100644 --- a/src/sqlancer/hive/HiveProvider.java +++ b/src/sqlancer/hive/HiveProvider.java @@ -1,5 +1,12 @@ package sqlancer.hive; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; @@ -13,13 +20,6 @@ import sqlancer.hive.gen.HiveInsertGenerator; import sqlancer.hive.gen.HiveTableGenerator; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; - -import com.google.auto.service.AutoService; - @AutoService(DatabaseProvider.class) public class HiveProvider extends SQLProviderAdapter { @@ -67,8 +67,8 @@ public void generateDatabase(HiveGlobalState globalState) throws Exception { throw new IgnoreMeException(); // TODO } - StatementExecutor se = new StatementExecutor(globalState, - Action.values(), HiveProvider::mapActions, (q) -> { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + HiveProvider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java index fcc17c77c..822eea163 100644 --- a/src/sqlancer/hive/HiveSchema.java +++ b/src/sqlancer/hive/HiveSchema.java @@ -1,5 +1,13 @@ package sqlancer.hive; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractRelationalTable; @@ -9,14 +17,6 @@ import sqlancer.common.schema.TableIndex; import sqlancer.hive.HiveSchema.HiveTable; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - public class HiveSchema extends AbstractSchema { public enum HiveDataType { diff --git a/src/sqlancer/hive/HiveToStringVisitor.java b/src/sqlancer/hive/HiveToStringVisitor.java index d23818916..bdcd31eaf 100644 --- a/src/sqlancer/hive/HiveToStringVisitor.java +++ b/src/sqlancer/hive/HiveToStringVisitor.java @@ -2,11 +2,11 @@ import sqlancer.common.ast.newast.NewToStringVisitor; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hive.ast.HiveCastOperation; import sqlancer.hive.ast.HiveConstant; import sqlancer.hive.ast.HiveExpression; import sqlancer.hive.ast.HiveJoin; import sqlancer.hive.ast.HiveSelect; -import sqlancer.hive.ast.HiveCastOperation; public class HiveToStringVisitor extends NewToStringVisitor { diff --git a/src/sqlancer/hive/ast/HiveSelect.java b/src/sqlancer/hive/ast/HiveSelect.java index b099ade02..8a0eddc06 100644 --- a/src/sqlancer/hive/ast/HiveSelect.java +++ b/src/sqlancer/hive/ast/HiveSelect.java @@ -4,9 +4,9 @@ import sqlancer.common.ast.SelectBase; import sqlancer.common.ast.newast.Select; -import sqlancer.hive.HiveToStringVisitor; import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.HiveToStringVisitor; public class HiveSelect extends SelectBase implements Select, HiveExpression { diff --git a/src/sqlancer/hive/ast/HiveTableReference.java b/src/sqlancer/hive/ast/HiveTableReference.java index a1b586825..7d23b4895 100644 --- a/src/sqlancer/hive/ast/HiveTableReference.java +++ b/src/sqlancer/hive/ast/HiveTableReference.java @@ -10,4 +10,4 @@ public HiveTableReference(HiveSchema.HiveTable table) { super(table); } -} \ No newline at end of file +} diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index bd9134de6..9f3de2514 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -1,12 +1,20 @@ package sqlancer.hive.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.hive.HiveGlobalState; -import sqlancer.hive.HiveSchema.*; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveDataType; +import sqlancer.hive.HiveSchema.HiveTable; import sqlancer.hive.ast.HiveBetweenOperation; import sqlancer.hive.ast.HiveBinaryOperation; import sqlancer.hive.ast.HiveCaseOperation; @@ -20,14 +28,8 @@ import sqlancer.hive.ast.HiveOrderingTerm; import sqlancer.hive.ast.HiveSelect; import sqlancer.hive.ast.HiveTableReference; -import sqlancer.hive.ast.HiveUnaryPrefixOperation; import sqlancer.hive.ast.HiveUnaryPostfixOperation; -import sqlancer.common.ast.BinaryOperatorNode.Operator; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; +import sqlancer.hive.ast.HiveUnaryPrefixOperation; public class HiveExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator { diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java index 6f57818fe..c1c4db2bd 100644 --- a/src/sqlancer/hive/gen/HiveTableGenerator.java +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -1,5 +1,8 @@ package sqlancer.hive.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -7,13 +10,10 @@ import sqlancer.hive.HiveErrors; import sqlancer.hive.HiveGlobalState; import sqlancer.hive.HiveSchema; -import sqlancer.hive.HiveToStringVisitor; import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveDataType; import sqlancer.hive.HiveSchema.HiveTable; - -import java.util.ArrayList; -import java.util.List; +import sqlancer.hive.HiveToStringVisitor; public class HiveTableGenerator { @@ -31,7 +31,7 @@ private enum ColumnConstraints { private final HiveExpressionGenerator gen; private final HiveTable table; private final List columnsToBeAdded = new ArrayList<>(); - private boolean setPrimaryKey = false; + private boolean setPrimaryKey; public HiveTableGenerator(HiveGlobalState globalState, String tableName) { this.tableName = tableName; @@ -57,7 +57,7 @@ private SQLQueryAdapter create() { if (i != 0) { sb.append(", "); } - appendColumn(i, errors); + appendColumn(i); } sb.append(")"); @@ -71,17 +71,17 @@ private SQLQueryAdapter create() { return new SQLQueryAdapter(sb.toString(), errors, true, false); } - private void appendColumn(int columnId, ExpectedErrors errors) { + private void appendColumn(int columnId) { String columnName = DBMSCommon.createColumnName(columnId); sb.append(columnName); sb.append(" "); HiveDataType randType = HiveSchema.HiveDataType.getRandomType(); sb.append(randType); columnsToBeAdded.add(new HiveColumn(columnName, table, randType)); - appendColumnConstraint(randType, errors); + appendColumnConstraint(); } - private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { + private void appendColumnConstraint() { /* * column_constraint_specification: : [ PRIMARY KEY|UNIQUE|NOT NULL|DEFAULT [default_value]|CHECK * [check_expression] ENABLE|DISABLE NOVALIDATE RELY/NORELY ] From 5d9d1d5ac747ed5a46eaf2f34ca79c869f04856d Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Fri, 23 May 2025 23:27:46 -0400 Subject: [PATCH 1102/1284] Add support for Alter Statistics and Set Statistics and remove force_parallel_mode --- src/sqlancer/postgres/PostgresProvider.java | 4 ++++ src/sqlancer/postgres/gen/PostgresCommon.java | 3 ++- .../postgres/gen/PostgresSetGenerator.java | 2 +- .../postgres/gen/PostgresStatisticsGenerator.java | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 2b2cb8585..347c7b6f2 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -99,6 +99,7 @@ public enum Action implements AbstractAction { }), // CREATE_STATISTICS(PostgresStatisticsGenerator::insert), // DROP_STATISTICS(PostgresStatisticsGenerator::remove), // + ALTER_STATISTICS(PostgresStatisticsGenerator::alter), // DELETE(PostgresDeleteGenerator::create), // DISCARD(PostgresDiscardGenerator::create), // DROP_INDEX(PostgresDropIndexGenerator::create), // @@ -150,6 +151,9 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case CREATE_STATISTICS: nrPerformed = r.getInteger(0, 5); break; + case ALTER_STATISTICS: + nrPerformed = r.getInteger(0, 2); + break; case DISCARD: case DROP_INDEX: nrPerformed = r.getInteger(0, 5); diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index a0cc21ae2..63bc885cb 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -434,6 +434,8 @@ private static void addTableConstraint(StringBuilder sb, PostgresTable table, Po } break; case EXCLUDE: + errors.add("exclusion constraints are not supported on partitioned tables"); + errors.add("unsupported EXCLUDE constraint with partition key definition"); sb.append("EXCLUDE "); sb.append("("); // TODO [USING index_method ] @@ -454,7 +456,6 @@ private static void addTableConstraint(StringBuilder sb, PostgresTable table, Po errors.add("exclusion constraints are not supported on partitioned tables"); errors.add("The exclusion operator must be related to the index operator class for the constraint"); errors.add("could not create exclusion constraint"); - // TODO: index parameters if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append("("); diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 440172c4b..38318b26b 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -112,7 +112,7 @@ private enum ConfigurationOption { JIT("jit", (r) -> Randomly.fromOptions(1, 0)), JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), - FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), + // FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), PLAN_CACHE_MODE("plan_cache_mode", (r) -> Randomly.fromOptions("auto", "force_generic_plan", "force_custom_plan")); diff --git a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java index e27f82800..9a82e29af 100644 --- a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java @@ -58,6 +58,20 @@ public static SQLQueryAdapter remove(PostgresGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), true); } + public static SQLQueryAdapter alter(PostgresGlobalState globalState) { + StringBuilder sb = new StringBuilder("ALTER STATISTICS "); + PostgresTable randomTable = globalState.getSchema().getRandomTable(); + List statistics = randomTable.getStatistics(); + if (statistics.isEmpty()) { + throw new IgnoreMeException(); + } + PostgresStatisticsObject randomStatistic = Randomly.fromList(statistics); + sb.append(randomStatistic.getName()); + sb.append(" SET STATISTICS "); + sb.append(Randomly.getNotCachedInteger(-1, 10000)); // -1 means default + return new SQLQueryAdapter(sb.toString(), true); + } + private static String getNewStatisticsName(PostgresTable randomTable) { List statistics = randomTable.getStatistics(); int i = 0; From 3fe905a4294e7bbb0dcbf1e9d8d6cfa5aa1c0a28 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 24 May 2025 13:20:19 -0400 Subject: [PATCH 1103/1284] Add support for Alter View Rename Column in Postgres v13 --- .../gen/PostgresAlterTableGenerator.java | 45 ++++++++++++++++--- .../postgres/gen/PostgresSetGenerator.java | 2 +- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index f12927e90..b4ad39ecd 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -48,7 +48,7 @@ protected enum Action { SET_LOGGED_UNLOGGED, // NOT_OF, // OWNER_TO, // - REPLICA_IDENTITY + REPLICA_IDENTITY, ALTER_VIEW_RENAME_COLUMN // RENAME COLUMN old_name TO new_name (for views) } public PostgresAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, @@ -98,6 +98,20 @@ public List getActions(ExpectedErrors errors) { // make it more likely that the ALTER TABLE succeeds action = Randomly.subset(Randomly.smallNumber(), Action.values()); } + + // If this is a view, only allow view-compatible operations + if (randomTable.isView()) { + // Remove all table-specific operations for views + action.removeIf(a -> a != Action.ALTER_VIEW_RENAME_COLUMN); + // If no view operations remain, add the rename column action + if (action.isEmpty()) { + action.add(Action.ALTER_VIEW_RENAME_COLUMN); + } + } else { + // Remove view-specific actions if this is a table + action.remove(Action.ALTER_VIEW_RENAME_COLUMN); + } + if (randomTable.getColumns().size() == 1) { action.remove(Action.ALTER_TABLE_DROP_COLUMN); } @@ -120,14 +134,24 @@ public SQLQueryAdapter generate() { int i = 0; List action = getActions(errors); StringBuilder sb = new StringBuilder(); - sb.append("ALTER TABLE "); - if (Randomly.getBoolean()) { - sb.append(" ONLY"); - errors.add("cannot use ONLY for foreign key on partitioned table"); + + // Check if we're dealing with a view operation + boolean isViewOperation = action.contains(Action.ALTER_VIEW_RENAME_COLUMN); + + if (isViewOperation) { + sb.append("ALTER VIEW "); + } else { + sb.append("ALTER TABLE "); + if (Randomly.getBoolean()) { + sb.append(" ONLY"); + errors.add("cannot use ONLY for foreign key on partitioned table"); + } } + sb.append(" "); sb.append(randomTable.getName()); sb.append(" "); + for (Action a : action) { if (i++ != 0) { sb.append(", "); @@ -364,6 +388,17 @@ public SQLQueryAdapter generate() { errors.add("cannot use invalid index"); } break; + case ALTER_VIEW_RENAME_COLUMN: + sb.append("RENAME COLUMN "); + PostgresColumn columnToRename = randomTable.getRandomColumn(); + sb.append(columnToRename.getName()); + sb.append(" TO "); + sb.append("new_" + columnToRename.getName() + "_" + r.getInteger(1, 1000)); + errors.add("column does not exist"); + errors.add("column name already exists"); + errors.add("cannot rename column of view"); + errors.add("permission denied"); + break; default: throw new AssertionError(a); } diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 440172c4b..38318b26b 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -112,7 +112,7 @@ private enum ConfigurationOption { JIT("jit", (r) -> Randomly.fromOptions(1, 0)), JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), - FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), + // FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), PLAN_CACHE_MODE("plan_cache_mode", (r) -> Randomly.fromOptions("auto", "force_generic_plan", "force_custom_plan")); From d28c960a130267670d932c8705a408326fb5a3b6 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 24 May 2025 16:30:29 -0400 Subject: [PATCH 1104/1284] Add support for Drop Database Force --- src/sqlancer/postgres/PostgresProvider.java | 24 +++++++++++++++---- .../postgres/gen/PostgresSetGenerator.java | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 2b2cb8585..6f5e14019 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -275,12 +275,28 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE } Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + createDatabaseCommand = getCreateDatabaseCommand(globalState); - globalState.getState().logStatement(createDatabaseCommand); - try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + + // postgres 13+ supports force + if (Randomly.getBoolean()) { + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE FORCE IF EXISTS " + databaseName); + } catch (SQLException e) { + // If force fails, fall back to regular drop + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + } + } else { + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } } + + // Execute the create database command try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); } diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 440172c4b..38318b26b 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -112,7 +112,7 @@ private enum ConfigurationOption { JIT("jit", (r) -> Randomly.fromOptions(1, 0)), JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), - FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), + // FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), PLAN_CACHE_MODE("plan_cache_mode", (r) -> Randomly.fromOptions("auto", "force_generic_plan", "force_custom_plan")); From a513d9a4b16aec61d011d1cf0c4e00027976d481 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 24 May 2025 19:57:30 -0400 Subject: [PATCH 1105/1284] Add a list dedicated for view actions only --- .../gen/PostgresAlterTableGenerator.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index b4ad39ecd..33f1b202e 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -48,9 +48,14 @@ protected enum Action { SET_LOGGED_UNLOGGED, // NOT_OF, // OWNER_TO, // - REPLICA_IDENTITY, ALTER_VIEW_RENAME_COLUMN // RENAME COLUMN old_name TO new_name (for views) + REPLICA_IDENTITY, // RENAME COLUMN old_name TO new_name (for views) + ALTER_VIEW_RENAME_COLUMN // RENAME COLUMN old_name TO new_name (for views) } + private static final List VIEW_ACTIONS = List.of( + Action.ALTER_VIEW_RENAME_COLUMN + ); + public PostgresAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, boolean generateOnlyKnown) { this.randomTable = randomTable; @@ -101,15 +106,15 @@ public List getActions(ExpectedErrors errors) { // If this is a view, only allow view-compatible operations if (randomTable.isView()) { - // Remove all table-specific operations for views - action.removeIf(a -> a != Action.ALTER_VIEW_RENAME_COLUMN); - // If no view operations remain, add the rename column action + // Remove all non-view operations + action.removeIf(a -> !VIEW_ACTIONS.contains(a)); + // If no view operations remain, add a random view operation if (action.isEmpty()) { - action.add(Action.ALTER_VIEW_RENAME_COLUMN); + action.add(VIEW_ACTIONS.get(r.getInteger(0, VIEW_ACTIONS.size() - 1))); } } else { // Remove view-specific actions if this is a table - action.remove(Action.ALTER_VIEW_RENAME_COLUMN); + action.removeIf(VIEW_ACTIONS::contains); } if (randomTable.getColumns().size() == 1) { From 865cb58dae82ee5c73ae5383388d9d4e34403b66 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 31 May 2025 10:54:06 -0400 Subject: [PATCH 1106/1284] Add support for select with ties --- .../postgres/ast/PostgresOrderByTerm.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java index 96debf741..76257215b 100644 --- a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java +++ b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java @@ -7,6 +7,8 @@ public class PostgresOrderByTerm implements PostgresExpression { private final PostgresExpression expr; private final PostgresOrder order; + private final int limit; + private final boolean ties; public enum PostgresOrder { ASC, DESC; @@ -22,6 +24,15 @@ public PostgresOrderByTerm(PostgresExpression expr, PostgresOrder order) { } this.expr = expr; this.order = order; + + if (Randomly.getBooleanWithRatherLowProbability()) { + this.limit = (int) Randomly.getPositiveOrZeroNonCachedInteger(); + this.ties = true; + } else { + this.limit = 0; + this.ties = false; + } + } // Constructor for window functions, might be removed in the future to have only one constructor @@ -31,6 +42,8 @@ public PostgresOrderByTerm(PostgresExpression expr, boolean ascending) { } this.expr = expr; this.order = ascending ? PostgresOrder.ASC : PostgresOrder.DESC; + this.limit = 0; + this.ties = false; } public PostgresExpression getExpr() { @@ -57,6 +70,10 @@ public PostgresDataType getExpressionType() { @Override public String toString() { - return String.format("%s %s", expr, order); + if (ties) { + return String.format("%s %s FETCH FIRST %d WITH TIES", expr, order, limit); + } else { + return String.format("%s %s", expr, order); + } } } From ec6f8a799f7e4778811dfa275f43e8ea3a849835 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 31 May 2025 21:22:34 -0400 Subject: [PATCH 1107/1284] Add support for alter column drop expression --- .../postgres/gen/PostgresAlterTableGenerator.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index f12927e90..ce8674ac9 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -34,6 +34,7 @@ protected enum Action { ALTER_COLUMN_SET_ATTRIBUTE_OPTION, // ALTER [ COLUMN ] column SET ( attribute_option = value [, ... ] ) ALTER_COLUMN_RESET_ATTRIBUTE_OPTION, // ALTER [ COLUMN ] column RESET ( attribute_option [, ... ] ) ALTER_COLUMN_SET_STORAGE, // ALTER [ COLUMN ] column SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } + ALTER_COLUMN_DROP_EXPRESSION, // ALTER [ COLUMN ] column DROP EXPRESSION [ IF EXISTS ] ADD_TABLE_CONSTRAINT, // ADD table_constraint [ NOT VALID ] ADD_TABLE_CONSTRAINT_USING_INDEX, // ADD table_constraint_using_index VALIDATE_CONSTRAINT, // VALIDATE CONSTRAINT constraint_name @@ -250,6 +251,19 @@ public SQLQueryAdapter generate() { errors.add("can only have storage"); errors.add("is an identity column"); break; + case ALTER_COLUMN_DROP_EXPRESSION: + alterColumn(randomTable, sb); + sb.append("DROP EXPRESSION"); + if (Randomly.getBoolean()) { + sb.append(" IF EXISTS"); + } + errors.add("is not a generated column"); + errors.add("is not a stored generated column"); + errors.add("cannot drop expression from inherited column"); + errors.add("cannot drop generation expression from inherited column"); + errors.add("must be applied to child tables too"); + errors.add("cannot drop expression from column"); + break; case ADD_TABLE_CONSTRAINT: sb.append("ADD "); sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); From afbdfe1b184602542f8d9bae5fe8fbdb60289fb8 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 31 May 2025 21:54:39 -0400 Subject: [PATCH 1108/1284] Add support for Create DB with Locale --- src/sqlancer/postgres/PostgresProvider.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 2b2cb8585..15e95b064 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -340,9 +340,13 @@ private String getCreateDatabaseCommand(PostgresGlobalState state) { sb.append(Randomly.fromOptions("utf8")); sb.append("' "); } - for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { - if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { - sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + if (Randomly.getBoolean() && !state.getCollates().isEmpty()) { + sb.append(String.format(" LOCALE = '%s' ", Randomly.fromList(state.getCollates()))); + } else { + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } } } sb.append(" TEMPLATE template0"); From dad655171c07643e64f34028b0157a1632e1fd9b Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 1 Jun 2025 21:52:07 -0400 Subject: [PATCH 1109/1284] Add support for tablespace generator --- src/sqlancer/postgres/PostgresProvider.java | 7 ++- .../gen/PostgresTableSpaceGenerator.java | 49 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 2b2cb8585..5c2897208 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -51,6 +51,7 @@ import sqlancer.postgres.gen.PostgresUpdateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; import sqlancer.postgres.gen.PostgresViewGenerator; +import sqlancer.postgres.gen.PostgresTableSpaceGenerator; // EXISTS // IN @@ -125,7 +126,8 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(PostgresViewGenerator::create); + CREATE_VIEW(PostgresViewGenerator::create), + CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); private final SQLQueryProvider sqlQueryProvider; @@ -186,6 +188,9 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case CREATE_VIEW: nrPerformed = r.getInteger(0, 2); break; + case CREATE_TABLESPACE: + nrPerformed = r.getInteger(0, 2); + break; case UPDATE: nrPerformed = r.getInteger(0, 10); break; diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java new file mode 100644 index 000000000..2f0065408 --- /dev/null +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.postgres.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; + +public class PostgresTableSpaceGenerator { + + private final ExpectedErrors errors = new ExpectedErrors(); + private final PostgresGlobalState globalState; + + public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; + errors.add("ERROR: directory \"/tmp/postgresql/tablespace1\" does not exist"); + errors.add("ERROR: directory \"/tmp/postgresql/tablespace2\" does not exist"); + errors.add("ERROR: directory \"/tmp/postgresql/tablespace3\" does not exist"); + errors.add("ERROR: directory \"/tmp/postgresql/tablespace4\" does not exist"); + errors.add("ERROR: directory \"/tmp/postgresql/tablespace5\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace1\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace2\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace3\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace4\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace5\" does not exist"); + errors.add("ERROR: must be a directory"); + errors.add("ERROR: permission denied"); + errors.add("ERROR: already exists"); + errors.add("ERROR: is not empty"); + errors.add("ERROR: cannot be created because system does not support tablespaces"); + } + + public static SQLQueryAdapter generate(PostgresGlobalState globalState) { + return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); + } + + private SQLQueryAdapter generateTableSpace() { + StringBuilder sb = new StringBuilder(); + int tableSpaceNum = globalState.getRandomly().getInteger(1, 5); // Use the instance method + + // CREATE TABLESPACE syntax + sb.append("CREATE TABLESPACE "); + sb.append("tablespace"); + sb.append(tableSpaceNum); + sb.append(" LOCATION '/tmp/postgresql/tablespace"); + sb.append(tableSpaceNum); + sb.append("'"); + + return new SQLQueryAdapter(sb.toString(), errors); + } +} From 5766078e2715f861655b7707cd5b734b89550b37 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 1 Jun 2025 22:06:09 -0400 Subject: [PATCH 1110/1284] Add a clearer execution statement --- src/sqlancer/postgres/PostgresProvider.java | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 6f5e14019..3162a03fb 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -276,27 +276,31 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - createDatabaseCommand = getCreateDatabaseCommand(globalState); - - // postgres 13+ supports force + String dropCommand = "DROP DATABASE"; if (Randomly.getBoolean()) { - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE FORCE IF EXISTS " + databaseName); - } catch (SQLException e) { - // If force fails, fall back to regular drop + dropCommand += " FORCE"; + } + dropCommand += " IF EXISTS " + databaseName; + + globalState.getState().logStatement(dropCommand + ";"); + try (Statement s = con.createStatement()) { + s.execute(dropCommand); + } catch (SQLException e) { + // If force fails, fall back to regular drop + if (dropCommand.contains("FORCE")) { + String fallbackDrop = "DROP DATABASE IF EXISTS " + databaseName; + globalState.getState().logStatement(fallbackDrop + ";"); try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + s.execute(fallbackDrop); } - } - } else { - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + } else { + throw e; } } - // Execute the create database command + // Create database section + createDatabaseCommand = getCreateDatabaseCommand(globalState); + globalState.getState().logStatement(createDatabaseCommand + ";"); try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); } From 5868650179c66fc23026e22e517b9438579381d0 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Tue, 3 Jun 2025 23:32:02 -0400 Subject: [PATCH 1111/1284] Fix AlterTableGenerator format issue --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index c8842f0a4..6e0c436ee 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -53,9 +53,7 @@ protected enum Action { ALTER_VIEW_RENAME_COLUMN // RENAME COLUMN old_name TO new_name (for views) } - private static final List VIEW_ACTIONS = List.of( - Action.ALTER_VIEW_RENAME_COLUMN - ); + private static final List VIEW_ACTIONS = List.of(Action.ALTER_VIEW_RENAME_COLUMN); public PostgresAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, boolean generateOnlyKnown) { From 38fcee824b8447660b7a078b7282430ab8c2ac68 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 5 Jun 2025 10:54:00 +0800 Subject: [PATCH 1112/1284] fix issue #1226 that incorrect constraint make the code can not generate the generated columns with the default configuration for SQLite --- src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index 1d045982b..4e83fbca5 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -51,7 +51,7 @@ public String createColumn(String columnName, SQLite3GlobalState globalState, Li if (Randomly.getBooleanWithRatherLowProbability()) { List constraints = Randomly.subset(Constraints.values()); if (!Randomly.getBooleanWithSmallProbability() - || globalState.getDbmsSpecificOptions().testGeneratedColumns) { + || !globalState.getDbmsSpecificOptions().testGeneratedColumns) { constraints.remove(Constraints.GENERATED_AS); } if (constraints.contains(Constraints.GENERATED_AS)) { From 25b51b6ca4f232f8b5a0d774b2668ce7d64171fd Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 5 Jun 2025 13:52:29 +0800 Subject: [PATCH 1113/1284] fix issue #1239 --- .../sqlite3/oracle/SQLite3RandomQuerySynthesizer.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java index d3d7ded31..f8f53ae3b 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java @@ -37,10 +37,11 @@ public static SQLite3Expression generate(SQLite3GlobalState globalState, int siz SQLite3Tables targetTables = s.getRandomTableNonEmptyTables(); List expressions = new ArrayList<>(); SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState) - .setColumns(s.getTables().getColumns()); - SQLite3ExpressionGenerator whereClauseGen = new SQLite3ExpressionGenerator(globalState); + .setColumns(targetTables.getColumns()); + SQLite3ExpressionGenerator whereClauseGen = new SQLite3ExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); SQLite3ExpressionGenerator aggregateGen = new SQLite3ExpressionGenerator(globalState) - .setColumns(s.getTables().getColumns()).allowAggregateFunctions(); + .setColumns(targetTables.getColumns()).allowAggregateFunctions(); // SELECT SQLite3Select select = new SQLite3Select(); From ebda7309b84389209f9ba2432cdb823da8cfd97b Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 8 Jun 2025 20:19:33 -0400 Subject: [PATCH 1114/1284] Add boolean flag for drop db force --- src/sqlancer/postgres/PostgresProvider.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 3162a03fb..3a2b6b357 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -277,7 +277,8 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); String dropCommand = "DROP DATABASE"; - if (Randomly.getBoolean()) { + boolean forceDrop = Randomly.getBoolean(); + if (forceDrop) { dropCommand += " FORCE"; } dropCommand += " IF EXISTS " + databaseName; @@ -287,7 +288,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE s.execute(dropCommand); } catch (SQLException e) { // If force fails, fall back to regular drop - if (dropCommand.contains("FORCE")) { + if (forceDrop) { String fallbackDrop = "DROP DATABASE IF EXISTS " + databaseName; globalState.getState().logStatement(fallbackDrop + ";"); try (Statement s = con.createStatement()) { From 7e0b0f4d33fc5a8e02f2abb763c3145124e2174d Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 8 Jun 2025 20:58:23 -0400 Subject: [PATCH 1115/1284] Add regex and OS option for TableSpaceGenerator --- src/sqlancer/postgres/PostgresOptions.java | 41 ++++++++++++++++++- src/sqlancer/postgres/PostgresProvider.java | 5 +-- .../gen/PostgresTableSpaceGenerator.java | 40 ++++++++++-------- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 659dc4ff6..e3b93269d 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -1,5 +1,6 @@ package sqlancer.postgres; +import java.io.File; import java.util.Arrays; import java.util.List; @@ -13,6 +14,7 @@ public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; + private static final boolean DEFAULT_TEST_TABLESPACES = determineDefaultTablespaceSupport(); @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) public boolean allowBulkInsert; @@ -23,6 +25,12 @@ public class PostgresOptions implements DBMSSpecificOptions getTestOracleFactory() { return oracle; } - } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 5c2897208..881239d85 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -46,12 +46,12 @@ import sqlancer.postgres.gen.PostgresSetGenerator; import sqlancer.postgres.gen.PostgresStatisticsGenerator; import sqlancer.postgres.gen.PostgresTableGenerator; +import sqlancer.postgres.gen.PostgresTableSpaceGenerator; import sqlancer.postgres.gen.PostgresTransactionGenerator; import sqlancer.postgres.gen.PostgresTruncateGenerator; import sqlancer.postgres.gen.PostgresUpdateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; import sqlancer.postgres.gen.PostgresViewGenerator; -import sqlancer.postgres.gen.PostgresTableSpaceGenerator; // EXISTS // IN @@ -126,8 +126,7 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(PostgresViewGenerator::create), - CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); + CREATE_VIEW(PostgresViewGenerator::create), CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); private final SQLQueryProvider sqlQueryProvider; diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java index 2f0065408..89f47cef2 100644 --- a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -3,24 +3,16 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; public class PostgresTableSpaceGenerator { - + private final ExpectedErrors errors = new ExpectedErrors(); private final PostgresGlobalState globalState; public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { this.globalState = globalState; - errors.add("ERROR: directory \"/tmp/postgresql/tablespace1\" does not exist"); - errors.add("ERROR: directory \"/tmp/postgresql/tablespace2\" does not exist"); - errors.add("ERROR: directory \"/tmp/postgresql/tablespace3\" does not exist"); - errors.add("ERROR: directory \"/tmp/postgresql/tablespace4\" does not exist"); - errors.add("ERROR: directory \"/tmp/postgresql/tablespace5\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace1\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace2\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace3\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace4\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace5\" does not exist"); + errors.addRegexString("ERROR: (?:tablespace )?directory \".*[\\\\/]tablespace[1-5]\" does not exist"); errors.add("ERROR: must be a directory"); errors.add("ERROR: permission denied"); errors.add("ERROR: already exists"); @@ -29,21 +21,37 @@ public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { } public static SQLQueryAdapter generate(PostgresGlobalState globalState) { + // Skip tablespace generation if the option is disabled + PostgresOptions options = globalState.getDbmsSpecificOptions(); + if (!options.testTablespaces) { + return null; + } return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); } private SQLQueryAdapter generateTableSpace() { StringBuilder sb = new StringBuilder(); - int tableSpaceNum = globalState.getRandomly().getInteger(1, 5); // Use the instance method - + int tableSpaceNum = globalState.getRandomly().getInteger(1, 5); + // CREATE TABLESPACE syntax sb.append("CREATE TABLESPACE "); sb.append("tablespace"); sb.append(tableSpaceNum); - sb.append(" LOCATION '/tmp/postgresql/tablespace"); - sb.append(tableSpaceNum); + sb.append(" LOCATION '"); + + // Get the base path from options and append the tablespace number + PostgresOptions options = globalState.getDbmsSpecificOptions(); + String path = options.tablespacePath + tableSpaceNum; + + // Convert backslashes to forward slashes for PostgreSQL + path = path.replace('\\', '/'); + + // Escape single quotes in the path + path = path.replace("'", "''"); + + sb.append(path); sb.append("'"); - + return new SQLQueryAdapter(sb.toString(), errors); } } From 4e1ec862893cec86df771b5fca89e6d56b322867 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Tue, 10 Jun 2025 15:34:19 +0800 Subject: [PATCH 1116/1284] Bump SQLite from 3.47.2.0 to 3.49.1.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 83068f498..734b44f5d 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ org.xerial sqlite-jdbc - 3.47.2.0 + 3.49.1.0 mysql From 22e928ad1dca97d8360d24b93e38e352339b8501 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Wed, 11 Jun 2025 21:58:19 +0800 Subject: [PATCH 1117/1284] Bump DuckDB from 1.2.0 to 1.3.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 83068f498..83ebf893d 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 1.2.0 + 1.3.0.0 com.facebook.presto From ffef7d12997476a9b55b07d67e4140286c947bd6 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Thu, 12 Jun 2025 11:21:56 +0800 Subject: [PATCH 1118/1284] Add the expected error for COLLATE on non-varchar Statement: SELECT t22.rowid FROM t22, t0 WHERE t0.c1 ORDER BY t22.c0, t22.rowid, ((1) COLLATE NOACCENT) DESC; Binder Error: COLLATE can only be applied to varchar columns --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index f5cf3eb94..394de69d4 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -44,6 +44,7 @@ public static List getExpressionErrors() { // collate errors.add("Cannot combine types with different collation!"); errors.add("collations are only supported for type varchar"); + errors.add("COLLATE can only be applied to varchar columns"); errors.add("Like pattern must not end with escape character!"); // LIKE From 495ad713bbca745e05a21c4b3cfd0fee5b93bc66 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Tue, 17 Jun 2025 22:41:17 -0400 Subject: [PATCH 1119/1284] Refactored the module --- src/sqlancer/postgres/PostgresOptions.java | 46 ++++++++++++++++++- src/sqlancer/postgres/PostgresProvider.java | 3 +- .../gen/PostgresTableSpaceGenerator.java | 11 ++--- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index e3b93269d..62a115cf4 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -14,7 +14,7 @@ public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; - private static final boolean DEFAULT_TEST_TABLESPACES = determineDefaultTablespaceSupport(); + private static Boolean defaultTestTablespaces = null; @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) public boolean allowBulkInsert; @@ -26,7 +26,7 @@ public class PostgresOptions implements DBMSSpecificOptions getTestOracleFactory() { return oracle; } + + public String getTablespacePath() { + if (tablespacePath == null || tablespacePath.trim().isEmpty()) { + throw new AssertionError("Tablespace path is null or empty. Please configure --tablespace-path"); + } + + File path = new File(tablespacePath); + + // Check if the directory exists or can be created + if (!path.exists()) { + if (!path.mkdirs()) { + throw new AssertionError("Cannot create tablespace directory: " + tablespacePath + + ". Please ensure the parent directory exists and you have write permissions."); + } + } + + // Check if it's actually a directory + if (!path.isDirectory()) { + throw new AssertionError("Tablespace path is not a directory: " + tablespacePath); + } + + // Check write permissions + if (!path.canWrite()) { + throw new AssertionError("No write permissions for tablespace directory: " + tablespacePath + + ". Please ensure you have write permissions to this directory."); + } + + return tablespacePath; + } + + public boolean isTestTablespaces() { + // If the user explicitly set the value via command line, use that + // Otherwise, use the OS-dependent default + return testTablespaces || getDefaultTablespaceSupport(); + } + + private static boolean getDefaultTablespaceSupport() { + if (defaultTestTablespaces == null) { + defaultTestTablespaces = determineDefaultTablespaceSupport(); + } + return defaultTestTablespaces; + } } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 881239d85..c3f161a7e 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -126,7 +126,8 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(PostgresViewGenerator::create), CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); + CREATE_VIEW(PostgresViewGenerator::create), + CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); private final SQLQueryProvider sqlQueryProvider; diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java index 89f47cef2..6c44043b8 100644 --- a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -12,8 +12,7 @@ public class PostgresTableSpaceGenerator { public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { this.globalState = globalState; - errors.addRegexString("ERROR: (?:tablespace )?directory \".*[\\\\/]tablespace[1-5]\" does not exist"); - errors.add("ERROR: must be a directory"); + errors.addRegexString("ERROR: (?:tablespace )?directory \".*[\\\\/]tablespace\\d+\" does not exist"); errors.add("ERROR: permission denied"); errors.add("ERROR: already exists"); errors.add("ERROR: is not empty"); @@ -23,7 +22,7 @@ public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { public static SQLQueryAdapter generate(PostgresGlobalState globalState) { // Skip tablespace generation if the option is disabled PostgresOptions options = globalState.getDbmsSpecificOptions(); - if (!options.testTablespaces) { + if (!options.isTestTablespaces()) { return null; } return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); @@ -31,7 +30,7 @@ public static SQLQueryAdapter generate(PostgresGlobalState globalState) { private SQLQueryAdapter generateTableSpace() { StringBuilder sb = new StringBuilder(); - int tableSpaceNum = globalState.getRandomly().getInteger(1, 5); + int tableSpaceNum = globalState.getRandomly().getInteger(1, Integer.MAX_VALUE); // CREATE TABLESPACE syntax sb.append("CREATE TABLESPACE "); @@ -39,9 +38,9 @@ private SQLQueryAdapter generateTableSpace() { sb.append(tableSpaceNum); sb.append(" LOCATION '"); - // Get the base path from options and append the tablespace number + // Get the validated base path from options and append the tablespace number PostgresOptions options = globalState.getDbmsSpecificOptions(); - String path = options.tablespacePath + tableSpaceNum; + String path = options.getTablespacePath() + tableSpaceNum; // Convert backslashes to forward slashes for PostgreSQL path = path.replace('\\', '/'); From 202bd3da76c5045d8f741bf5cb49364246d4590e Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Tue, 17 Jun 2025 22:48:16 -0400 Subject: [PATCH 1120/1284] Reformat the code --- src/sqlancer/postgres/PostgresOptions.java | 31 +++++++++++---------- src/sqlancer/postgres/PostgresProvider.java | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 62a115cf4..ebef13a16 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -14,7 +14,7 @@ public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; - private static Boolean defaultTestTablespaces = null; + private static Boolean defaultTestTablespaces; @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) public boolean allowBulkInsert; @@ -22,11 +22,14 @@ public class PostgresOptions implements DBMSSpecificOptions oracle = Arrays.asList(PostgresOracleFactory.QUERY_PARTITIONING); + @Parameter(names = "--connection-timeout", description = "Timeout in seconds for connecting to the server", arity = 1) + public int connectionTimeoutInSeconds; + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) public boolean testCollations = true; @Parameter(names = "--test-tablespaces", description = "Specifies whether to test tablespace creation (default is OS-dependent)", arity = 1) - public boolean testTablespaces = false; + public boolean testTablespaces; @Parameter(names = "--tablespace-path", description = "Base path for tablespace directories (default is OS-dependent)", arity = 1) public String tablespacePath = getDefaultTablespacePath(); @@ -76,31 +79,29 @@ public List getTestOracleFactory() { } public String getTablespacePath() { - if (tablespacePath == null || tablespacePath.trim().isEmpty()) { + if (tablespacePath == null || tablespacePath.isBlank()) { throw new AssertionError("Tablespace path is null or empty. Please configure --tablespace-path"); } - + File path = new File(tablespacePath); - + // Check if the directory exists or can be created - if (!path.exists()) { - if (!path.mkdirs()) { - throw new AssertionError("Cannot create tablespace directory: " + tablespacePath + - ". Please ensure the parent directory exists and you have write permissions."); - } + if (!path.exists() && !path.mkdirs()) { + throw new AssertionError("Cannot create tablespace directory: " + tablespacePath + + ". Please ensure the parent directory exists and you have write permissions."); } - + // Check if it's actually a directory if (!path.isDirectory()) { throw new AssertionError("Tablespace path is not a directory: " + tablespacePath); } - + // Check write permissions if (!path.canWrite()) { - throw new AssertionError("No write permissions for tablespace directory: " + tablespacePath + - ". Please ensure you have write permissions to this directory."); + throw new AssertionError("No write permissions for tablespace directory: " + tablespacePath + + ". Please ensure you have write permissions to this directory."); } - + return tablespacePath; } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index c3f161a7e..5b22cd83d 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -126,7 +126,7 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(PostgresViewGenerator::create), + CREATE_VIEW(PostgresViewGenerator::create), // CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); private final SQLQueryProvider sqlQueryProvider; From 0cc62d25ef318aa4b6ea5cf7336cbaabaa736420 Mon Sep 17 00:00:00 2001 From: HDQ Date: Wed, 9 Jul 2025 12:30:32 +0800 Subject: [PATCH 1121/1284] Add the SQLQueryError class for DQE and its test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The core idea of DQE is that the SELECT, UPDATE and DELETE queries with the same predicate φ should access the same rows. If these queries access different rows, DQE reveals a potential logic bug in the target DBMS. SQLQueryError class is used to compare the errors --- src/sqlancer/common/query/SQLQueryError.java | 116 ++++++++++++++++++ .../common/query/SQLQueryErrorTest.java | 93 ++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 src/sqlancer/common/query/SQLQueryError.java create mode 100644 test/sqlancer/common/query/SQLQueryErrorTest.java diff --git a/src/sqlancer/common/query/SQLQueryError.java b/src/sqlancer/common/query/SQLQueryError.java new file mode 100644 index 000000000..cde2a40fb --- /dev/null +++ b/src/sqlancer/common/query/SQLQueryError.java @@ -0,0 +1,116 @@ +package sqlancer.common.query; + +public class SQLQueryError implements Comparable { + + public enum ErrorLevel { + WARNING, ERROR + } + + private ErrorLevel level; + private int code; + private String message; + + public void setLevel(String level) { + // value of is case-sensitive + this.level = ErrorLevel.valueOf(level.toUpperCase()); + } + + public void setCode(int code) { + this.code = code; + } + + public void setMessage(String message) { + this.message = message; + } + + public ErrorLevel getLevel() { + return level; + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + public boolean hasSameLevel(SQLQueryError that) { + if (level == null && that.getLevel() == null) { + return true; + } else if (level != null && level.equals(that.getLevel())) { + return true; + } else { + return false; + } + } + + public boolean hasSameCodeAndMessage(SQLQueryError that) { + if (code != that.getCode()) { + return false; + } + if (message == null && that.getMessage() == null) { + return true; + } else if (message != null && message.equals(that.getMessage())) { + return true; + } else { + return false; + } + } + + @Override + public boolean equals(Object that) { + if (that == null) { + return false; + } + if (that instanceof SQLQueryError) { + SQLQueryError thatError = (SQLQueryError) that; + if (hasSameLevel(thatError) && hasSameCodeAndMessage(thatError)) { + return true; + } + } + return false; + } + + @Override + public String toString() { + return String.format("Level: %s; Code: %d; Message: %s.", level, code, message); + } + + @Override + public int compareTo(SQLQueryError that) { + if (code < that.getCode()) { + return -1; + } else if (code > that.getCode()) { + return 1; + } + + if (level == null && that.getLevel() != null) { + return -1; + } + if (level != null && that.getLevel() == null) { + return 1; + } + if (level != null && that.getLevel() != null) { + int res = level.compareTo(that.getLevel()); + if (res != 0) { + return res; + } + } + + if (message == null && that.getMessage() != null) { + return -1; + } + if (message != null && that.getMessage() == null) { + return 1; + } + if (message != null && that.getMessage() != null) { + int res = message.compareTo(that.getMessage()); + if (res != 0) { + return res; + } + } + + return 0; + } +} \ No newline at end of file diff --git a/test/sqlancer/common/query/SQLQueryErrorTest.java b/test/sqlancer/common/query/SQLQueryErrorTest.java new file mode 100644 index 000000000..0b80de176 --- /dev/null +++ b/test/sqlancer/common/query/SQLQueryErrorTest.java @@ -0,0 +1,93 @@ +package sqlancer.common.query; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SQLQueryErrorTest { + @Test + public void testSettersAndGetters() { + SQLQueryError error = new SQLQueryError(); + error.setLevel("error"); + error.setCode(123); + error.setMessage("Test message"); + assertEquals(SQLQueryError.ErrorLevel.ERROR, error.getLevel()); + assertEquals(123, error.getCode()); + assertEquals("Test message", error.getMessage()); + } + + @Test + public void testHasSameLevel() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setLevel("warning"); + e2.setLevel("warning"); + assertTrue(e1.hasSameLevel(e2)); + e2.setLevel("error"); + assertFalse(e1.hasSameLevel(e2)); + } + + @Test + public void testHasSameCodeAndMessage() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setCode(1); + e2.setCode(1); + e1.setMessage("msg"); + e2.setMessage("msg"); + assertTrue(e1.hasSameCodeAndMessage(e2)); + e2.setCode(2); + assertFalse(e1.hasSameCodeAndMessage(e2)); + e2.setCode(1); + e2.setMessage("other"); + assertFalse(e1.hasSameCodeAndMessage(e2)); + } + + @Test + public void testEquals() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setLevel("error"); + e1.setCode(1); + e1.setMessage("msg"); + e2.setLevel("error"); + e2.setCode(1); + e2.setMessage("msg"); + assertEquals(e1, e2); + e2.setLevel("warning"); + assertNotEquals(e1, e2); + } + + @Test + public void testToString() { + SQLQueryError e = new SQLQueryError(); + e.setLevel("error"); + e.setCode(1); + e.setMessage("msg"); + String str = e.toString(); + assertTrue(str.contains("Level: ERROR")); + assertTrue(str.contains("Code: 1")); + assertTrue(str.contains("Message: msg")); + } + + @Test + public void testCompareTo() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setCode(1); + e2.setCode(2); + assertTrue(e1.compareTo(e2) < 0); + e2.setCode(1); + e1.setLevel("error"); + e2.setLevel("warning"); + assertTrue(e1.compareTo(e2) > 0 || e1.compareTo(e2) < 0); + e2.setLevel("error"); + e1.setMessage("a"); + e2.setMessage("b"); + assertTrue(e1.compareTo(e2) < 0); + } +} + From cc561e9feb760ffd7d6e11ff3626abc97c08fbf9 Mon Sep 17 00:00:00 2001 From: HDQ Date: Tue, 15 Jul 2025 21:47:47 +0800 Subject: [PATCH 1122/1284] Promote ErrorLevel and compare methods --- src/sqlancer/common/query/SQLQueryError.java | 57 ++++++++----------- .../common/query/SQLQueryErrorTest.java | 22 +++---- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/src/sqlancer/common/query/SQLQueryError.java b/src/sqlancer/common/query/SQLQueryError.java index cde2a40fb..4ab253616 100644 --- a/src/sqlancer/common/query/SQLQueryError.java +++ b/src/sqlancer/common/query/SQLQueryError.java @@ -10,9 +10,8 @@ public enum ErrorLevel { private int code; private String message; - public void setLevel(String level) { - // value of is case-sensitive - this.level = ErrorLevel.valueOf(level.toUpperCase()); + public void setLevel(ErrorLevel level) { + this.level = level; } public void setCode(int code) { @@ -36,12 +35,10 @@ public String getMessage() { } public boolean hasSameLevel(SQLQueryError that) { - if (level == null && that.getLevel() == null) { - return true; - } else if (level != null && level.equals(that.getLevel())) { - return true; + if (level == null) { + return that.getLevel() == null; } else { - return false; + return level.equals(that.getLevel()); } } @@ -49,12 +46,10 @@ public boolean hasSameCodeAndMessage(SQLQueryError that) { if (code != that.getCode()) { return false; } - if (message == null && that.getMessage() == null) { - return true; - } else if (message != null && message.equals(that.getMessage())) { - return true; + if (message == null) { + return that.getMessage() == null; } else { - return false; + return message.equals(that.getMessage()); } } @@ -65,9 +60,7 @@ public boolean equals(Object that) { } if (that instanceof SQLQueryError) { SQLQueryError thatError = (SQLQueryError) that; - if (hasSameLevel(thatError) && hasSameCodeAndMessage(thatError)) { - return true; - } + return hasSameLevel(thatError) && hasSameCodeAndMessage(thatError); } return false; } @@ -87,27 +80,27 @@ public int compareTo(SQLQueryError that) { if (level == null && that.getLevel() != null) { return -1; - } - if (level != null && that.getLevel() == null) { - return 1; - } - if (level != null && that.getLevel() != null) { - int res = level.compareTo(that.getLevel()); - if (res != 0) { - return res; + } else { + if (that.getLevel() == null) { + return 1; + } else { + int res = level.compareTo(that.getLevel()); + if (res != 0) { + return res; + } } } if (message == null && that.getMessage() != null) { return -1; - } - if (message != null && that.getMessage() == null) { - return 1; - } - if (message != null && that.getMessage() != null) { - int res = message.compareTo(that.getMessage()); - if (res != 0) { - return res; + } else { + if (that.getMessage() == null) { + return 1; + } else { + int res = message.compareTo(that.getMessage()); + if (res != 0) { + return res; + } } } diff --git a/test/sqlancer/common/query/SQLQueryErrorTest.java b/test/sqlancer/common/query/SQLQueryErrorTest.java index 0b80de176..e0065788e 100644 --- a/test/sqlancer/common/query/SQLQueryErrorTest.java +++ b/test/sqlancer/common/query/SQLQueryErrorTest.java @@ -11,7 +11,7 @@ public class SQLQueryErrorTest { @Test public void testSettersAndGetters() { SQLQueryError error = new SQLQueryError(); - error.setLevel("error"); + error.setLevel(SQLQueryError.ErrorLevel.ERROR); error.setCode(123); error.setMessage("Test message"); assertEquals(SQLQueryError.ErrorLevel.ERROR, error.getLevel()); @@ -23,10 +23,10 @@ public void testSettersAndGetters() { public void testHasSameLevel() { SQLQueryError e1 = new SQLQueryError(); SQLQueryError e2 = new SQLQueryError(); - e1.setLevel("warning"); - e2.setLevel("warning"); + e1.setLevel(SQLQueryError.ErrorLevel.WARNING); + e2.setLevel(SQLQueryError.ErrorLevel.WARNING); assertTrue(e1.hasSameLevel(e2)); - e2.setLevel("error"); + e2.setLevel(SQLQueryError.ErrorLevel.ERROR); assertFalse(e1.hasSameLevel(e2)); } @@ -50,21 +50,21 @@ public void testHasSameCodeAndMessage() { public void testEquals() { SQLQueryError e1 = new SQLQueryError(); SQLQueryError e2 = new SQLQueryError(); - e1.setLevel("error"); + e1.setLevel(SQLQueryError.ErrorLevel.ERROR); e1.setCode(1); e1.setMessage("msg"); - e2.setLevel("error"); + e2.setLevel(SQLQueryError.ErrorLevel.ERROR); e2.setCode(1); e2.setMessage("msg"); assertEquals(e1, e2); - e2.setLevel("warning"); + e2.setLevel(SQLQueryError.ErrorLevel.WARNING); assertNotEquals(e1, e2); } @Test public void testToString() { SQLQueryError e = new SQLQueryError(); - e.setLevel("error"); + e.setLevel(SQLQueryError.ErrorLevel.ERROR); e.setCode(1); e.setMessage("msg"); String str = e.toString(); @@ -81,10 +81,10 @@ public void testCompareTo() { e2.setCode(2); assertTrue(e1.compareTo(e2) < 0); e2.setCode(1); - e1.setLevel("error"); - e2.setLevel("warning"); + e1.setLevel(SQLQueryError.ErrorLevel.ERROR); + e2.setLevel(SQLQueryError.ErrorLevel.WARNING); assertTrue(e1.compareTo(e2) > 0 || e1.compareTo(e2) < 0); - e2.setLevel("error"); + e2.setLevel(SQLQueryError.ErrorLevel.ERROR); e1.setMessage("a"); e2.setMessage("b"); assertTrue(e1.compareTo(e2) < 0); From f12c6e1a4b0bedc196d07a538bd926b7278cb47a Mon Sep 17 00:00:00 2001 From: HDQ Date: Wed, 16 Jul 2025 13:44:20 +0800 Subject: [PATCH 1123/1284] Run mvn formatter:format --- src/sqlancer/postgres/PostgresProvider.java | 6 +++--- test/sqlancer/common/query/SQLQueryErrorTest.java | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 9fb975667..9cf4c5a75 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -279,14 +279,14 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE } Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - + String dropCommand = "DROP DATABASE"; boolean forceDrop = Randomly.getBoolean(); if (forceDrop) { dropCommand += " FORCE"; } dropCommand += " IF EXISTS " + databaseName; - + globalState.getState().logStatement(dropCommand + ";"); try (Statement s = con.createStatement()) { s.execute(dropCommand); @@ -302,7 +302,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE throw e; } } - + // Create database section createDatabaseCommand = getCreateDatabaseCommand(globalState); globalState.getState().logStatement(createDatabaseCommand + ";"); diff --git a/test/sqlancer/common/query/SQLQueryErrorTest.java b/test/sqlancer/common/query/SQLQueryErrorTest.java index e0065788e..dea1e7a53 100644 --- a/test/sqlancer/common/query/SQLQueryErrorTest.java +++ b/test/sqlancer/common/query/SQLQueryErrorTest.java @@ -90,4 +90,3 @@ public void testCompareTo() { assertTrue(e1.compareTo(e2) < 0); } } - From ab1bead036362dff7d986e47dacdb1e825164f79 Mon Sep 17 00:00:00 2001 From: HDQ Date: Wed, 16 Jul 2025 14:31:51 +0800 Subject: [PATCH 1124/1284] Rewrite hashCode method for SQLQueryError.java --- src/sqlancer/common/query/SQLQueryError.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/common/query/SQLQueryError.java b/src/sqlancer/common/query/SQLQueryError.java index 4ab253616..41f604930 100644 --- a/src/sqlancer/common/query/SQLQueryError.java +++ b/src/sqlancer/common/query/SQLQueryError.java @@ -1,5 +1,7 @@ package sqlancer.common.query; +import java.util.Objects; + public class SQLQueryError implements Comparable { public enum ErrorLevel { @@ -65,6 +67,11 @@ public boolean equals(Object that) { return false; } + @Override + public int hashCode() { + return Objects.hash(level, code, message); + } + @Override public String toString() { return String.format("Level: %s; Code: %d; Message: %s.", level, code, message); @@ -106,4 +113,4 @@ public int compareTo(SQLQueryError that) { return 0; } -} \ No newline at end of file +} From 4b0b66d6bf02e76ebe084188dbb990cfd73a796c Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 26 Jul 2025 21:59:06 +0800 Subject: [PATCH 1125/1284] fix wrong value in the tidb_enable_stmt_summary Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 11f599fc0..c59d0dc2f 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -47,7 +47,7 @@ private enum Action { // java.sql.SQLException: Fast analyze hasn't reached General Availability and only support analyze version 1 // currently TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), - TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions(0, 1)), + TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("", "table", "global")), TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // TIDB_ENABLE_CHUNK_RPC("tidb_enable_chunk_rpc", (r) -> Randomly.fromOptions(0, 1)); From 603d31ae8f4f16552977224e915e7cf39d09855d Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 27 Jul 2025 11:15:15 +0800 Subject: [PATCH 1126/1284] update tidb version as v8.5.2 Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 888c04d06..484c241d5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -609,7 +609,7 @@ jobs: - name: Set up TiDB run: | docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v7.5.1 + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" From 5aace839806814bd8e365c902f1852567aa3682f Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 27 Jul 2025 18:32:32 -0400 Subject: [PATCH 1127/1284] Removing permision denied error --- src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java index 6c44043b8..3890d5160 100644 --- a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -13,7 +13,6 @@ public class PostgresTableSpaceGenerator { public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { this.globalState = globalState; errors.addRegexString("ERROR: (?:tablespace )?directory \".*[\\\\/]tablespace\\d+\" does not exist"); - errors.add("ERROR: permission denied"); errors.add("ERROR: already exists"); errors.add("ERROR: is not empty"); errors.add("ERROR: cannot be created because system does not support tablespaces"); From 10dc6f6726844154ad41270bb08c96e83b5c9443 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 11:56:05 +0800 Subject: [PATCH 1128/1284] update tidb image Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 4 ++-- .settings/org.eclipse.jdt.core.prefs | 9 ++++++--- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 484c241d5..b059b1859 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -608,8 +608,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 + docker pull hawkingrei/tidb-playground:v8.5.2 + docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:v8.5.2 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 0a659c1a7..3af1c5ad8 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.compliance=11 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,6 +28,7 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -73,6 +74,7 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -110,5 +112,6 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=1.8 +org.eclipse.jdt.core.compiler.source=11 diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index c59d0dc2f..f32e77980 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -47,7 +47,7 @@ private enum Action { // java.sql.SQLException: Fast analyze hasn't reached General Availability and only support analyze version 1 // currently TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), - TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("", "table", "global")), + TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("", "`table`", "global")), TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // TIDB_ENABLE_CHUNK_RPC("tidb_enable_chunk_rpc", (r) -> Randomly.fromOptions(0, 1)); From b02c1af5a4aadec8ac2711f4fc3baac368133611 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 12:27:14 +0800 Subject: [PATCH 1129/1284] update Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 4 ++-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b059b1859..888c04d06 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -608,8 +608,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull hawkingrei/tidb-playground:v8.5.2 - docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:v8.5.2 + docker pull pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v7.5.1 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index f32e77980..c997f6717 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -47,7 +47,7 @@ private enum Action { // java.sql.SQLException: Fast analyze hasn't reached General Availability and only support analyze version 1 // currently TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), - TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("", "`table`", "global")), + TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("``", "`table`", "global")), TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // TIDB_ENABLE_CHUNK_RPC("tidb_enable_chunk_rpc", (r) -> Randomly.fromOptions(0, 1)); From df514783dbc9f5dc598d4488256926a4fd3bd22e Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 14:21:03 +0800 Subject: [PATCH 1130/1284] update Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 2 +- .settings/org.eclipse.core.resources.prefs | 4 ++++ .settings/org.eclipse.jdt.apt.core.prefs | 4 ++++ .settings/org.eclipse.jdt.core.prefs | 9 +++------ .settings/org.eclipse.m2e.core.prefs | 4 ++++ 5 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 .settings/org.eclipse.jdt.apt.core.prefs create mode 100644 .settings/org.eclipse.m2e.core.prefs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 888c04d06..484c241d5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -609,7 +609,7 @@ jobs: - name: Set up TiDB run: | docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v7.5.1 + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 000000000..104c8ec98 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +encoding/=UTF-8 +encoding/src=UTF-8 +encoding/test=UTF-8 diff --git a/.settings/org.eclipse.jdt.apt.core.prefs b/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 000000000..dfa4f3adb --- /dev/null +++ b/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=true +org.eclipse.jdt.apt.genSrcDir=target/generated-sources/annotations +org.eclipse.jdt.apt.genTestSrcDir=target/generated-test-sources/test-annotations diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 3af1c5ad8..0a659c1a7 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=11 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,7 +28,6 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore -org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -74,7 +73,6 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore -org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -112,6 +110,5 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=11 +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 000000000..f897a7f1c --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 From ccf25fd730006cee0b77a2a4faa307a9e9ad9c33 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 14:23:59 +0800 Subject: [PATCH 1131/1284] update Signed-off-by: Weizhen Wang --- .settings/org.eclipse.core.resources.prefs | 4 ---- .settings/org.eclipse.jdt.apt.core.prefs | 4 ---- .settings/org.eclipse.m2e.core.prefs | 4 ---- 3 files changed, 12 deletions(-) delete mode 100644 .settings/org.eclipse.core.resources.prefs delete mode 100644 .settings/org.eclipse.jdt.apt.core.prefs delete mode 100644 .settings/org.eclipse.m2e.core.prefs diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 104c8ec98..000000000 --- a/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,4 +0,0 @@ -eclipse.preferences.version=1 -encoding/=UTF-8 -encoding/src=UTF-8 -encoding/test=UTF-8 diff --git a/.settings/org.eclipse.jdt.apt.core.prefs b/.settings/org.eclipse.jdt.apt.core.prefs deleted file mode 100644 index dfa4f3adb..000000000 --- a/.settings/org.eclipse.jdt.apt.core.prefs +++ /dev/null @@ -1,4 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.apt.aptEnabled=true -org.eclipse.jdt.apt.genSrcDir=target/generated-sources/annotations -org.eclipse.jdt.apt.genTestSrcDir=target/generated-test-sources/test-annotations diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs deleted file mode 100644 index f897a7f1c..000000000 --- a/.settings/org.eclipse.m2e.core.prefs +++ /dev/null @@ -1,4 +0,0 @@ -activeProfiles= -eclipse.preferences.version=1 -resolveWorkspaceProjects=true -version=1 From 279049acf9715bb6f710425d87cb3d9d321f31a0 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 14:52:02 +0800 Subject: [PATCH 1132/1284] all test for tidb use v8.5.2 Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 484c241d5..457ef173e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -608,7 +608,7 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:latest + docker pull pingcap/tidb:v8.5.2 docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 sleep 10 - name: Create SQLancer user @@ -635,8 +635,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + docker pull pingcap/tidb:v8.5.2 + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" From 6c8166b53adbed337090ba8aec5e1086e3514a90 Mon Sep 17 00:00:00 2001 From: HDQ Date: Sun, 3 Aug 2025 19:34:01 +0800 Subject: [PATCH 1133/1284] Introduce DQE for MySQL in SQLancer and add test to workflow --- .github/workflows/main.yml | 1 + src/sqlancer/common/oracle/DQEBase.java | 156 ++++++ .../common/query/SQLQueryAdapter.java | 66 ++- .../common/query/SQLancerResultSet.java | 19 + src/sqlancer/mysql/MySQLOracleFactory.java | 7 + src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 518 ++++++++++++++++++ test/sqlancer/dbms/TestMySQLDQE.java | 21 + 7 files changed, 781 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/common/oracle/DQEBase.java create mode 100644 src/sqlancer/mysql/oracle/MySQLDQEOracle.java create mode 100644 test/sqlancer/dbms/TestMySQLDQE.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 457ef173e..7b0e4e410 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -467,6 +467,7 @@ jobs: MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLCERT + MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLDQE oceanbase: name: DBMS Tests (OceanBase) diff --git a/src/sqlancer/common/oracle/DQEBase.java b/src/sqlancer/common/oracle/DQEBase.java new file mode 100644 index 000000000..3781ea227 --- /dev/null +++ b/src/sqlancer/common/oracle/DQEBase.java @@ -0,0 +1,156 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryError; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.mysql.MySQLSchema.MySQLTables; + +/* + * In DBMSs, SELECT, UPDATE and DELETE queries utilize predicates (i.e., WHERE clauses) to specify which rows to retrieve, update or delete, respectively. + * If they use the same predicate φ, they should access the same rows in a database. + * Ideally, DBMSs can adopt the same implementations for predicate evaluation in SELECT, UPDATE and DELETE queries. + * However, a DBMS usually adopts different implementations for predicate evaluation in SELECT, UPDATE and DELETE queries due to various optimization choices. + * Inconsistent implementations for predicate evaluation among these queries can cause SELECT, UPDATE and DELETE queries with the same predicate φ to access different rows. + * + * + * Inspired by this key observation, we propose Differential Query Execution(DQE), a novel and general approach to detect logic bugs in SELECT, UPDATE and DELETE queries. + * DQE solves the test oracle problem by executing SELECT, UPDATE and DELETE queries with the same predicate φ, and observing inconsistencies among their execution results. + * For example, if a row that is updated by an UPDATE query with a predicate φ does not appear in the query result of a SELECT query with the same predicate φ, a logic bug is detected in the target DBMS. + * The key challenge of DQE is to automatically obtain the accessed rows for a given SELECT, UPDATE or DELETE query. + * To address this challenge, we append two extra columns to each table in a database, to uniquely identify each row and track whether a row has been modified, respectively. + * We further rewrite SELECT and UPDATE queries to identify their accessed rows. + * + * more information see [DQE paper](https://ieeexplore.ieee.org/document/10172736) + */ + +public abstract class DQEBase> { + + public static final String COLUMN_ROWID = "rowId"; + public static final String COLUMN_UPDATED = "updated"; + + protected final S state; + protected final ExpectedErrors selectExpectedErrors = new ExpectedErrors(); + protected final ExpectedErrors updateExpectedErrors = new ExpectedErrors(); + protected final ExpectedErrors deleteExpectedErrors = new ExpectedErrors(); + + protected final Main.StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + + public DQEBase(S state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } + + public abstract String generateSelectStatement(MySQLTables tables, String tableName, String whereClauseStr); + + public abstract String generateUpdateStatement(MySQLTables tables, String tableName, String whereClauseStr); + + public abstract String generateDeleteStatement(MySQLTables tables, String tableName, String whereClauseStr); + + /** + * Add auxiliary columns to the database A abstract method, subclasses need to implement it. + * + * @param table + * + * @throws SQLException + */ + public abstract void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException; + + public void dropAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { + String tableName = table.getName(); + String dropColumnRowId = String.format("ALTER TABLE %s DROP COLUMN %s", tableName, COLUMN_ROWID); + new SQLQueryAdapter(dropColumnRowId).execute(state); + String dropColumnUpdated = String.format("ALTER TABLE %s DROP COLUMN %s", tableName, COLUMN_UPDATED); + new SQLQueryAdapter(dropColumnUpdated).execute(state); + } + + // This interface is to record Error code + public interface UpdateErrorCodes { + + } + + public interface ErrorCodeStrategy { + Set getUpdateSpecificErrorCodes(); + + Set getDeleteSpecificErrorCodes(); + + } + + /** + * The core idea of DQE is that the SELECT, UPDATE and DELETE queries with the same predicate φ should access the + * same rows. If these queries access different rows, DQE reveals a potential logic bug in the target DBMS. + */ + public static class SQLQueryResult { + + private final Map, Set> accessedRows; // Table name with respect rows + private final List queryErrors; + + public SQLQueryResult(Map, Set> accessedRows, + List queryErrors) { + this.accessedRows = accessedRows; + this.queryErrors = queryErrors; + } + + public Map, Set> getAccessedRows() { + return accessedRows; + } + + public List getQueryErrors() { + return queryErrors; + } + + public boolean hasEmptyErrors() { + return queryErrors.isEmpty(); + } + + public boolean hasErrors() { + return !hasEmptyErrors(); + } + + public boolean hasSameErrors(SQLQueryResult that) { + if (queryErrors.size() != that.getQueryErrors().size()) { + return false; + } else { + Collections.sort(queryErrors); + Collections.sort(that.getQueryErrors()); + for (int i = 0; i < queryErrors.size(); i++) { + if (!queryErrors.get(i).equals(that.getQueryErrors().get(i))) { + return false; + } + } + } + return true; + } + + public boolean hasAccessedRows() { + if (accessedRows.isEmpty()) { + return false; + } + for (Set accessedRow : accessedRows.values()) { + if (!accessedRow.isEmpty()) { + return true; + } + } + return false; + } + + public boolean hasSameAccessedRows(SQLQueryResult that) { + return accessedRows.equals(that.getAccessedRows()); + } + + } +} diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 3784ed635..0b94ae3fb 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -81,17 +81,55 @@ public String getUnterminatedQueryString() { return result; } + /** + * This method is used to mostly oracles, which need to report exceptions. We set the reportException parameter to + * true by default meaning that exceptions are reported. + * + * @param globalState + * @param fills + * + * @return whether the query was executed successfully + * + * @param + * + * @throws SQLException + */ @Override public > boolean execute(G globalState, String... fills) throws SQLException { + return execute(globalState, true, fills); + } + + /** + * This method is used to DQE oracles, DQE does not check exception separately, while other testing methods may + * need. We use reportException to control this behavior. For a specific DBMS used DQE oracle, we call this method + * and pass a boolean value of false as an argument. + * + * @param globalState + * @param reportException + * @param fills + * + * @return whether the query was executed successfully + * + * @param + * + * @throws SQLException + */ + public > boolean execute(G globalState, boolean reportException, + String... fills) throws SQLException { + return internalExecute(globalState.getConnection(), reportException, fills); + } + + protected > boolean internalExecute(SQLConnection connection, + boolean reportException, String... fills) throws SQLException { Statement s; if (fills.length > 0) { - s = globalState.getConnection().prepareStatement(fills[0]); + s = connection.prepareStatement(fills[0]); for (int i = 1; i < fills.length; i++) { ((PreparedStatement) s).setString(i, fills[i]); } } else { - s = globalState.getConnection().createStatement(); + s = connection.createStatement(); } try { if (fills.length > 0) { @@ -103,7 +141,9 @@ public String getUnterminatedQueryString() { return true; } catch (Exception e) { Main.nrUnsuccessfulActions.addAndGet(1); - checkException(e); + if (reportException) { + checkException(e); + } return false; } finally { s.close(); @@ -127,14 +167,24 @@ public void checkException(Exception e) throws AssertionError { @Override public > SQLancerResultSet executeAndGet(G globalState, String... fills) throws SQLException { + return executeAndGet(globalState, true, fills); + } + + public > SQLancerResultSet executeAndGet(G globalState, + boolean reportException, String... fills) throws SQLException { + return internalExecuteAndGet(globalState.getConnection(), reportException, fills); + } + + protected > SQLancerResultSet internalExecuteAndGet( + SQLConnection connection, boolean reportException, String... fills) throws SQLException { Statement s; if (fills.length > 0) { - s = globalState.getConnection().prepareStatement(fills[0]); + s = connection.prepareStatement(fills[0]); for (int i = 1; i < fills.length; i++) { ((PreparedStatement) s).setString(i, fills[i]); } } else { - s = globalState.getConnection().createStatement(); + s = connection.createStatement(); } ResultSet result; try { @@ -151,9 +201,11 @@ public void checkException(Exception e) throws AssertionError { } catch (Exception e) { s.close(); Main.nrUnsuccessfulActions.addAndGet(1); - checkException(e); + if (reportException) { + checkException(e); + } + return null; } - return null; } @Override diff --git a/src/sqlancer/common/query/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java index d1221a7f7..9466bcad2 100644 --- a/src/sqlancer/common/query/SQLancerResultSet.java +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -4,6 +4,8 @@ import java.sql.ResultSet; import java.sql.SQLException; +import sqlancer.common.query.SQLQueryError.ErrorLevel; + public class SQLancerResultSet implements Closeable { ResultSet rs; @@ -42,6 +44,23 @@ public String getString(int i) throws SQLException { } } + // This method is to fetch the result of SQL: "SHOW WARNINGS;" + public ErrorLevel getErrorLevel(String colName) throws SQLException { + String res = rs.getString(colName); + if (res == null) { + return ErrorLevel.WARNING; + } + return "ERROR".equalsIgnoreCase(res.trim()) ? ErrorLevel.ERROR : ErrorLevel.WARNING; + } + + public String getString(String colName) throws SQLException { + return rs.getString(colName); + } + + public int getInt(String colName) throws SQLException { + return rs.getInt(colName); + } + public boolean isClosed() throws SQLException { return rs.isClosed(); } diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index a0170d685..83e08677a 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -10,6 +10,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLancerResultSet; import sqlancer.mysql.gen.MySQLExpressionGenerator; +import sqlancer.mysql.oracle.MySQLDQEOracle; import sqlancer.mysql.oracle.MySQLDQPOracle; import sqlancer.mysql.oracle.MySQLFuzzer; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; @@ -75,5 +76,11 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLDQPOracle(globalState); } + }, + DQE { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLDQEOracle(globalState); + } }; } diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java new file mode 100644 index 000000000..142e56c22 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -0,0 +1,518 @@ +package sqlancer.mysql.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.beust.jcommander.Strings; + +import sqlancer.Randomly; +import sqlancer.common.oracle.DQEBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryError; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractTable; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + +public class MySQLDQEOracle extends DQEBase implements TestOracle { + + private final MySQLSchema schema; + private static final String APPEND_ORDER_BY = "%s ORDER BY %s"; + private static final String APPEND_LIMIT = "%s LIMIT %d"; + private final List orderColumns = new ArrayList<>(); + private boolean generateLimit; + private boolean generateOrderBy; + private boolean operateOnSingleTable; + private int limit; + + public MySQLDQEOracle(MySQLGlobalState state) { + super(state); + schema = state.getSchema(); + + MySQLErrors.addExpressionErrors(selectExpectedErrors); + + MySQLErrors.addExpressionErrors(updateExpectedErrors); + updateExpectedErrors.add("cannot be null"); + updateExpectedErrors.add("Duplicate entry"); + updateExpectedErrors.add("The value specified for generated column"); + + MySQLErrors.addExpressionErrors(deleteExpectedErrors); + deleteExpectedErrors.add("a foreign key constraint fails"); + } + + @Override + public String generateSelectStatement(MySQLTables tables, String tableName, String whereClauseStr) { + operateOnSingleTable = tables.getTables().size() == 1; + List selectColumns = new ArrayList<>(); + for (MySQLTable table : tables.getTables()) { + selectColumns.add(table.getName() + "." + COLUMN_ROWID); + } + if (operateOnSingleTable && Randomly.getBooleanWithSmallProbability()) { + generateOrderBy = true; + // generate order by columns + for (MySQLColumn column : Randomly.nonEmptySubset(tables.getColumns())) { + orderColumns.add(column.getFullQualifiedName()); + } + + if (Randomly.getBooleanWithRatherLowProbability()) { + generateLimit = true; + limit = (int) Randomly.getNotCachedInteger(1, 10); + } + } + + String selectStmt = String.format("SELECT %s FROM %s WHERE %s", Strings.join(",", selectColumns).toLowerCase(), + tableName, whereClauseStr); + if (generateOrderBy) { + selectStmt = String.format(APPEND_ORDER_BY, selectStmt, String.join(",", orderColumns)); + if (generateLimit) { + selectStmt = String.format(APPEND_LIMIT, selectStmt, limit); + } + } + return selectStmt; + } + + @Override + public String generateUpdateStatement(MySQLTables tables, String tableName, String whereClauseStr) { + List updateColumns = new ArrayList<>(); + for (MySQLTable table : tables.getTables()) { + updateColumns.add(String.format("%s = 1", table.getName() + "." + COLUMN_UPDATED)); + } + String updateStmt = String.format("UPDATE %s SET %s WHERE %s", tableName, Strings.join(",", updateColumns), + whereClauseStr); + if (generateOrderBy) { + updateStmt = String.format(APPEND_ORDER_BY, updateStmt, String.join(",", orderColumns)); + if (generateLimit) { + updateStmt = String.format(APPEND_LIMIT, updateStmt, limit); + } + } + return updateStmt; + } + + @Override + public String generateDeleteStatement(MySQLTables tables, String tableName, String whereClauseStr) { + String deleteStmt; + if (operateOnSingleTable) { + deleteStmt = String.format("DELETE FROM %s WHERE %s", tableName, whereClauseStr); + if (generateOrderBy) { + deleteStmt = String.format(APPEND_ORDER_BY, deleteStmt, String.join(",", orderColumns)); + if (generateLimit) { + deleteStmt = String.format(APPEND_LIMIT, deleteStmt, limit); + } + } + } else { + deleteStmt = String.format("DELETE %s FROM %s WHERE %s", tableName, tableName, whereClauseStr); + } + return deleteStmt; + } + + @Override + public void check() throws SQLException { + + MySQLTables tables = schema.getRandomTableNonEmptyTables(); + String tableName = tables.getTables().stream().map(AbstractTable::getName).collect(Collectors.joining(",")); + + // DQE does not support aggregate functions, windows functions + // This method does not generate them, may need some configurations if they can be generated + MySQLExpressionGenerator expressionGenerator = new MySQLExpressionGenerator(state) + .setColumns(tables.getColumns()); + MySQLExpression whereClause = expressionGenerator.generateExpression(); + + // MySQLVisitor is not deterministic, we should keep it only once. + // Especially, in MySQLUnaryPostfixOperation and MySQLUnaryPrefixOperation + String whereClauseStr = MySQLVisitor.asString(whereClause); + + // Generate a SELECT statement + String selectStmt = generateSelectStatement(tables, tableName, whereClauseStr); + + // Generate an UPDATE statement + String updateStmt = generateUpdateStatement(tables, tableName, whereClauseStr); + + // Generate a DELETE statement + String deleteStmt = generateDeleteStatement(tables, tableName, whereClauseStr); + + for (MySQLTable table : tables.getTables()) { + addAuxiliaryColumns(table); + } + + state.getState().getLocalState().log(selectStmt); + SQLQueryResult selectExecutionResult = executeSelect(selectStmt, tables); + state.getState().getLocalState().log(selectExecutionResult.getAccessedRows().values().toString()); + state.getState().getLocalState().log(selectExecutionResult.getQueryErrors().toString()); + + state.getState().getLocalState().log(updateStmt); + SQLQueryResult updateExecutionResult = executeUpdate(updateStmt, tables); + state.getState().getLocalState().log(updateExecutionResult.getAccessedRows().values().toString()); + state.getState().getLocalState().log(updateExecutionResult.getQueryErrors().toString()); + + state.getState().getLocalState().log(deleteStmt); + SQLQueryResult deleteExecutionResult = executeDelete(deleteStmt, tables); + state.getState().getLocalState().log(deleteExecutionResult.getAccessedRows().values().toString()); + state.getState().getLocalState().log(deleteExecutionResult.getQueryErrors().toString()); + + String compareSelectAndUpdate = compareSelectAndUpdate(selectExecutionResult, updateExecutionResult); + String compareSelectAndDelete = compareSelectAndDelete(selectExecutionResult, deleteExecutionResult); + String compareUpdateAndDelete = compareUpdateAndDelete(updateExecutionResult, deleteExecutionResult); + + String errorMessage = compareSelectAndUpdate == null ? "" : compareSelectAndUpdate + "\n"; + errorMessage += compareSelectAndDelete == null ? "" : compareSelectAndDelete + "\n"; + errorMessage += compareUpdateAndDelete == null ? "" : compareUpdateAndDelete + "\n"; + + if (!errorMessage.equals("")) { + throw new AssertionError(errorMessage); + } else { + state.getState().getLocalState().log("PASS"); + } + + for (MySQLTable table : tables.getTables()) { + dropAuxiliaryColumns(table); + } + } + + public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult updateResult) { + if (updateResult.hasEmptyErrors()) { + if (selectResult.hasErrors()) { + return "SELECT has errors, but UPDATE does not."; + } + if (!selectResult.hasSameAccessedRows(updateResult)) { + return "SELECT accessed different rows from UPDATE."; + } + return null; + } else { // update has errors + if (hasUpdateSpecificErrors(updateResult)) { + if (updateResult.hasAccessedRows()) { + return "UPDATE accessed non-empty rows when specific errors happen."; + } else { + // we do not compare update with select when update has specific errors + return null; + } + } + + // update errors should all appear in the select errors + List queryErrors = new ArrayList<>(selectResult.getQueryErrors()); + for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { + SQLQueryError updateError = updateResult.getQueryErrors().get(i); + boolean found = false; + for (int j = 0; j < queryErrors.size(); j++) { + SQLQueryError selectError = queryErrors.get(j); + if (selectError.hasSameCodeAndMessage(updateError)) { + queryErrors.remove(selectError); + found = true; + break; + } + } + if (!found) { + return "SELECT has different errors from UPDATE."; + } + } + + if (hasStopErrors(updateResult)) { + if (updateResult.hasAccessedRows()) { + return "UPDATE accessed non-empty rows when stop errors happen."; + } + } else { + if (!selectResult.hasSameAccessedRows(updateResult)) { + return "SELECT accessed different rows from UPDATE when errors happen."; + } + + } + + return null; + } + } + + public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { + if (deleteResult.hasEmptyErrors()) { + if (selectResult.hasErrors()) { + return "SELECT has errors, but DELETE does not."; + } + if (!selectResult.hasSameAccessedRows(deleteResult)) { + return "SELECT accessed different rows from DELETE."; + } + return null; + } else { // delete has errors + if (hasDeleteSpecificErrors(deleteResult)) { + if (deleteResult.hasAccessedRows()) { + return "DELETE accessed non-empty rows when specific errors happen."; + } else { + // we do not compare delete with select when delete has specific errors + return null; + } + } + + // delete errors should all appear in the select errors + List queryErrors = new ArrayList<>(selectResult.getQueryErrors()); + for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { + SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); + boolean found = false; + for (int j = 0; j < queryErrors.size(); j++) { + SQLQueryError selectError = queryErrors.get(j); + if (selectError.hasSameCodeAndMessage(deleteError)) { + queryErrors.remove(deleteError); + found = true; + break; + } + } + if (!found) { + return "SELECT has different errors from DELETE."; + } + } + + if (hasStopErrors(deleteResult)) { + if (deleteResult.hasAccessedRows()) { + return "DELETE accessed non-empty rows when stop errors happen."; + } + } else { + if (!selectResult.hasSameAccessedRows(deleteResult)) { + return "SELECT accessed different rows from DELETE when errors happen."; + } + } + + return null; + } + } + + public String compareUpdateAndDelete(SQLQueryResult updateResult, SQLQueryResult deleteResult) { + if (updateResult.hasEmptyErrors() && deleteResult.hasEmptyErrors()) { + if (updateResult.hasSameAccessedRows(deleteResult)) { + return null; + } else { + return "UPDATE accessed different rows from DELETE."; + } + } else { // update or delete has errors + boolean hasSpecificErrors = false; + + if (hasUpdateSpecificErrors(updateResult)) { + hasSpecificErrors = true; + if (updateResult.hasAccessedRows()) { + return "UPDATE accessed non-empty rows when specific errors happen."; + } + } + + if (hasDeleteSpecificErrors(deleteResult)) { + hasSpecificErrors = true; + if (deleteResult.hasAccessedRows()) { + return "DELETE accessed non-empty rows when specific errors happen."; + } + } + + // when one of these statements has specific errors, do not compare them + if (hasSpecificErrors) { + return null; + } + + if (!updateResult.hasSameErrors(deleteResult)) { + return "UPDATE has different errors from DELETE."; + } else { + if (!hasStopErrors(updateResult)) { + if (!updateResult.hasSameAccessedRows(deleteResult)) { + return "UPDATE accessed different rows from DELETE."; + } + } else { + if (updateResult.hasAccessedRows() || deleteResult.hasAccessedRows()) { + return "UPDATE or DELETE accessed non-empty rows when stop errors happen."; + } + } + } + + return null; + } + } + + /* + * when update violates column constraints, such as not null, unique, primary key and generated column, we cannot + * compare it with other queries. + */ + private boolean hasUpdateSpecificErrors(SQLQueryResult updateResult) { + return updateResult.getQueryErrors().stream().anyMatch( + error -> new MySQLErrorCodeStrategy().getUpdateSpecificErrorCodes().contains(error.getCode())); + } + + /* + * when delete violates column constraints, such as foreign key, we cannot compare it with other queries. + */ + private boolean hasDeleteSpecificErrors(SQLQueryResult deleteResult) { + return deleteResult.getQueryErrors().stream().anyMatch( + error -> new MySQLErrorCodeStrategy().getDeleteSpecificErrorCodes().contains(error.getCode())); + + } + + private boolean hasStopErrors(SQLQueryResult queryResult) { + return queryResult.getQueryErrors().stream() + .anyMatch(error -> error.getLevel() == SQLQueryError.ErrorLevel.ERROR); + } + + private SQLQueryResult executeSelect(String selectStmt, MySQLTables tables) throws SQLException { + Map, Set> accessedRows = new HashMap<>(); + List queryErrors; + SQLancerResultSet resultSet = null; + try { + resultSet = new SQLQueryAdapter(selectStmt, selectExpectedErrors).executeAndGet(state, false); + } catch (SQLException ignored) { + // we ignore this error, and use get errors to catch it + } finally { + queryErrors = getErrors(); + + if (resultSet != null) { + for (MySQLTable table : tables.getTables()) { + HashSet rows = new HashSet<>(); + accessedRows.put(table, rows); + } + while (resultSet.next()) { + for (MySQLTable table : tables.getTables()) { + accessedRows.get(table).add(resultSet.getString(table.getName() + "." + COLUMN_ROWID)); + } + } + resultSet.close(); + } + } + + return new SQLQueryResult(accessedRows, queryErrors); + } + + private SQLQueryResult executeUpdate(String updateStmt, MySQLTables tables) throws SQLException { + Map, Set> accessedRows = new HashMap<>(); + List queryErrors; + try { + new SQLQueryAdapter("BEGIN").execute(state, false); + new SQLQueryAdapter(updateStmt, updateExpectedErrors).execute(state, false); + } catch (SQLException ignored) { + // we ignore this error, and we use get errors to catch it + } finally { + queryErrors = getErrors(); + + for (MySQLTable table : tables.getTables()) { + String tableName = table.getName(); + String rowId = tableName + "." + COLUMN_ROWID; + String updated = tableName + "." + COLUMN_UPDATED; + String selectRowIdWithUpdated = String.format("SELECT %s FROM %s WHERE %s = 1", rowId, tableName, + updated); + SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowIdWithUpdated).executeAndGet(state, false); + HashSet rows = new HashSet<>(); + if (resultSet != null) { + while (resultSet.next()) { + rows.add(resultSet.getString(rowId)); + } + resultSet.close(); + } + accessedRows.put(table, rows); + } + + new SQLQueryAdapter("ROLLBACK").execute(state, false); + } + + return new SQLQueryResult(accessedRows, queryErrors); + } + + private SQLQueryResult executeDelete(String deleteStmt, MySQLTables tables) throws SQLException { + Map, Set> accessedRows = new HashMap<>(); + List queryErrors; + try { + for (MySQLTable table : tables.getTables()) { + String tableName = table.getName(); + String rowId = tableName + "." + COLUMN_ROWID; + String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); + SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowId).executeAndGet(state, false); + HashSet rows = new HashSet<>(); + if (resultSet != null) { + while (resultSet.next()) { + rows.add(resultSet.getString(rowId)); + } + resultSet.close(); + } + accessedRows.put(table, rows); + } + + new SQLQueryAdapter("BEGIN").execute(state, false); + new SQLQueryAdapter(deleteStmt, deleteExpectedErrors).execute(state, false); + } catch (SQLException ignored) { + // we ignore this error, and use get errors to catch it + } finally { + queryErrors = getErrors(); + + for (MySQLTable table : tables.getTables()) { + String tableName = table.getName(); + String rowId = tableName + "." + COLUMN_ROWID; + String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); + SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowId).executeAndGet(state, false); + HashSet rows = new HashSet<>(); + if (resultSet != null) { + while (resultSet.next()) { + rows.add(resultSet.getString(rowId)); + } + resultSet.close(); + } + + accessedRows.get(table).removeAll(rows); + } + + new SQLQueryAdapter("ROLLBACK").execute(state, false); + } + + return new SQLQueryResult(accessedRows, queryErrors); + } + + private List getErrors() throws SQLException { + SQLancerResultSet resultSet = new SQLQueryAdapter("SHOW WARNINGS").executeAndGet(state, false); + List queryErrors = new ArrayList<>(); + if (resultSet != null) { + while (resultSet.next()) { + SQLQueryError queryError = new SQLQueryError(); + queryError.setLevel(resultSet.getErrorLevel("Level")); + queryError.setCode(resultSet.getInt("Code")); + queryError.setMessage(resultSet.getString("Message")); + queryErrors.add(queryError); + } + resultSet.close(); + } + + return queryErrors; + } + + @Override + public void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { + String tableName = table.getName(); + + String addColumnRowID = String.format("ALTER TABLE %s ADD %s TEXT", tableName, COLUMN_ROWID); + new SQLQueryAdapter(addColumnRowID).execute(state, false); + state.getState().getLocalState().log(addColumnRowID); + + String addColumnUpdated = String.format("ALTER TABLE %s ADD %s INT DEFAULT 0", tableName, COLUMN_UPDATED); + new SQLQueryAdapter(addColumnUpdated).execute(state, false); + state.getState().getLocalState().log(addColumnUpdated); + + String updateRowsWithUniqueID = String.format("UPDATE %s SET %s = UUID()", tableName, COLUMN_ROWID); + new SQLQueryAdapter(updateRowsWithUniqueID).execute(state, false); + state.getState().getLocalState().log(updateRowsWithUniqueID); + } + + public static class MySQLErrorCodeStrategy implements ErrorCodeStrategy { + @Override + public Set getUpdateSpecificErrorCodes() { + // 1048, Column 'c0' cannot be null + // 1062, Duplicate entry '2' for key 't1.i0 + // 3105, The value specified for generated column 'c1' in table 't1' is not allowed + return Set.of(1048, 1062, 3105); + } + + @Override + public Set getDeleteSpecificErrorCodes() { + // 1451, Cannot delete or update a parent row: a foreign key constraint fails + return Set.of(1451); + } + } +} diff --git a/test/sqlancer/dbms/TestMySQLDQE.java b/test/sqlancer/dbms/TestMySQLDQE.java new file mode 100644 index 000000000..c137c1d0a --- /dev/null +++ b/test/sqlancer/dbms/TestMySQLDQE.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestMySQLDQE { + + @Test + public void testMySQL() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); + // Run with 0 queries as there are false positives for every mutation + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, + "mysql", "--oracle", "DQE" })); + } + +} From 1f64426f4cf3460ee3bd984e44382265a1059b06 Mon Sep 17 00:00:00 2001 From: HDQ Date: Fri, 8 Aug 2025 14:01:29 +0800 Subject: [PATCH 1134/1284] Fix: remove MySQL specific logic and reuse existing method --- .../common/query/SQLancerResultSet.java | 11 ------ src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 39 +++++-------------- 2 files changed, 10 insertions(+), 40 deletions(-) diff --git a/src/sqlancer/common/query/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java index 9466bcad2..912a6a882 100644 --- a/src/sqlancer/common/query/SQLancerResultSet.java +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -4,8 +4,6 @@ import java.sql.ResultSet; import java.sql.SQLException; -import sqlancer.common.query.SQLQueryError.ErrorLevel; - public class SQLancerResultSet implements Closeable { ResultSet rs; @@ -44,15 +42,6 @@ public String getString(int i) throws SQLException { } } - // This method is to fetch the result of SQL: "SHOW WARNINGS;" - public ErrorLevel getErrorLevel(String colName) throws SQLException { - String res = rs.getString(colName); - if (res == null) { - return ErrorLevel.WARNING; - } - return "ERROR".equalsIgnoreCase(res.trim()) ? ErrorLevel.ERROR : ErrorLevel.WARNING; - } - public String getString(String colName) throws SQLException { return rs.getString(colName); } diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index 142e56c22..d4e15de01 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -1,5 +1,7 @@ package sqlancer.mysql.oracle; +import static sqlancer.ComparatorHelper.getResultSetFirstColumnAsString; + import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; @@ -136,13 +138,10 @@ public void check() throws SQLException { // Especially, in MySQLUnaryPostfixOperation and MySQLUnaryPrefixOperation String whereClauseStr = MySQLVisitor.asString(whereClause); - // Generate a SELECT statement String selectStmt = generateSelectStatement(tables, tableName, whereClauseStr); - // Generate an UPDATE statement String updateStmt = generateUpdateStatement(tables, tableName, whereClauseStr); - // Generate a DELETE statement String deleteStmt = generateDeleteStatement(tables, tableName, whereClauseStr); for (MySQLTable table : tables.getTables()) { @@ -401,14 +400,8 @@ private SQLQueryResult executeUpdate(String updateStmt, MySQLTables tables) thro String updated = tableName + "." + COLUMN_UPDATED; String selectRowIdWithUpdated = String.format("SELECT %s FROM %s WHERE %s = 1", rowId, tableName, updated); - SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowIdWithUpdated).executeAndGet(state, false); - HashSet rows = new HashSet<>(); - if (resultSet != null) { - while (resultSet.next()) { - rows.add(resultSet.getString(rowId)); - } - resultSet.close(); - } + HashSet rows = new HashSet<>( + getResultSetFirstColumnAsString(selectRowIdWithUpdated, updateExpectedErrors, state)); accessedRows.put(table, rows); } @@ -426,14 +419,8 @@ private SQLQueryResult executeDelete(String deleteStmt, MySQLTables tables) thro String tableName = table.getName(); String rowId = tableName + "." + COLUMN_ROWID; String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); - SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowId).executeAndGet(state, false); - HashSet rows = new HashSet<>(); - if (resultSet != null) { - while (resultSet.next()) { - rows.add(resultSet.getString(rowId)); - } - resultSet.close(); - } + HashSet rows = new HashSet<>( + getResultSetFirstColumnAsString(selectRowId, deleteExpectedErrors, state)); accessedRows.put(table, rows); } @@ -448,15 +435,8 @@ private SQLQueryResult executeDelete(String deleteStmt, MySQLTables tables) thro String tableName = table.getName(); String rowId = tableName + "." + COLUMN_ROWID; String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); - SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowId).executeAndGet(state, false); - HashSet rows = new HashSet<>(); - if (resultSet != null) { - while (resultSet.next()) { - rows.add(resultSet.getString(rowId)); - } - resultSet.close(); - } - + HashSet rows = new HashSet<>( + getResultSetFirstColumnAsString(selectRowId, deleteExpectedErrors, state)); accessedRows.get(table).removeAll(rows); } @@ -472,7 +452,8 @@ private List getErrors() throws SQLException { if (resultSet != null) { while (resultSet.next()) { SQLQueryError queryError = new SQLQueryError(); - queryError.setLevel(resultSet.getErrorLevel("Level")); + queryError.setLevel(resultSet.getString("Level").equalsIgnoreCase("ERROR") + ? SQLQueryError.ErrorLevel.ERROR : SQLQueryError.ErrorLevel.WARNING); queryError.setCode(resultSet.getInt("Code")); queryError.setMessage(resultSet.getString("Message")); queryErrors.add(queryError); From 83cf8995f467467fc611aef71b1f66d1127c69de Mon Sep 17 00:00:00 2001 From: HDQ Date: Fri, 8 Aug 2025 16:13:48 +0800 Subject: [PATCH 1135/1284] refactor(MySQLDQEOracle): extract duplicated code as method --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 54 +++++++++---------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index d4e15de01..0e43d7a86 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -171,7 +171,7 @@ public void check() throws SQLException { errorMessage += compareSelectAndDelete == null ? "" : compareSelectAndDelete + "\n"; errorMessage += compareUpdateAndDelete == null ? "" : compareUpdateAndDelete + "\n"; - if (!errorMessage.equals("")) { + if (!errorMessage.isEmpty()) { throw new AssertionError(errorMessage); } else { state.getState().getLocalState().log("PASS"); @@ -190,7 +190,6 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult if (!selectResult.hasSameAccessedRows(updateResult)) { return "SELECT accessed different rows from UPDATE."; } - return null; } else { // update has errors if (hasUpdateSpecificErrors(updateResult)) { if (updateResult.hasAccessedRows()) { @@ -202,19 +201,10 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult } // update errors should all appear in the select errors - List queryErrors = new ArrayList<>(selectResult.getQueryErrors()); + List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); - boolean found = false; - for (int j = 0; j < queryErrors.size(); j++) { - SQLQueryError selectError = queryErrors.get(j); - if (selectError.hasSameCodeAndMessage(updateError)) { - queryErrors.remove(selectError); - found = true; - break; - } - } - if (!found) { + if (!isFound(selectErrors, updateError)) { return "SELECT has different errors from UPDATE."; } } @@ -227,11 +217,28 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult if (!selectResult.hasSameAccessedRows(updateResult)) { return "SELECT accessed different rows from UPDATE when errors happen."; } - } + } + return null; + } - return null; + /** + * + * @param selectErrors selectQueryErrors + * @param targetError update or delete queryError + * @return is targetError found in selectQueryErrors + */ + private static boolean isFound(List selectErrors, SQLQueryError targetError) { + boolean found = false; + for (int i = 0; i < selectErrors.size(); i++) { + SQLQueryError selectError = selectErrors.get(i); + if (selectError.hasSameCodeAndMessage(targetError)) { + selectErrors.remove(i); + found = true; + break; + } } + return found; } public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { @@ -242,7 +249,6 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult if (!selectResult.hasSameAccessedRows(deleteResult)) { return "SELECT accessed different rows from DELETE."; } - return null; } else { // delete has errors if (hasDeleteSpecificErrors(deleteResult)) { if (deleteResult.hasAccessedRows()) { @@ -254,19 +260,10 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult } // delete errors should all appear in the select errors - List queryErrors = new ArrayList<>(selectResult.getQueryErrors()); + List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); - boolean found = false; - for (int j = 0; j < queryErrors.size(); j++) { - SQLQueryError selectError = queryErrors.get(j); - if (selectError.hasSameCodeAndMessage(deleteError)) { - queryErrors.remove(deleteError); - found = true; - break; - } - } - if (!found) { + if (!isFound(selectErrors, deleteError)) { return "SELECT has different errors from DELETE."; } } @@ -280,9 +277,8 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult return "SELECT accessed different rows from DELETE when errors happen."; } } - - return null; } + return null; } public String compareUpdateAndDelete(SQLQueryResult updateResult, SQLQueryResult deleteResult) { From e1b5861557b01d4edbcbf766b0c9177d5e1c4f93 Mon Sep 17 00:00:00 2001 From: luliqwerty Date: Fri, 8 Aug 2025 16:29:31 +0800 Subject: [PATCH 1136/1284] refactor(MySQLDQEOracle): run mvn formatter:format --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index 0e43d7a86..bb6737215 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -224,8 +224,11 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult /** * - * @param selectErrors selectQueryErrors - * @param targetError update or delete queryError + * @param selectErrors + * selectQueryErrors + * @param targetError + * update or delete queryError + * * @return is targetError found in selectQueryErrors */ private static boolean isFound(List selectErrors, SQLQueryError targetError) { From 131febb402173a0477840e68de28e46a75c5bc77 Mon Sep 17 00:00:00 2001 From: luliqwerty Date: Sun, 10 Aug 2025 22:08:46 +0800 Subject: [PATCH 1137/1284] refactor: fix some code style - DQEBase: remove unnecessary method - MySQLDQEOracle: simplify adding expectedErrors - MySQLErrors: add a expectedErrors --- src/sqlancer/common/oracle/DQEBase.java | 7 ------- src/sqlancer/mysql/MySQLErrors.java | 1 + src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 13 +++++-------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/common/oracle/DQEBase.java b/src/sqlancer/common/oracle/DQEBase.java index 3781ea227..ba9af3bd1 100644 --- a/src/sqlancer/common/oracle/DQEBase.java +++ b/src/sqlancer/common/oracle/DQEBase.java @@ -1,7 +1,6 @@ package sqlancer.common.oracle; import java.sql.SQLException; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -117,16 +116,10 @@ public boolean hasEmptyErrors() { return queryErrors.isEmpty(); } - public boolean hasErrors() { - return !hasEmptyErrors(); - } - public boolean hasSameErrors(SQLQueryResult that) { if (queryErrors.size() != that.getQueryErrors().size()) { return false; } else { - Collections.sort(queryErrors); - Collections.sort(that.getQueryErrors()); for (int i = 0; i < queryErrors.size(); i++) { if (!queryErrors.get(i).equals(that.getQueryErrors().get(i))) { return false; diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 13159e49e..f01b30e03 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -53,6 +53,7 @@ public static List getInsertUpdateErrors() { errors.add("Data truncated for functional index"); errors.add("cannot be null"); errors.add("Incorrect decimal value"); + errors.add("The value specified for generated column"); return errors; } diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index bb6737215..4585fc830 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -49,9 +50,7 @@ public MySQLDQEOracle(MySQLGlobalState state) { MySQLErrors.addExpressionErrors(selectExpectedErrors); MySQLErrors.addExpressionErrors(updateExpectedErrors); - updateExpectedErrors.add("cannot be null"); - updateExpectedErrors.add("Duplicate entry"); - updateExpectedErrors.add("The value specified for generated column"); + MySQLErrors.addInsertUpdateErrors(updateExpectedErrors); MySQLErrors.addExpressionErrors(deleteExpectedErrors); deleteExpectedErrors.add("a foreign key constraint fails"); @@ -173,8 +172,6 @@ public void check() throws SQLException { if (!errorMessage.isEmpty()) { throw new AssertionError(errorMessage); - } else { - state.getState().getLocalState().log("PASS"); } for (MySQLTable table : tables.getTables()) { @@ -184,7 +181,7 @@ public void check() throws SQLException { public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult updateResult) { if (updateResult.hasEmptyErrors()) { - if (selectResult.hasErrors()) { + if (!selectResult.hasEmptyErrors()) { return "SELECT has errors, but UPDATE does not."; } if (!selectResult.hasSameAccessedRows(updateResult)) { @@ -246,7 +243,7 @@ private static boolean isFound(List selectErrors, SQLQueryError t public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { if (deleteResult.hasEmptyErrors()) { - if (selectResult.hasErrors()) { + if (!selectResult.hasEmptyErrors()) { return "SELECT has errors, but DELETE does not."; } if (!selectResult.hasSameAccessedRows(deleteResult)) { @@ -459,7 +456,7 @@ private List getErrors() throws SQLException { } resultSet.close(); } - + Collections.sort(queryErrors); return queryErrors; } From df600ff1251471783739fa4d4fd2fbea7ba66fef Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Wed, 13 Aug 2025 13:55:42 +0800 Subject: [PATCH 1138/1284] Serialize StateToReproduce directly --- src/sqlancer/Main.java | 20 ++++++++- src/sqlancer/MainOptions.java | 7 ++++ src/sqlancer/StateToReproduce.java | 67 ++++++++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index eb3ad7f0c..87023b0a2 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.Writer; import java.nio.file.Files; +import java.nio.file.Path; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -78,6 +79,7 @@ public static final class StateLogger { public FileWriter currentFileWriter; private FileWriter queryPlanFileWriter; private FileWriter reduceFileWriter; + private Path reproduceFilePath; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; @@ -127,7 +129,13 @@ public StateLogger(String databaseName, DatabaseProvider provider, Main reduceFileDir.mkdir(); } this.reduceFile = new File(reduceFileDir, databaseName + "-reduce.log"); - + } + if (options.serializeReproduceState()) { + File reproduceFileDir = new File(dir, "reproduce"); + if (!reproduceFileDir.exists()) { + reproduceFileDir.mkdir(); + } + reproduceFilePath = new File(reproduceFileDir, databaseName + ".ser").toPath(); } this.databaseProvider = provider; } @@ -341,6 +349,10 @@ private String removeNamesFromQueryPlans(String queryPlan) { result = result.replaceAll("i[0-9]+", "i0"); // Avoid duplicate indexes return result + "\n"; } + + public Path getReproduceFilePath() { + return reproduceFilePath; + } } public static class QueryManager { @@ -461,6 +473,9 @@ public void run() throws Exception { throw new AssertionError(e); } + if (options.serializeReproduceState() && reproducer != null) { + stateToRepro.serialize(logger.getReproduceFilePath()); + } if (options.reduceAST() && !options.useReducer()) { throw new AssertionError("To reduce AST, use-reducer option must be enabled first"); } @@ -675,6 +690,9 @@ private boolean run(MainOptions options, ExecutorService execService, executor.getStateToReproduce().exception = reduce.getMessage(); executor.getLogger().logFileWriter = null; executor.getLogger().logException(reduce, executor.getStateToReproduce()); + if (options.serializeReproduceState()) { + executor.getStateToReproduce().serialize(executor.getLogger().getReproduceFilePath()); + } return false; } finally { try { diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index a6a252fa3..a5142fcf0 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -123,6 +123,9 @@ public class MainOptions { @Parameter(names = "--database-prefix", description = "The prefix used for each database created") private String databasePrefix = "database"; // NOPMD + @Parameter(names = "--serialize-reproduce-state", description = "Serialize the state to reproduce") + private boolean serializeReproduceState = false; // NOPMD + @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") private boolean useReducer = false; // NOPMD @@ -304,6 +307,10 @@ public boolean performConnectionTest() { return useConnectionTest; } + public boolean serializeReproduceState() { + return serializeReproduceState; + } + public boolean useReducer() { return useReducer; } diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 33bfe3398..a8586745f 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -1,19 +1,26 @@ package sqlancer; import java.io.Closeable; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; import sqlancer.common.query.Query; -public class StateToReproduce { +public class StateToReproduce implements Serializable { + private static final long serialVersionUID = 1L; - private List> statements = new ArrayList<>(); + private transient List> statements = new ArrayList<>(); private final String databaseName; - private final DatabaseProvider databaseProvider; + private transient DatabaseProvider databaseProvider; public String databaseVersion; @@ -131,6 +138,60 @@ public OracleRunReproductionState createLocalState() { return new OracleRunReproductionState(); } + public void serialize(Path path) { + try (ObjectOutputStream oos = new ObjectOutputStream(Files.newOutputStream(path))) { + oos.writeObject(this); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + public static StateToReproduce deserialize(Path path) { + try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(path))) { + return (StateToReproduce) ois.readObject(); + } catch (IOException | ClassNotFoundException e) { + throw new AssertionError(e); + } + } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + + List statementStrings = new ArrayList<>(); + for (Query q : this.statements) { + statementStrings.add(q.getLogString()); + } + out.writeObject(statementStrings); + out.writeObject(this.databaseProvider != null ? this.databaseProvider.getDBMSName() : null); + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + this.statements = new ArrayList<>(); + List statementStrings = (List) in.readObject(); + String dbmsName = (String) in.readObject(); + + DatabaseProvider provider = null; + if (dbmsName != null) { + List> providers = Main.getDBMSProviders(); + for (DatabaseProvider p : providers) { + if (p.getDBMSName().equals(dbmsName)) { + provider = p; + break; + } + } + } + + if (provider == null || statementStrings == null) { + throw new AssertionError("Database provider or statement is null"); + } + this.databaseProvider = provider; + for (String s : statementStrings) { + this.statements.add(provider.getLoggableFactory().getQueryForStateToReproduce(s)); + } + } + public void setStatements(List> statements) { this.statements = statements; } From cfc7877f2d4a292284206b35f5fd1ad748fabe66 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Fri, 15 Aug 2025 14:31:53 +0800 Subject: [PATCH 1139/1284] Serialize full List> in StateToReproduce --- pom.xml | 1 + src/sqlancer/StateToReproduce.java | 17 +---------------- src/sqlancer/common/log/Loggable.java | 4 +++- src/sqlancer/common/query/ExpectedErrors.java | 4 +++- src/sqlancer/common/query/SQLQueryAdapter.java | 4 +++- 5 files changed, 11 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 734b44f5d..3292576a8 100644 --- a/pom.xml +++ b/pom.xml @@ -114,6 +114,7 @@ 11 ${project.basedir}/.settings/org.eclipse.jdt.core.prefs + -Xlint:-serial eclipse true diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index a8586745f..0e5bd1816 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -16,7 +16,7 @@ public class StateToReproduce implements Serializable { private static final long serialVersionUID = 1L; - private transient List> statements = new ArrayList<>(); + private List> statements = new ArrayList<>(); private final String databaseName; @@ -157,19 +157,11 @@ public static StateToReproduce deserialize(Path path) { private void writeObject(ObjectOutputStream out) throws IOException { out.defaultWriteObject(); - List statementStrings = new ArrayList<>(); - for (Query q : this.statements) { - statementStrings.add(q.getLogString()); - } - out.writeObject(statementStrings); out.writeObject(this.databaseProvider != null ? this.databaseProvider.getDBMSName() : null); } - @SuppressWarnings("unchecked") private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); - this.statements = new ArrayList<>(); - List statementStrings = (List) in.readObject(); String dbmsName = (String) in.readObject(); DatabaseProvider provider = null; @@ -182,14 +174,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE } } } - - if (provider == null || statementStrings == null) { - throw new AssertionError("Database provider or statement is null"); - } this.databaseProvider = provider; - for (String s : statementStrings) { - this.statements.add(provider.getLoggableFactory().getQueryForStateToReproduce(s)); - } } public void setStatements(List> statements) { diff --git a/src/sqlancer/common/log/Loggable.java b/src/sqlancer/common/log/Loggable.java index 0d5fef218..7796009ee 100644 --- a/src/sqlancer/common/log/Loggable.java +++ b/src/sqlancer/common/log/Loggable.java @@ -1,5 +1,7 @@ package sqlancer.common.log; -public interface Loggable { +import java.io.Serializable; + +public interface Loggable extends Serializable { String getLogString(); } diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index 38d5c9757..eb281efbe 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -1,5 +1,6 @@ package sqlancer.common.query; +import java.io.Serializable; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -11,7 +12,8 @@ * result in an error "UNIQUE constraint violated" when it attempts to insert a duplicate value in a column declared as * UNIQUE. */ -public class ExpectedErrors { +public class ExpectedErrors implements Serializable { + private static final long serialVersionUID = 1L; private final Set errors; private final Set regexes; diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 3784ed635..22cb704f8 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -1,5 +1,6 @@ package sqlancer.common.query; +import java.io.Serializable; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -9,7 +10,8 @@ import sqlancer.Main; import sqlancer.SQLConnection; -public class SQLQueryAdapter extends Query { +public class SQLQueryAdapter extends Query implements Serializable { + private static final long serialVersionUID = 1L; private final String query; private final ExpectedErrors expectedErrors; From 30d879acb9f1919ac8f8cbd622bda31e3edfb826 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Sun, 17 Aug 2025 16:32:52 +0800 Subject: [PATCH 1140/1284] Add serialVersionUID --- pom.xml | 1 - src/sqlancer/cnosdb/query/CnosDBOtherQuery.java | 2 ++ src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java | 1 + src/sqlancer/cnosdb/query/CnosDBSelectQuery.java | 1 + src/sqlancer/common/log/LoggedString.java | 1 + src/sqlancer/common/query/Query.java | 1 + src/sqlancer/common/query/SQLQueryResultCheckAdapter.java | 1 + src/sqlancer/postgres/gen/PostgresDiscardGenerator.java | 1 + src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java | 1 + test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java | 1 + 10 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3292576a8..734b44f5d 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,6 @@ 11 ${project.basedir}/.settings/org.eclipse.jdt.core.prefs - -Xlint:-serial eclipse true diff --git a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java index bd48c7b7e..f0a37056c 100644 --- a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java +++ b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java @@ -6,6 +6,8 @@ import sqlancer.common.query.ExpectedErrors; public class CnosDBOtherQuery extends CnosDBQueryAdapter { + private static final long serialVersionUID = 1L; + public CnosDBOtherQuery(String query, ExpectedErrors errors) { super(query, errors); } diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java index 2b2dda5e6..115f96ffc 100644 --- a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java +++ b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java @@ -5,6 +5,7 @@ import sqlancer.common.query.Query; public abstract class CnosDBQueryAdapter extends Query { + private static final long serialVersionUID = 1L; String query; ExpectedErrors errors; diff --git a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java index ca7f61c3a..1c9228182 100644 --- a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java +++ b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java @@ -7,6 +7,7 @@ import sqlancer.common.query.SQLancerResultSet; public class CnosDBSelectQuery extends CnosDBQueryAdapter { + private static final long serialVersionUID = 1L; CnosDBResultSet resultSet; public CnosDBSelectQuery(String query, ExpectedErrors errors) { diff --git a/src/sqlancer/common/log/LoggedString.java b/src/sqlancer/common/log/LoggedString.java index 4f449034f..696203a78 100644 --- a/src/sqlancer/common/log/LoggedString.java +++ b/src/sqlancer/common/log/LoggedString.java @@ -1,6 +1,7 @@ package sqlancer.common.log; public class LoggedString implements Loggable { + private static final long serialVersionUID = 1L; private final String loggedString; diff --git a/src/sqlancer/common/query/Query.java b/src/sqlancer/common/query/Query.java index 44efc0084..ca90619c7 100644 --- a/src/sqlancer/common/query/Query.java +++ b/src/sqlancer/common/query/Query.java @@ -5,6 +5,7 @@ import sqlancer.common.log.Loggable; public abstract class Query implements Loggable { + private static final long serialVersionUID = 1L; /** * Gets the query string, which is guaranteed to be terminated with a semicolon. diff --git a/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java index 8f6a01a4b..1eaae2424 100644 --- a/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java +++ b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java @@ -9,6 +9,7 @@ import sqlancer.SQLConnection; public class SQLQueryResultCheckAdapter extends SQLQueryAdapter { + private static final long serialVersionUID = 1L; private final Consumer rsChecker; diff --git a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java index c977fa272..0c02efcd0 100644 --- a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java @@ -25,6 +25,7 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { } sb.append(what); return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { + private static final long serialVersionUID = 1L; @Override public boolean couldAffectSchema() { diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java index 40ccdfa0d..156fb97da 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java @@ -25,6 +25,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { } sb.append(what); return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { + private static final long serialVersionUID = 1L; @Override public boolean couldAffectSchema() { diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java index 37389109f..9767723e0 100644 --- a/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java @@ -7,6 +7,7 @@ import java.sql.SQLException; public class VirtualDBQuery extends SQLQueryAdapter { + private static final long serialVersionUID = 1L; public VirtualDBQuery(String query) { // Since the base class must check the format From a28ede83778842b4468a814ebcf3a9670263597f Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Wed, 20 Aug 2025 13:43:07 +0800 Subject: [PATCH 1141/1284] Add serialization tests --- src/sqlancer/StateToReproduce.java | 4 + test/sqlancer/TestStateToReproduce.java | 109 ++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 test/sqlancer/TestStateToReproduce.java diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 0e5bd1816..c849b535d 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -47,6 +47,10 @@ public String getDatabaseVersion() { return databaseVersion; } + public DatabaseProvider getDatabaseProvider() { + return databaseProvider; + } + /** * Logs the statement string without executing the corresponding statement. * diff --git a/test/sqlancer/TestStateToReproduce.java b/test/sqlancer/TestStateToReproduce.java new file mode 100644 index 000000000..6fcdc22f1 --- /dev/null +++ b/test/sqlancer/TestStateToReproduce.java @@ -0,0 +1,109 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3Provider; + +public class TestStateToReproduce { + + @TempDir + Path tempDir; + + @Test + public void testBasicFields() throws IOException { + SQLite3Provider provider = new SQLite3Provider(); + StateToReproduce state = new StateToReproduce("test_db", provider); + state.databaseVersion = "3.36.0"; + state.seedValue = 12345L; + state.exception = "Test exception message"; + + Path file = tempDir.resolve("test_basic.ser"); + state.serialize(file); + StateToReproduce result = StateToReproduce.deserialize(file); + + assertEquals(state.getDatabaseName(), result.getDatabaseName()); + assertEquals(state.getDatabaseVersion(), result.getDatabaseVersion()); + assertEquals(state.getSeedValue(), result.getSeedValue()); + assertEquals(state.getException(), result.getException()); + } + + @Test + public void testStatements() throws IOException { + SQLite3Provider provider = new SQLite3Provider(); + StateToReproduce state = new StateToReproduce("test_statements", provider); + List> statements = new ArrayList<>(); + + ExpectedErrors errors1 = new ExpectedErrors(); + errors1.add("syntax error"); + errors1.add("table already exists"); + statements.add(new SQLQueryAdapter("CREATE TABLE test (id INTEGER);", errors1)); + + ExpectedErrors errors2 = new ExpectedErrors(); + errors2.add("constraint failed"); + statements.add(new SQLQueryAdapter("INSERT INTO test VALUES (1);", errors2)); + + statements.add(new SQLQueryAdapter("SELECT * FROM test;", new ExpectedErrors())); + state.setStatements(statements); + + Path file = tempDir.resolve("test_statements.ser"); + state.serialize(file); + StateToReproduce result = StateToReproduce.deserialize(file); + + List> resultStatements = result.getStatements(); + assertEquals(3, resultStatements.size()); + + Query q1 = resultStatements.get(0); + Query q2 = resultStatements.get(1); + Query q3 = resultStatements.get(2); + + assertEquals("CREATE TABLE test (id INTEGER);", q1.getLogString()); + assertEquals("INSERT INTO test VALUES (1);", q2.getLogString()); + assertEquals("SELECT * FROM test;", q3.getLogString()); + + ExpectedErrors e1 = q1.getExpectedErrors(); + ExpectedErrors e2 = q2.getExpectedErrors(); + ExpectedErrors e3 = q3.getExpectedErrors(); + + assertTrue(e1.errorIsExpected("syntax error")); + assertTrue(e1.errorIsExpected("table already exists")); + assertFalse(e1.errorIsExpected("constraint failed")); + + assertFalse(e2.errorIsExpected("syntax error")); + assertTrue(e2.errorIsExpected("constraint failed")); + + assertFalse(e3.errorIsExpected("syntax error")); + assertFalse(e3.errorIsExpected("constraint failed")); + } + + @Test + public void testDatabaseProvider() throws IOException { + SQLite3Provider provider = new SQLite3Provider(); + StateToReproduce state = new StateToReproduce("test_provider", provider); + state.logStatement("CREATE TABLE test (id INTEGER);"); + + Path file = tempDir.resolve("test_provider.ser"); + state.serialize(file); + StateToReproduce result = StateToReproduce.deserialize(file); + + // Verify databaseProvider is correctly deserialized + assertEquals("sqlite3", result.getDatabaseProvider().getDBMSName()); + + // Verify databaseProvider functionality by testing logStatement + result.logStatement("INSERT INTO test VALUES (1);"); + assertEquals(2, result.getStatements().size()); + assertEquals("INSERT INTO test VALUES (1);", result.getStatements().get(1).getLogString()); + } +} \ No newline at end of file From 150d2d53ca27074eea3de7c3ae312d83248056ed Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Sun, 24 Aug 2025 22:39:52 +0800 Subject: [PATCH 1142/1284] Fix StateToReproduce serialization --- src/sqlancer/Main.java | 1 + src/sqlancer/StateToReproduce.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 87023b0a2..1f2642f95 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -691,6 +691,7 @@ private boolean run(MainOptions options, ExecutorService execService, executor.getLogger().logFileWriter = null; executor.getLogger().logException(reduce, executor.getStateToReproduce()); if (options.serializeReproduceState()) { + executor.getStateToReproduce().logStatement(reduce.getMessage()); // add the error statement executor.getStateToReproduce().serialize(executor.getLogger().getReproduceFilePath()); } return false; diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index c849b535d..e44d0ccf6 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -28,7 +28,7 @@ public class StateToReproduce implements Serializable { String exception; - public OracleRunReproductionState localState; + public transient OracleRunReproductionState localState; public StateToReproduce(String databaseName, DatabaseProvider databaseProvider) { this.databaseName = databaseName; From f7310f96163a05730db51e9009ac174cfc4689e3 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Sun, 24 Aug 2025 22:49:21 +0800 Subject: [PATCH 1143/1284] Add a standalone reducer --- src/sqlancer/StandaloneReducer.java | 130 ++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 src/sqlancer/StandaloneReducer.java diff --git a/src/sqlancer/StandaloneReducer.java b/src/sqlancer/StandaloneReducer.java new file mode 100644 index 000000000..dfc0cc348 --- /dev/null +++ b/src/sqlancer/StandaloneReducer.java @@ -0,0 +1,130 @@ +package sqlancer; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.Query; + +/** + * A standalone tool to reduce bug-triggering SQL statements using the delta debugging algorithm. + */ +public class StandaloneReducer { + private int partitionNum = 2; + private final StateToReproduce originalState; + private final DatabaseProvider databaseProvider; + + public StandaloneReducer(Path serFilePath) throws Exception { + this.originalState = StateToReproduce.deserialize(serFilePath); + this.databaseProvider = originalState.getDatabaseProvider(); + if (this.databaseProvider == null) { + throw new IllegalStateException("Failed to get database provider from .ser file"); + } + } + + /** + * Performs the main reduction algorithm using partition-based delta debugging. + * + * @return List of reduced SQL statements that still trigger bugs. + */ + public List> reduce() throws Exception { + List> queries = new ArrayList<>(originalState.getStatements()); + if (queries.size() <= 1) { + return queries; + } + + partitionNum = 2; + while (queries.size() >= 2) { + boolean changedInThisPass = false; + List> result = tryReduction(queries); + + if (result.size() < queries.size()) { + queries = result; + changedInThisPass = true; + } + + if (changedInThisPass) { + partitionNum = 2; + } else { + if (partitionNum >= queries.size()) { + break; + } + partitionNum = Math.min(partitionNum * 2, queries.size()); + } + } + + System.out.println("Reduction completed successfully!"); + System.out.println("Final size: " + queries.size() + " statements (" + + String.format("%.1f", (1.0 - (double) queries.size() / originalState.getStatements().size()) * 100) + + "% reduction)"); + System.out.println("Final queries:"); + for (Query query : queries) { + System.out.println(query.getQueryString()); + } + + return queries; + } + + private List> tryReduction(List> queries) throws Exception { + int start = 0; + int subLength = queries.size() / partitionNum; + + while (start < queries.size()) { + List> candidateQueries = new ArrayList<>(queries); + int endPoint = Math.min(start + subLength, candidateQueries.size()); + candidateQueries.subList(start, endPoint).clear(); + + if (testExceptionStillExists(candidateQueries)) { + return candidateQueries; + } + + start += subLength; + } + + return queries; + } + + // Test if bug still exists with reduced query set + @SuppressWarnings("unchecked") + private , O extends DBMSSpecificOptions, C extends SQLancerDBConnection> boolean testExceptionStillExists( + List> queries) { + try { + DatabaseProvider typedProvider = (DatabaseProvider) databaseProvider; + G globalState = typedProvider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + + try (C connection = typedProvider.createDatabase(globalState)) { + globalState.setConnection(connection); + for (Query query : queries) { + try { + Query typedQuery = (Query) query; + typedQuery.execute(globalState); + } catch (Throwable e) { + // Any exception is considered a success + return true; + } + } + // No exception occurred + return false; + } + } catch (Throwable e) { + return true; + } + } + + public static void main(String[] args) { + try { + if (args.length == 0) { + System.err.println( + "Usage: java -cp target/sqlancer-2.0.0.jar sqlancer.StandaloneReducer "); + System.exit(1); + } + StandaloneReducer reducer = new StandaloneReducer(Paths.get(args[0])); + reducer.reduce(); + } catch (Throwable e) { + System.err.println("ERROR: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } +} From 513d0d8a15fb356c85198cdce2370862fc3935af Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Mon, 25 Aug 2025 13:06:51 +0800 Subject: [PATCH 1144/1284] Write final queries to a file --- src/sqlancer/StandaloneReducer.java | 30 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/StandaloneReducer.java b/src/sqlancer/StandaloneReducer.java index dfc0cc348..813160060 100644 --- a/src/sqlancer/StandaloneReducer.java +++ b/src/sqlancer/StandaloneReducer.java @@ -1,5 +1,7 @@ package sqlancer; +import java.io.FileWriter; +import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -14,13 +16,16 @@ public class StandaloneReducer { private int partitionNum = 2; private final StateToReproduce originalState; private final DatabaseProvider databaseProvider; + private final Path outputPath; - public StandaloneReducer(Path serFilePath) throws Exception { - this.originalState = StateToReproduce.deserialize(serFilePath); + public StandaloneReducer(Path inputPath, Path outputPath) throws Exception { + this.originalState = StateToReproduce.deserialize(inputPath); this.databaseProvider = originalState.getDatabaseProvider(); if (this.databaseProvider == null) { throw new IllegalStateException("Failed to get database provider from .ser file"); } + this.outputPath = outputPath != null ? outputPath + : Paths.get(inputPath.toString().replaceAll("\\.ser$", ".sql")); } /** @@ -54,14 +59,15 @@ public List> reduce() throws Exception { } } - System.out.println("Reduction completed successfully!"); + try (PrintWriter writer = new PrintWriter(new FileWriter(outputPath.toFile()))) { + for (Query query : queries) { + writer.println(query.getQueryString()); + } + } + System.out.println("Reduction completed successfully! SQL statements written to: " + outputPath.toString()); System.out.println("Final size: " + queries.size() + " statements (" + String.format("%.1f", (1.0 - (double) queries.size() / originalState.getStatements().size()) * 100) + "% reduction)"); - System.out.println("Final queries:"); - for (Query query : queries) { - System.out.println(query.getQueryString()); - } return queries; } @@ -100,7 +106,8 @@ private List> tryReduction(List> queries) throws Exception { Query typedQuery = (Query) query; typedQuery.execute(globalState); } catch (Throwable e) { - // Any exception is considered a success + // Any exception not declared as an expected error by the query indicates that an (unexpected) + // exception still exists return true; } } @@ -116,10 +123,13 @@ public static void main(String[] args) { try { if (args.length == 0) { System.err.println( - "Usage: java -cp target/sqlancer-2.0.0.jar sqlancer.StandaloneReducer "); + "Usage: java -cp target/sqlancer-2.0.0.jar sqlancer.StandaloneReducer [output-file]"); System.exit(1); } - StandaloneReducer reducer = new StandaloneReducer(Paths.get(args[0])); + Path inputPath = Paths.get(args[0]); + Path outputPath = args.length > 1 ? Paths.get(args[1]) : null; + + StandaloneReducer reducer = new StandaloneReducer(inputPath, outputPath); reducer.reduce(); } catch (Throwable e) { System.err.println("ERROR: " + e.getMessage()); From 6a43fe08183ef9602f1c886733d5f1e24fb3d501 Mon Sep 17 00:00:00 2001 From: luliqwerty <3535768174@qq.com> Date: Wed, 27 Aug 2025 13:06:29 +0800 Subject: [PATCH 1145/1284] doc(README): Add DQE Oracle Testing Method in README --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f88b77faa..134f47666 100644 --- a/README.md +++ b/README.md @@ -75,15 +75,16 @@ Usage: SQLancer [options] [command] [command options] SQLancer has pioneered and includes multiple approaches for DBMS testing, as outlined below in chronological order. -| Technique | Venue | Links | Description | -|-----------------------------------------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Pivoted Query Synthesis (PQS) | OSDI 2020 | [Paper](https://www.usenix.org/system/files/osdi20-rigger.pdf) [Video](https://www.youtube.com/watch?v=0aeDyXgzo04 ) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described here. PQS effectively detects bugs, but requires more implementation effort than other testing approaches that follow a metamorphic testing or differential testing methodology. Thus, it is currently unmaintained. | -| Non-optimizing Reference Engine Construction (NoREC) | ESEC/FSE 2020 | [Paper](https://arxiv.org/abs/2007.08292) [Video](https://www.youtube.com/watch?v=4mbzytrWJhQ) | NoREC aims to find optimization bugs. It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. The approach applies primarily to simple queries with a filter predicate. | -| Ternary Logic Partitioning (TLP) | OOPSLA 2020 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3428279) [Video](https://www.youtube.com/watch?v=FN9OLbGh0VI) | TLP partitions a query into three partitioning queries, whose results are composed and compared to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. It is among the most widely adopted testing techniques. | -| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | QPG is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | -| Cardinality Estimation Restriction Testing (CERT) | ICSE 2024 | [Paper](https://arxiv.org/pdf/2306.00355) [Code](https://github.com/sqlancer/sqlancer/issues/822) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. From a given input query, it derives a more restrictive query, whose estimated cardinality should be no more than that of the original query. A violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. CERT is the only test oracle that is part of SQLancer that was designed to find performance issues. | -| Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | -| Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | +| Technique | Venue | Links | Description | +|-----------------------------------------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Pivoted Query Synthesis (PQS) | OSDI 2020 | [Paper](https://www.usenix.org/system/files/osdi20-rigger.pdf) [Video](https://www.youtube.com/watch?v=0aeDyXgzo04 ) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described here. PQS effectively detects bugs, but requires more implementation effort than other testing approaches that follow a metamorphic testing or differential testing methodology. Thus, it is currently unmaintained. | +| Non-optimizing Reference Engine Construction (NoREC) | ESEC/FSE 2020 | [Paper](https://arxiv.org/abs/2007.08292) [Video](https://www.youtube.com/watch?v=4mbzytrWJhQ) | NoREC aims to find optimization bugs. It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. The approach applies primarily to simple queries with a filter predicate. | +| Ternary Logic Partitioning (TLP) | OOPSLA 2020 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3428279) [Video](https://www.youtube.com/watch?v=FN9OLbGh0VI) | TLP partitions a query into three partitioning queries, whose results are composed and compared to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. It is among the most widely adopted testing techniques. | +| Differential Query Execution (DQE) | ICSE 2023 | [Paper](https://ieeexplore.ieee.org/document/10172736) [Code](https://github.com/sqlancer/sqlancer/pull/1251) | Differential Query Execution (DQE) is a novel and general approach to detect logic bugs in SELECT, UPDATE and DELETE queries. DQE solves the test oracle problem by executing SELECT, UPDATE and DELETE queries with the same predicate φ, and observing inconsistencies among their execution results. For example, if a row that is updated by an UPDATE query with a predicate φ does not appear in the query result of a SELECT query with the same predicate φ, a logic bug is detected in the target DBMS. We append two extra columns to each table in a database to uniquely identify each row and track whether a row has been modified. We further rewrite SELECT and UPDATE queries to identify their accessed rows. DQE supports MySQL. | +| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | QPG is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | +| Cardinality Estimation Restriction Testing (CERT) | ICSE 2024 | [Paper](https://arxiv.org/pdf/2306.00355) [Code](https://github.com/sqlancer/sqlancer/issues/822) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. From a given input query, it derives a more restrictive query, whose estimated cardinality should be no more than that of the original query. A violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. CERT is the only test oracle that is part of SQLancer that was designed to find performance issues. | +| Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | +| Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | Please find the `.bib` entries [here](docs/PAPERS.md). | From 8749a6ecb05f049deeacb8593de3fcd44bee92d0 Mon Sep 17 00:00:00 2001 From: luliqwerty <3535768174@qq.com> Date: Tue, 2 Sep 2025 08:29:04 +0800 Subject: [PATCH 1146/1284] feat(DQE): Add Extensibility to DQE --- src/sqlancer/common/oracle/DQEBase.java | 18 ++++++------- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 25 +++++++++++-------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/common/oracle/DQEBase.java b/src/sqlancer/common/oracle/DQEBase.java index ba9af3bd1..0d954b115 100644 --- a/src/sqlancer/common/oracle/DQEBase.java +++ b/src/sqlancer/common/oracle/DQEBase.java @@ -13,7 +13,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryError; import sqlancer.common.schema.AbstractRelationalTable; -import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.common.schema.AbstractTables; /* * In DBMSs, SELECT, UPDATE and DELETE queries utilize predicates (i.e., WHERE clauses) to specify which rows to retrieve, update or delete, respectively. @@ -54,19 +54,15 @@ public DQEBase(S state) { this.options = state.getOptions(); } - public abstract String generateSelectStatement(MySQLTables tables, String tableName, String whereClauseStr); + public abstract String generateSelectStatement(AbstractTables tables, String tableName, + String whereClauseStr); - public abstract String generateUpdateStatement(MySQLTables tables, String tableName, String whereClauseStr); + public abstract String generateUpdateStatement(AbstractTables tables, String tableName, + String whereClauseStr); - public abstract String generateDeleteStatement(MySQLTables tables, String tableName, String whereClauseStr); + public abstract String generateDeleteStatement(String tableName, String whereClauseStr); - /** - * Add auxiliary columns to the database A abstract method, subclasses need to implement it. - * - * @param table - * - * @throws SQLException - */ + // Add auxiliary columns to the database A abstract method, subclasses need to implement it. public abstract void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException; public void dropAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index 4585fc830..de64b0415 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -22,6 +22,7 @@ import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTables; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -57,16 +58,17 @@ public MySQLDQEOracle(MySQLGlobalState state) { } @Override - public String generateSelectStatement(MySQLTables tables, String tableName, String whereClauseStr) { + public String generateSelectStatement(AbstractTables tables, String tableName, String whereClauseStr) { operateOnSingleTable = tables.getTables().size() == 1; List selectColumns = new ArrayList<>(); - for (MySQLTable table : tables.getTables()) { + MySQLTables mySQLTables = (MySQLTables) tables; + for (MySQLTable table : mySQLTables.getTables()) { selectColumns.add(table.getName() + "." + COLUMN_ROWID); } if (operateOnSingleTable && Randomly.getBooleanWithSmallProbability()) { generateOrderBy = true; // generate order by columns - for (MySQLColumn column : Randomly.nonEmptySubset(tables.getColumns())) { + for (MySQLColumn column : Randomly.nonEmptySubset(mySQLTables.getColumns())) { orderColumns.add(column.getFullQualifiedName()); } @@ -88,9 +90,10 @@ public String generateSelectStatement(MySQLTables tables, String tableName, Stri } @Override - public String generateUpdateStatement(MySQLTables tables, String tableName, String whereClauseStr) { + public String generateUpdateStatement(AbstractTables tables, String tableName, String whereClauseStr) { List updateColumns = new ArrayList<>(); - for (MySQLTable table : tables.getTables()) { + MySQLTables mySQLTables = (MySQLTables) tables; + for (MySQLTable table : mySQLTables.getTables()) { updateColumns.add(String.format("%s = 1", table.getName() + "." + COLUMN_UPDATED)); } String updateStmt = String.format("UPDATE %s SET %s WHERE %s", tableName, Strings.join(",", updateColumns), @@ -105,7 +108,7 @@ public String generateUpdateStatement(MySQLTables tables, String tableName, Stri } @Override - public String generateDeleteStatement(MySQLTables tables, String tableName, String whereClauseStr) { + public String generateDeleteStatement(String tableName, String whereClauseStr) { String deleteStmt; if (operateOnSingleTable) { deleteStmt = String.format("DELETE FROM %s WHERE %s", tableName, whereClauseStr); @@ -141,7 +144,7 @@ public void check() throws SQLException { String updateStmt = generateUpdateStatement(tables, tableName, whereClauseStr); - String deleteStmt = generateDeleteStatement(tables, tableName, whereClauseStr); + String deleteStmt = generateDeleteStatement(tableName, whereClauseStr); for (MySQLTable table : tables.getTables()) { addAuxiliaryColumns(table); @@ -201,7 +204,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); - if (!isFound(selectErrors, updateError)) { + if (notFound(selectErrors, updateError)) { return "SELECT has different errors from UPDATE."; } } @@ -228,7 +231,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult * * @return is targetError found in selectQueryErrors */ - private static boolean isFound(List selectErrors, SQLQueryError targetError) { + private static boolean notFound(List selectErrors, SQLQueryError targetError) { boolean found = false; for (int i = 0; i < selectErrors.size(); i++) { SQLQueryError selectError = selectErrors.get(i); @@ -238,7 +241,7 @@ private static boolean isFound(List selectErrors, SQLQueryError t break; } } - return found; + return !found; } public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { @@ -263,7 +266,7 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); - if (!isFound(selectErrors, deleteError)) { + if (notFound(selectErrors, deleteError)) { return "SELECT has different errors from DELETE."; } } From d76843182e205d154afb04afbc355245d47c934c Mon Sep 17 00:00:00 2001 From: luliqwerty <3535768174@qq.com> Date: Tue, 2 Sep 2025 16:05:57 +0800 Subject: [PATCH 1147/1284] refactor: rename notFound method in MySQLDQEOracle.java --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index de64b0415..8ddb6f315 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -204,7 +204,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); - if (notFound(selectErrors, updateError)) { + if (!isFound(selectErrors, updateError)) { return "SELECT has different errors from UPDATE."; } } @@ -231,7 +231,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult * * @return is targetError found in selectQueryErrors */ - private static boolean notFound(List selectErrors, SQLQueryError targetError) { + private static boolean isFound(List selectErrors, SQLQueryError targetError) { boolean found = false; for (int i = 0; i < selectErrors.size(); i++) { SQLQueryError selectError = selectErrors.get(i); @@ -241,7 +241,7 @@ private static boolean notFound(List selectErrors, SQLQueryError break; } } - return !found; + return found; } public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { @@ -266,7 +266,7 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); - if (notFound(selectErrors, deleteError)) { + if (!isFound(selectErrors, deleteError)) { return "SELECT has different errors from DELETE."; } } From d04835346e1d02b3a7fdb513dc5f62554cfe1b37 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 15 Sep 2025 14:55:18 +0800 Subject: [PATCH 1148/1284] tidb: fix wrong type judgement and analyze index Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 8 ++++---- src/sqlancer/tidb/TiDBSchema.java | 3 +++ src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java | 11 +++++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b0e4e410..5c53192aa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -609,8 +609,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:v8.5.2 - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 + docker pull hawkingrei/tidb-playground:nightly-2025-09-16 + docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" @@ -636,8 +636,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:v8.5.2 - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 + docker pull hawkingrei/tidb-playground:nightly-2025-09-16 + docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 7c808beee..32e00504b 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -251,17 +251,20 @@ private static TiDBCompositeDataType getColumnType(String typeString) { primitiveType = TiDBDataType.INT; size = 1; break; + case "tinyint": case "tinyint(2)": case "tinyint(3)": case "tinyint(4)": primitiveType = TiDBDataType.INT; size = 1; break; + case "smallint": case "smallint(5)": case "smallint(6)": primitiveType = TiDBDataType.INT; size = 2; break; + case "int": case "int(10)": case "int(11)": primitiveType = TiDBDataType.INT; diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 8b43c635a..ede857dc5 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -1,10 +1,12 @@ package sqlancer.tidb.gen; import java.sql.SQLException; +import java.util.List; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.TableIndex; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -17,12 +19,17 @@ private TiDBAnalyzeTableGenerator() { public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); + List indexes = table.getIndexes(); + indexes.removeIf(index -> index.getIndexName().contains("PRIMARY")); + boolean analyzeIndex = !indexes.isEmpty() && Randomly.getBoolean(); StringBuilder sb = new StringBuilder("ANALYZE TABLE "); sb.append(table.getName()); if (analyzeIndex) { sb.append(" INDEX "); - sb.append(table.getRandomIndex().getIndexName()); + sb.append(Randomly.fromList(indexes).getIndexName()); + } + if (!analyzeIndex && Randomly.getBoolean()) { + sb.append(" ALL COLUMNS"); } if (Randomly.getBoolean()) { sb.append(" WITH "); From f60bfedded7ccd53c8e0b5679dfc6680d0c0ec56 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 18 Sep 2025 10:51:56 +0800 Subject: [PATCH 1149/1284] tidb: skip Data too long errors Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBErrors.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 4f45debce..528acd0bb 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -108,6 +108,9 @@ public static List getInsertErrors() { errors.add("is not valid for CHARACTER SET"); errors.add("for function inet_aton"); errors.add("'Empty pattern is invalid' from regexp"); + errors.add("Data too long for expression index"); + errors.add("Data too long for column"); + errors.add("Data Too Long"); return errors; } From c58cc468cca9a89de59add149aa7af845fbebfcc Mon Sep 17 00:00:00 2001 From: Mohab-Sobhy Date: Thu, 18 Sep 2025 23:25:09 +0300 Subject: [PATCH 1150/1284] Add DeleteGenerator for MariaDB --- src/sqlancer/mariadb/MariaDBProvider.java | 15 +-- .../mariadb/gen/MariaDBDeleteGenerator.java | 94 +++++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 48254b8ec..8f7eb78ad 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -19,13 +19,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; -import sqlancer.mariadb.gen.MariaDBIndexGenerator; -import sqlancer.mariadb.gen.MariaDBInsertGenerator; -import sqlancer.mariadb.gen.MariaDBSetGenerator; -import sqlancer.mariadb.gen.MariaDBTableAdminCommandGenerator; -import sqlancer.mariadb.gen.MariaDBTableGenerator; -import sqlancer.mariadb.gen.MariaDBTruncateGenerator; -import sqlancer.mariadb.gen.MariaDBUpdateGenerator; +import sqlancer.mariadb.gen.*; @AutoService(DatabaseProvider.class) public class MariaDBProvider extends SQLProviderAdapter { @@ -47,6 +41,7 @@ enum Action { SET, // TRUNCATE, // UPDATE, // + DELETE, } @Override @@ -77,6 +72,9 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { case CREATE_INDEX: nrPerformed = globalState.getRandomly().getInteger(0, 2); break; + case DELETE: + nrPerformed = 10; + break; case SET: nrPerformed = 20; break; @@ -140,6 +138,9 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { case SET: query = MariaDBSetGenerator.set(globalState.getRandomly(), options); break; + case DELETE: + query = MariaDBDeleteGenerator.delete(globalState.getSchema(), globalState.getRandomly()); + break; default: throw new AssertionError(nextAction); } diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java new file mode 100644 index 000000000..d1f3d2600 --- /dev/null +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -0,0 +1,94 @@ +package sqlancer.mariadb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTables; +import sqlancer.mariadb.MariaDBSchema; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.ast.MariaDBVisitor; + +import java.util.Collections; + +public final class MariaDBDeleteGenerator { + + private MariaDBDeleteGenerator() { + } + + public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { + MariaDBTable table = schema.getRandomTable(); + + MariaDBExpressionGenerator expressionGenerator = new MariaDBExpressionGenerator(r); + + AbstractTables tablesAndColumns = + new AbstractTables<>(Collections.singletonList(table)); + expressionGenerator.setTablesAndColumns(tablesAndColumns); + + ExpectedErrors errors = new ExpectedErrors(); + + errors.add("foreign key constraint fails"); + errors.add("cannot delete or update a parent row"); + errors.add("Data truncated"); + errors.add("Division by 0"); + errors.add("Incorrect value"); + + StringBuilder sb = new StringBuilder("DELETE"); + + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" LOW_PRIORITY"); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" QUICK"); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" IGNORE"); + } + + sb.append(" FROM "); + sb.append(table.getName()); + + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(MariaDBVisitor.asString( + MariaDBExpressionGenerator.getRandomConstant(r) + )); + } else { + sb.append(MariaDBVisitor.asString( + expressionGenerator.getRandomExpression() + )); + } + } + + // ORDER BY + LIMIT + if (Randomly.getBooleanWithRatherLowProbability() && !table.getColumns().isEmpty()) { + sb.append(" ORDER BY "); + sb.append(Randomly.fromList(table.getColumns()).getName()); + if (Randomly.getBoolean()) { + sb.append(Randomly.getBoolean() ? " ASC" : " DESC"); + } + } + + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" LIMIT "); + sb.append(Randomly.getNotCachedInteger(1, 10)); + } + + // RETURNING clause (MariaDB >= 10.5) + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" RETURNING "); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(MariaDBVisitor.asString( + MariaDBExpressionGenerator.getRandomConstant(r) + )); + } else { + sb.append(MariaDBVisitor.asString( + expressionGenerator.getRandomExpression() + )); + } + } + + return new SQLQueryAdapter(sb.toString(), errors); + } +} From fde4da74ad9222ac537a6e9ad3f0bfd55daa95a8 Mon Sep 17 00:00:00 2001 From: Mohab-Sobhy Date: Wed, 8 Oct 2025 10:59:55 +0300 Subject: [PATCH 1151/1284] Format code to fix linter error --- .../mariadb/gen/MariaDBDeleteGenerator.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index d1f3d2600..28136eb08 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -21,8 +21,8 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { MariaDBExpressionGenerator expressionGenerator = new MariaDBExpressionGenerator(r); - AbstractTables tablesAndColumns = - new AbstractTables<>(Collections.singletonList(table)); + AbstractTables tablesAndColumns = new AbstractTables<>( + Collections.singletonList(table)); expressionGenerator.setTablesAndColumns(tablesAndColumns); ExpectedErrors errors = new ExpectedErrors(); @@ -51,13 +51,9 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { if (Randomly.getBoolean()) { sb.append(" WHERE "); if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString( - MariaDBExpressionGenerator.getRandomConstant(r) - )); + sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); } else { - sb.append(MariaDBVisitor.asString( - expressionGenerator.getRandomExpression() - )); + sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); } } @@ -79,13 +75,9 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" RETURNING "); if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString( - MariaDBExpressionGenerator.getRandomConstant(r) - )); + sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); } else { - sb.append(MariaDBVisitor.asString( - expressionGenerator.getRandomExpression() - )); + sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); } } From 18ab56017c46be4ea0309b57ce990f176ac72dbe Mon Sep 17 00:00:00 2001 From: Mohab-Sobhy Date: Mon, 3 Nov 2025 01:11:36 +0200 Subject: [PATCH 1152/1284] Fix Checkstyle violations and add expected errors for invalid regex in MariaDBDeleteGenerator --- src/sqlancer/mariadb/MariaDBProvider.java | 9 ++++++++- .../mariadb/gen/MariaDBDeleteGenerator.java | 15 +++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 8f7eb78ad..c8037c019 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -19,7 +19,14 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; -import sqlancer.mariadb.gen.*; +import sqlancer.mariadb.gen.MariaDBDeleteGenerator; +import sqlancer.mariadb.gen.MariaDBIndexGenerator; +import sqlancer.mariadb.gen.MariaDBInsertGenerator; +import sqlancer.mariadb.gen.MariaDBSetGenerator; +import sqlancer.mariadb.gen.MariaDBTableAdminCommandGenerator; +import sqlancer.mariadb.gen.MariaDBTableGenerator; +import sqlancer.mariadb.gen.MariaDBTruncateGenerator; +import sqlancer.mariadb.gen.MariaDBUpdateGenerator; @AutoService(DatabaseProvider.class) public class MariaDBProvider extends SQLProviderAdapter { diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index 28136eb08..6d85eb891 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -1,16 +1,16 @@ package sqlancer.mariadb.gen; +import java.util.Collections; + import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTables; import sqlancer.mariadb.MariaDBSchema; -import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.ast.MariaDBVisitor; -import java.util.Collections; - public final class MariaDBDeleteGenerator { private MariaDBDeleteGenerator() { @@ -81,6 +81,13 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { } } - return new SQLQueryAdapter(sb.toString(), errors); + String query = sb.toString(); + if (query.contains("RLIKE") || query.contains("REGEXP")) { + errors.add("Regex error"); + errors.add("quantifier does not follow a repeatable item"); + errors.add("Got error"); + } + + return new SQLQueryAdapter(query, errors); } } From 35105ec31d873b0309e86b6563a7f02ff00e409e Mon Sep 17 00:00:00 2001 From: Mohab-Sobhy Date: Thu, 20 Nov 2025 08:43:50 +0200 Subject: [PATCH 1153/1284] Modify DELETE generation probability --- src/sqlancer/mariadb/MariaDBProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index c8037c019..a9737f549 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -80,7 +80,7 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case DELETE: - nrPerformed = 10; + nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case SET: nrPerformed = 20; From d84210022ea9d80f8060e8d05b38d68555485b94 Mon Sep 17 00:00:00 2001 From: Pratyksh Gupta Date: Sat, 6 Dec 2025 12:38:59 +0530 Subject: [PATCH 1154/1284] Fix #1249: Correct REINDEX INDEX syntax to use single index instead of concatenating all --- src/sqlancer/postgres/gen/PostgresReindexGenerator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index 9bb5ec5cd..dc0d2cf34 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -1,7 +1,6 @@ package sqlancer.postgres.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -39,7 +38,7 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { if (indexes.isEmpty()) { throw new IgnoreMeException(); } - sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining())); + sb.append(Randomly.fromList(indexes).getIndexName()); break; case TABLE: sb.append("TABLE "); From 05bb28ce7a6f8e1152293fc8d080a94f9e4dc480 Mon Sep 17 00:00:00 2001 From: Aman Date: Sun, 7 Dec 2025 12:33:03 +0530 Subject: [PATCH 1155/1284] Add Spark support for TLP Oracle --- .github/workflows/main.yml | 38 ++ pom.xml | 9 +- src/sqlancer/Main.java | 2 + src/sqlancer/spark/SparkErrors.java | 67 ++++ src/sqlancer/spark/SparkGlobalState.java | 11 + src/sqlancer/spark/SparkOptions.java | 43 +++ src/sqlancer/spark/SparkProvider.java | 122 +++++++ src/sqlancer/spark/SparkSchema.java | 114 ++++++ src/sqlancer/spark/SparkToStringVisitor.java | 120 +++++++ .../spark/ast/SparkBetweenOperation.java | 10 + .../spark/ast/SparkBinaryOperation.java | 11 + .../spark/ast/SparkCaseOperation.java | 13 + .../spark/ast/SparkCastOperation.java | 25 ++ .../spark/ast/SparkColumnReference.java | 11 + src/sqlancer/spark/ast/SparkConstant.java | 194 ++++++++++ src/sqlancer/spark/ast/SparkExpression.java | 7 + src/sqlancer/spark/ast/SparkFunction.java | 13 + src/sqlancer/spark/ast/SparkInOperation.java | 12 + src/sqlancer/spark/ast/SparkJoin.java | 46 +++ src/sqlancer/spark/ast/SparkOrderingTerm.java | 10 + src/sqlancer/spark/ast/SparkSelect.java | 42 +++ .../spark/ast/SparkTableReference.java | 13 + .../spark/ast/SparkUnaryPostfixOperation.java | 13 + .../spark/ast/SparkUnaryPrefixOperation.java | 12 + .../spark/gen/SparkExpressionGenerator.java | 336 ++++++++++++++++++ .../spark/gen/SparkInsertGenerator.java | 47 +++ .../spark/gen/SparkTableGenerator.java | 100 ++++++ test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestSparkTLP.java | 20 ++ 29 files changed, 1460 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/spark/SparkErrors.java create mode 100644 src/sqlancer/spark/SparkGlobalState.java create mode 100644 src/sqlancer/spark/SparkOptions.java create mode 100644 src/sqlancer/spark/SparkProvider.java create mode 100644 src/sqlancer/spark/SparkSchema.java create mode 100644 src/sqlancer/spark/SparkToStringVisitor.java create mode 100644 src/sqlancer/spark/ast/SparkBetweenOperation.java create mode 100644 src/sqlancer/spark/ast/SparkBinaryOperation.java create mode 100644 src/sqlancer/spark/ast/SparkCaseOperation.java create mode 100644 src/sqlancer/spark/ast/SparkCastOperation.java create mode 100644 src/sqlancer/spark/ast/SparkColumnReference.java create mode 100644 src/sqlancer/spark/ast/SparkConstant.java create mode 100644 src/sqlancer/spark/ast/SparkExpression.java create mode 100644 src/sqlancer/spark/ast/SparkFunction.java create mode 100644 src/sqlancer/spark/ast/SparkInOperation.java create mode 100644 src/sqlancer/spark/ast/SparkJoin.java create mode 100644 src/sqlancer/spark/ast/SparkOrderingTerm.java create mode 100644 src/sqlancer/spark/ast/SparkSelect.java create mode 100644 src/sqlancer/spark/ast/SparkTableReference.java create mode 100644 src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java create mode 100644 src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java create mode 100644 src/sqlancer/spark/gen/SparkExpressionGenerator.java create mode 100644 src/sqlancer/spark/gen/SparkInsertGenerator.java create mode 100644 src/sqlancer/spark/gen/SparkTableGenerator.java create mode 100644 test/sqlancer/dbms/TestSparkTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c53192aa..41bd92d3c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -333,6 +333,44 @@ jobs: - name: Run Tests run: HIVE_AVAILABLE=true mvn -Dtest=TestHiveTLP test + spark: + name: DBMS Tests (Spark) + runs-on: ubuntu-latest + + services: + spark: + image: apache/spark:3.5.1 + ports: + - 10000:10000 + + command: >- + /opt/spark/bin/spark-submit + --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 + --name "Thrift JDBC/ODBC Server" + --master local[*] + --driver-memory 4g + --conf spark.hive.server2.thrift.port=10000 + --conf spark.sql.warehouse.dir=/tmp/spark-warehouse + spark-internal + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + + - name: Build SQLancer + run: mvn -B package -DskipTests=true + + - name: Run Tests + run: SPARK_AVAILABLE=true mvn -Dtest=TestSparkTLP test + hsqldb: name: DBMS Tests (HSQLB) runs-on: ubuntu-latest diff --git a/pom.xml b/pom.xml index 2037b71ce..c7a38c9aa 100644 --- a/pom.xml +++ b/pom.xml @@ -329,7 +329,7 @@ org.slf4j - slf4j-simple + slf4j-simple 2.0.6 @@ -381,7 +381,7 @@ org.apache.hive hive-jdbc - 4.0.1 + 3.1.2 org.apache.hive @@ -393,6 +393,11 @@ hive-cli 4.0.1 + + org.apache.hadoop + hadoop-common + 3.2.4 + diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 1f2642f95..f778bd7da 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -48,6 +48,7 @@ import sqlancer.tidb.TiDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.spark.SparkProvider; public final class Main { @@ -756,6 +757,7 @@ private static void checkForIssue799(List> providers) providers.add(new DuckDBProvider()); providers.add(new H2Provider()); providers.add(new HiveProvider()); + providers.add(new SparkProvider()); providers.add(new HSQLDBProvider()); providers.add(new MariaDBProvider()); providers.add(new MaterializeProvider()); diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java new file mode 100644 index 000000000..97c8056a3 --- /dev/null +++ b/src/sqlancer/spark/SparkErrors.java @@ -0,0 +1,67 @@ +package sqlancer.spark; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class SparkErrors { + + private SparkErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("cannot resolve"); + errors.add("AnalysisException"); + errors.add("data type mismatch"); + errors.add("undefined function"); + errors.add("mismatched input"); + errors.add("due to data type mismatch"); + + // --- Invalid Literals + errors.add("The value of the typed literal"); + + errors.add("DATATYPE_MISMATCH"); + errors.add("cannot be cast to"); + + errors.add("Overflow"); + errors.add("Divide by zero"); // Common if spark.sql.ansi.enabled is true + errors.add("division by zero"); + + // --- Group By / Aggregation errors --- + errors.add("grouping expressions"); + errors.add("expression is neither present in the group by"); + errors.add("is not a valid grouping expression"); + errors.add("is not contained in either an aggregate function or the GROUP BY clause"); + errors.add("PARSE_SYNTAX_ERROR"); + errors.add("Syntax error"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("not enough data columns"); + errors.add("cannot write to"); + errors.add("incompatible types"); + errors.add("too many data columns"); + errors.add("cannot be cast to"); + errors.add("Error running query"); + errors.add("The value of the typed literal"); + errors.add("Cannot safely cast"); // Found in logs: Decimal -> Date + errors.add("AnalysisException"); // Spark throws this for almost all insert failures + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkGlobalState.java b/src/sqlancer/spark/SparkGlobalState.java new file mode 100644 index 000000000..e79826332 --- /dev/null +++ b/src/sqlancer/spark/SparkGlobalState.java @@ -0,0 +1,11 @@ +package sqlancer.spark; + +import sqlancer.SQLGlobalState; + +public class SparkGlobalState extends SQLGlobalState { + + @Override + protected SparkSchema readSchema() throws Exception { + return SparkSchema.fromConnection(getConnection(), getDatabaseName()); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkOptions.java b/src/sqlancer/spark/SparkOptions.java new file mode 100644 index 000000000..c9422a910 --- /dev/null +++ b/src/sqlancer/spark/SparkOptions.java @@ -0,0 +1,43 @@ +package sqlancer.spark; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.spark.gen.SparkExpressionGenerator; + +@Parameters(separators = "=", commandDescription = "Spark SQL (default port: " + SparkOptions.DEFAULT_PORT + + ", default host: " + SparkOptions.DEFAULT_HOST + ")") +public class SparkOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 10000; + + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(SparkOracleFactory.TLPWhere); + + public enum SparkOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(SparkGlobalState globalState) throws SQLException { + SparkExpressionGenerator gen = new SparkExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SparkErrors.getExpressionErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }; + } + + @Override + public List getTestOracleFactory() { + return oracle; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java new file mode 100644 index 000000000..f53ca10a8 --- /dev/null +++ b/src/sqlancer/spark/SparkProvider.java @@ -0,0 +1,122 @@ +package sqlancer.spark; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.spark.gen.SparkInsertGenerator; +import sqlancer.spark.gen.SparkTableGenerator; + +@AutoService(DatabaseProvider.class) +public class SparkProvider extends SQLProviderAdapter { + + public SparkProvider() { + super(SparkGlobalState.class, SparkOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(SparkInsertGenerator::getQuery); // You will need to create this class + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(SparkGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(SparkGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + default: + throw new AssertionError(a); + } + } + + @Override + public void generateDatabase(SparkGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + String tableName = globalState.getSchema().getFreeTableName(); + SQLQueryAdapter qt = SparkTableGenerator.generate(globalState, tableName); + success = globalState.executeStatement(qt); + } while (!success); + } + + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + SparkProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + + if (host == null) { + host = SparkOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = SparkOptions.DEFAULT_PORT; + } + + String databaseName = globalState.getDatabaseName(); + + // Spark uses the Hive driver for JDBC usually + String url = String.format("jdbc:hive2://%s:%d/%s", host, port, "default"); + + // Connect to default to create the fuzzing DB + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + con.close(); + + // Connect to the specific fuzzing DB + con = DriverManager.getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName), username, + password); + try (Statement s = con.createStatement()) { + // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the generator loves to do. + s.execute("SET spark.sql.ansi.enabled=false"); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "spark"; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java new file mode 100644 index 000000000..849652b19 --- /dev/null +++ b/src/sqlancer/spark/SparkSchema.java @@ -0,0 +1,114 @@ +package sqlancer.spark; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.spark.SparkSchema.SparkTable; + +public class SparkSchema extends AbstractSchema { + + public enum SparkDataType { + STRING, INTEGER, DOUBLE, BOOLEAN, TIMESTAMP, DATE; + + public static SparkDataType getRandomType() { + return Randomly.fromList(Arrays.asList(values())); + } + } + + public static class SparkColumn extends AbstractTableColumn { + public SparkColumn(String name, SparkTable table, SparkDataType type) { + super(name, table, type); + } + } + + public static class SparkTables extends AbstractTables { + public SparkTables(List tables) { + super(tables); + } + } + + public static class SparkTable extends AbstractRelationalTable { + public SparkTable(String name, List columns, boolean isView) { + super(name, columns, Collections.emptyList(), isView); + } + } + + public SparkSchema(List databaseTables) { + super(databaseTables); + } + + public static SparkSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.toLowerCase().startsWith("v"); + SparkTable t = new SparkTable(tableName, databaseColumns, isView); + for (SparkColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new SparkSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet tableRs = s.executeQuery("SHOW TABLES"); + while (tableRs.next()) { + // Spark SHOW TABLES output: database, tableName, isTemporary + String tableName = tableRs.getString("tableName"); + tableNames.add(tableName); + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("DESCRIBE %s", tableName))) { + while (rs.next()) { + String columnName = rs.getString("col_name"); + String dataType = rs.getString("data_type"); + // Filter out Spark partition info or comments usually at bottom of describe + if (columnName.startsWith("#") || columnName.isEmpty()) + continue; + + columns.add(new SparkColumn(columnName, null, getColumnType(dataType))); + } + } + } + return columns; + } + + private static SparkDataType getColumnType(String typeString) { + String upper = typeString.toUpperCase(); + if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) + return SparkDataType.STRING; + if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) + return SparkDataType.INTEGER; + if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) + return SparkDataType.DOUBLE; + if (upper.startsWith("BOOLEAN")) + return SparkDataType.BOOLEAN; + if (upper.startsWith("TIMESTAMP")) + return SparkDataType.TIMESTAMP; + if (upper.startsWith("DATE")) + return SparkDataType.DATE; + return SparkDataType.STRING; // Fallback + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkToStringVisitor.java b/src/sqlancer/spark/SparkToStringVisitor.java new file mode 100644 index 000000000..91f47e32c --- /dev/null +++ b/src/sqlancer/spark/SparkToStringVisitor.java @@ -0,0 +1,120 @@ +package sqlancer.spark; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.spark.ast.SparkCastOperation; +import sqlancer.spark.ast.SparkConstant; +import sqlancer.spark.ast.SparkExpression; +import sqlancer.spark.ast.SparkJoin; +import sqlancer.spark.ast.SparkSelect; + +public class SparkToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(SparkExpression expr) { + if (expr instanceof SparkConstant) { + visit((SparkConstant) expr); + } else if (expr instanceof SparkSelect) { + visit((SparkSelect) expr); + } else if (expr instanceof SparkJoin) { + visit((SparkJoin) expr); + } else if (expr instanceof SparkCastOperation) { + visit((SparkCastOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(SparkConstant constant) { + sb.append(constant.toString()); + } + + private void visit(SparkSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + // Spark supports OFFSET, though strictly usually with LIMIT or in newer versions + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + private void visit(SparkJoin join) { + switch (join.getJoinType()) { + case INNER: + sb.append(" INNER JOIN "); + break; + case LEFT_OUTER: + sb.append(" LEFT JOIN "); + break; + case RIGHT_OUTER: + sb.append(" RIGHT JOIN "); + break; + case FULL_OUTER: + sb.append(" FULL JOIN "); + break; + case LEFT_SEMI: + sb.append(" LEFT SEMI JOIN "); + break; + // Spark also supports LEFT ANTI, which Hive might lack in some older versions + case LEFT_ANTI: + sb.append(" LEFT ANTI JOIN "); + break; + case CROSS: + sb.append(" CROSS JOIN "); + break; + default: + throw new UnsupportedOperationException("Join type not supported in Spark visitor: " + join.getJoinType()); + } + visit((TableReferenceNode) join.getRightTable()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + private void visit(SparkCastOperation cast) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + sb.append(cast.getType()); + sb.append(")"); + } + + public static String asString(SparkExpression expr) { + SparkToStringVisitor visitor = new SparkToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkBetweenOperation.java b/src/sqlancer/spark/ast/SparkBetweenOperation.java new file mode 100644 index 000000000..f229c1c7c --- /dev/null +++ b/src/sqlancer/spark/ast/SparkBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class SparkBetweenOperation extends NewBetweenOperatorNode implements SparkExpression { + + public SparkBetweenOperation(SparkExpression left, SparkExpression middle, SparkExpression right, boolean isTrue) { + super(left, middle, right, isTrue); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkBinaryOperation.java b/src/sqlancer/spark/ast/SparkBinaryOperation.java new file mode 100644 index 000000000..04af0ec4c --- /dev/null +++ b/src/sqlancer/spark/ast/SparkBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class SparkBinaryOperation extends NewBinaryOperatorNode implements SparkExpression { + + public SparkBinaryOperation(SparkExpression left, SparkExpression right, Operator op) { + super(left, right, op); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkCaseOperation.java b/src/sqlancer/spark/ast/SparkCaseOperation.java new file mode 100644 index 000000000..fb1ee0cd8 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkCaseOperation.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class SparkCaseOperation extends NewCaseOperatorNode implements SparkExpression { + + public SparkCaseOperation(SparkExpression switchCondition, List conditions, + List expressions, SparkExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkCastOperation.java b/src/sqlancer/spark/ast/SparkCastOperation.java new file mode 100644 index 000000000..3bc5eb30d --- /dev/null +++ b/src/sqlancer/spark/ast/SparkCastOperation.java @@ -0,0 +1,25 @@ +package sqlancer.spark.ast; + +import sqlancer.spark.SparkSchema.SparkDataType; + +public class SparkCastOperation implements SparkExpression { + + private final SparkExpression expression; + private final SparkDataType type; + + public SparkCastOperation(SparkExpression expression, SparkDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + public SparkExpression getExpression() { + return expression; + } + + public SparkDataType getType() { + return type; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkColumnReference.java b/src/sqlancer/spark/ast/SparkColumnReference.java new file mode 100644 index 000000000..75e92d267 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.spark.SparkSchema.SparkColumn; + +public class SparkColumnReference extends ColumnReferenceNode implements SparkExpression { + + public SparkColumnReference(SparkColumn column) { + super(column); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkConstant.java b/src/sqlancer/spark/ast/SparkConstant.java new file mode 100644 index 000000000..9f73af59f --- /dev/null +++ b/src/sqlancer/spark/ast/SparkConstant.java @@ -0,0 +1,194 @@ +package sqlancer.spark.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public abstract class SparkConstant implements SparkExpression { + + public boolean isNull() { + return false; + } + + public static class SparkNullConstant extends SparkConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public String toString() { + return "NULL"; + } + } + + public static class SparkIntConstant extends SparkConstant { + + private final long value; + + public SparkIntConstant(long value) { + this.value = value; + } + + public long getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class SparkDoubleConstant extends SparkConstant { + + private final double value; + + public SparkDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "CAST('Infinity' AS DOUBLE)"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "CAST('-Infinity' AS DOUBLE)"; + } else if (Double.isNaN(value)) { + return "CAST('NaN' AS DOUBLE)"; + } + return String.valueOf(value); + } + } + + public static class SparkDecimalConstant extends SparkConstant { + + private final BigDecimal value; + + public SparkDecimalConstant(BigDecimal value) { + this.value = value; + } + + public BigDecimal getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class SparkTimestampConstant extends SparkConstant { + + private final String textRepr; + + public SparkTimestampConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); // Spark prefers full timestamp + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + } + + public static class SparkDateConstant extends SparkConstant { + + private final String textRepr; + + public SparkDateConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + } + + public static class SparkStringConstant extends SparkConstant { + + private final String value; + + public SparkStringConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''").replace("\\", "\\\\") + "'"; + } + } + + public static class SparkBooleanConstant extends SparkConstant { + + private final boolean value; + + public SparkBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static SparkConstant createNullConstant() { + return new SparkNullConstant(); + } + + public static SparkConstant createIntConstant(long value) { + return new SparkIntConstant(value); + } + + public static SparkConstant createDoubleConstant(double value) { + return new SparkDoubleConstant(value); + } + + public static SparkConstant createDecimalConstant(BigDecimal value) { + return new SparkDecimalConstant(value); + } + + public static SparkConstant createTimestampConstant(long value) { + return new SparkTimestampConstant(value); + } + + public static SparkConstant createDateConstant(long value) { + return new SparkDateConstant(value); + } + + public static SparkConstant createStringConstant(String value) { + return new SparkStringConstant(value); + } + + public static SparkConstant createBooleanConstant(boolean value) { + return new SparkBooleanConstant(value); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkExpression.java b/src/sqlancer/spark/ast/SparkExpression.java new file mode 100644 index 000000000..a130096e3 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkExpression.java @@ -0,0 +1,7 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.spark.SparkSchema.SparkColumn; + +public interface SparkExpression extends Expression { +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkFunction.java b/src/sqlancer/spark/ast/SparkFunction.java new file mode 100644 index 000000000..d5740ee36 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkFunction.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class SparkFunction extends NewFunctionNode implements SparkExpression { + + public SparkFunction(List args, F func) { + super(args, func); + } + +} diff --git a/src/sqlancer/spark/ast/SparkInOperation.java b/src/sqlancer/spark/ast/SparkInOperation.java new file mode 100644 index 000000000..37a80e3ff --- /dev/null +++ b/src/sqlancer/spark/ast/SparkInOperation.java @@ -0,0 +1,12 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class SparkInOperation extends NewInOperatorNode implements SparkExpression { + + public SparkInOperation(SparkExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkJoin.java b/src/sqlancer/spark/ast/SparkJoin.java new file mode 100644 index 000000000..44da7fba4 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkJoin.java @@ -0,0 +1,46 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.Join; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; + +public class SparkJoin implements SparkExpression, Join { + + private final SparkTableReference leftTable; + private final SparkTableReference rightTable; + private final JoinType joinType; + private SparkExpression onClause; + + public enum JoinType { + INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, LEFT_ANTI, CROSS; + } + + public SparkJoin(SparkTableReference leftTable, SparkTableReference rightTable, JoinType joinType, + SparkExpression onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onClause = onClause; + } + + public SparkTableReference getLeftTable() { + return leftTable; + } + + public SparkTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public SparkExpression getOnClause() { + return onClause; + } + + @Override + public void setOnClause(SparkExpression onClause) { + this.onClause = onClause; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkOrderingTerm.java b/src/sqlancer/spark/ast/SparkOrderingTerm.java new file mode 100644 index 000000000..824801c00 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkOrderingTerm.java @@ -0,0 +1,10 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class SparkOrderingTerm extends NewOrderingTerm implements SparkExpression { + + public SparkOrderingTerm(SparkExpression expr, Ordering ordering) { + super(expr, ordering); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkSelect.java b/src/sqlancer/spark/ast/SparkSelect.java new file mode 100644 index 000000000..0986ce0a6 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkSelect.java @@ -0,0 +1,42 @@ +package sqlancer.spark.ast; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkSelect extends SelectBase + implements Select, SparkExpression { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (SparkExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (SparkJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return SparkToStringVisitor.asString(this); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkTableReference.java b/src/sqlancer/spark/ast/SparkTableReference.java new file mode 100644 index 000000000..92a59ad3d --- /dev/null +++ b/src/sqlancer/spark/ast/SparkTableReference.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.spark.SparkSchema; + +public class SparkTableReference extends TableReferenceNode + implements SparkExpression { + + public SparkTableReference(SparkSchema.SparkTable table) { + super(table); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java new file mode 100644 index 000000000..f1082a655 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class SparkUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements SparkExpression { + + public SparkUnaryPostfixOperation(SparkExpression expr, Operator op) { + super(expr, op); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java new file mode 100644 index 000000000..d1bd94ab4 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class SparkUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements SparkExpression { + + public SparkUnaryPrefixOperation(SparkExpression expr, Operator op) { + super(expr, op); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/gen/SparkExpressionGenerator.java b/src/sqlancer/spark/gen/SparkExpressionGenerator.java new file mode 100644 index 000000000..faf8a07f0 --- /dev/null +++ b/src/sqlancer/spark/gen/SparkExpressionGenerator.java @@ -0,0 +1,336 @@ +package sqlancer.spark.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkDataType; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.ast.SparkBetweenOperation; +import sqlancer.spark.ast.SparkBinaryOperation; +import sqlancer.spark.ast.SparkCaseOperation; +import sqlancer.spark.ast.SparkCastOperation; +import sqlancer.spark.ast.SparkColumnReference; +import sqlancer.spark.ast.SparkConstant; +import sqlancer.spark.ast.SparkExpression; +import sqlancer.spark.ast.SparkFunction; +import sqlancer.spark.ast.SparkInOperation; +import sqlancer.spark.ast.SparkJoin; +import sqlancer.spark.ast.SparkOrderingTerm; +import sqlancer.spark.ast.SparkSelect; +import sqlancer.spark.ast.SparkTableReference; +import sqlancer.spark.ast.SparkUnaryPostfixOperation; +import sqlancer.spark.ast.SparkUnaryPrefixOperation; + +public class SparkExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { + + private final SparkGlobalState globalState; + private List tables; + + private enum Expression { + UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, CAST, FUNC, BETWEEN, IN, + CASE; + } + + public SparkExpressionGenerator(SparkGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public SparkExpression negatePredicate(SparkExpression predicate) { + return new SparkUnaryPrefixOperation(predicate, SparkUnaryPrefixOperator.NOT); + } + + @Override + public SparkExpression isNull(SparkExpression expr) { + return new SparkUnaryPostfixOperation(expr, SparkUnaryPostfixOperator.IS_NULL); + } + + @Override + protected SparkExpression generateExpression(int depth) { + return generateExpressionInternal(depth); + } + + private SparkExpression generateExpressionInternal(int depth) throws AssertionError { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBooleanWithRatherLowProbability()) { + allowAggregates = false; // aggregate function calls cannot be nested + SparkAggregateFunction aggregate = SparkAggregateFunction.getRandom(); + return new SparkFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + + switch (expr) { + case UNARY_PREFIX: + return new SparkUnaryPrefixOperation(generateExpression(depth + 1), SparkUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new SparkUnaryPostfixOperation(generateExpression(depth + 1), SparkUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = SparkBinaryComparisonOperator.getRandom(); + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = SparkBinaryLogicalOperator.getRandom(); + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + SparkBinaryArithmeticOperator.getRandom()); + case CAST: + return new SparkCastOperation(generateExpression(depth + 1), SparkDataType.getRandomType()); + case FUNC: + SparkFunc func = SparkFunc.getRandom(); + return new SparkFunction<>(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new SparkBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new SparkInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new SparkCaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(expr); + } + } + + @Override + public SparkExpression generateConstant() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return SparkConstant.createNullConstant(); + } + SparkDataType[] values = SparkDataType.values(); + SparkDataType constantType = Randomly.fromOptions(values); + switch (constantType) { + case STRING: + return SparkConstant.createStringConstant(globalState.getRandomly().getString()); + case INTEGER: + return SparkConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DOUBLE: + return SparkConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BOOLEAN: + return SparkConstant.createBooleanConstant(Randomly.getBoolean()); + case TIMESTAMP: + return SparkConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case DATE: + return SparkConstant.createDateConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(constantType); + } + } + + @Override + protected SparkExpression generateColumn() { + SparkColumn column = Randomly.fromList(columns); + return new SparkColumnReference(column); + } + + @Override + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (SparkExpression curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new SparkOrderingTerm(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + } + + @Override + public SparkExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + return this; + } + + @Override + public SparkExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public SparkSelect generateSelect() { + return new SparkSelect(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new SparkTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean allowAggregates) { + if (Randomly.getBoolean()) { + return List.of(new SparkColumnReference(new SparkColumn("*", null, null))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new SparkColumnReference(c)) + .collect(Collectors.toList()); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + public enum SparkUnaryPrefixOperator implements Operator { + NOT("NOT"), PLUS("+"), MINUS("-"), BITWISE_NOT("~"); + + private String textRepr; + + SparkUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkUnaryPostfixOperator implements Operator { + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + SparkUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryComparisonOperator implements Operator { + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), RLIKE("RLIKE"); + + private String textRepr; + + SparkBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryLogicalOperator implements Operator { + AND("AND"), OR("OR"); + + private String textRepr; + + SparkBinaryLogicalOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryArithmeticOperator implements Operator { + // Spark supports || for concat, and bitwise operators &, |, ^ + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), BITWISE_AND("&"), BITWISE_OR("|"), + BITWISE_XOR("^"); + + private String textRepr; + + SparkBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkAggregateFunction { + COUNT(1), SUM(1), AVG(1), MIN(1), MAX(1), VARIANCE(1), VAR_SAMP(1), STDDEV_POP(1), STDDEV_SAMP(1), COVAR_POP(2), + COVAR_SAMP(2), CORR(2); + + private int nrArgs; + + SparkAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static SparkAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + public enum SparkFunc { + ROUND(2), FLOOR(1), ABS(1), CEIL(1); + + private int nrArgs; + private boolean isVariadic; + + SparkFunc(int nrArgs) { + this(nrArgs, false); + } + + SparkFunc(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static SparkFunc getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java new file mode 100644 index 000000000..29232fdb2 --- /dev/null +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -0,0 +1,47 @@ +package sqlancer.spark.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.spark.SparkErrors; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkInsertGenerator extends AbstractInsertGenerator { + + private final SparkGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + private final SparkExpressionGenerator gen; + + public SparkInsertGenerator(SparkGlobalState globalState) { + this.globalState = globalState; + this.gen = new SparkExpressionGenerator(globalState); + } + + public static SQLQueryAdapter getQuery(SparkGlobalState globalState) { + return new SparkInsertGenerator(globalState).generate(); + } + + @Override + protected void insertValue(SparkColumn column) { + sb.append(SparkToStringVisitor.asString(gen.generateConstant())); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + SparkTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + + sb.append(" VALUES "); + + List columns = table.getColumns(); + insertColumns(columns); + + SparkErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java new file mode 100644 index 000000000..68cafdafb --- /dev/null +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -0,0 +1,100 @@ +package sqlancer.spark.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.spark.SparkErrors; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkDataType; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkTableGenerator { + + private enum ColumnConstraints { + NOT_NULL, DEFAULT + // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources (Parquet/ORC) + // without specific catalogs (like Delta/Iceberg), so we limit to constraints Spark SQL widely accepts. + } + + private final SparkGlobalState globalState; + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + private final SparkExpressionGenerator gen; + private final SparkTable table; + private final List columnsToBeAdded = new ArrayList<>(); + + public SparkTableGenerator(SparkGlobalState globalState, String tableName) { + this.tableName = tableName; + this.globalState = globalState; + this.table = new SparkTable(tableName, columnsToBeAdded, false); + this.gen = new SparkExpressionGenerator(globalState).setColumns(columnsToBeAdded); + } + + public static SQLQueryAdapter generate(SparkGlobalState globalState, String tableName) { + SparkTableGenerator generator = new SparkTableGenerator(globalState, tableName); + return generator.create(); + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + + sb.append("CREATE TABLE "); + sb.append(globalState.getDatabaseName()); + sb.append("."); + sb.append(tableName); + sb.append(" ("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(i); + } + sb.append(")"); + sb.append(" USING PARQUET"); + + // TODO: implement PARTITION BY clause + // TODO: implement CLUSTERED BY clauses + // TODO: implement ROW FORMAT and STORED AS clauses + // TODO: randomly add some predefined TABLEPROPERTIES + + SparkErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + + private void appendColumn(int columnId) { + String columnName = DBMSCommon.createColumnName(columnId); + sb.append(columnName); + sb.append(" "); + SparkDataType randType = SparkSchema.SparkDataType.getRandomType(); + sb.append(randType); + columnsToBeAdded.add(new SparkColumn(columnName, table, randType)); + appendColumnConstraint(); + } + + private void appendColumnConstraint() { + if (Randomly.getBoolean()) { + return; + } + + ColumnConstraints constraint = Randomly.fromOptions(ColumnConstraints.values()); + switch (constraint) { + case NOT_NULL: + sb.append(" NOT NULL"); + break; + case DEFAULT: + sb.append(" DEFAULT "); + sb.append(SparkToStringVisitor.asString(gen.generateConstant())); + sb.append(" "); + break; + default: + throw new AssertionError(constraint); + } + } +} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index f5aeefa12..f6be45648 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -11,6 +11,7 @@ public class TestConfig { public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String HIVE_ENV = "HIVE_AVAILABLE"; + public static final String SPARK_ENV = "SPARK_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; public static final String MYSQL_ENV = "MYSQL_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestSparkTLP.java b/test/sqlancer/dbms/TestSparkTLP.java new file mode 100644 index 000000000..83302ceff --- /dev/null +++ b/test/sqlancer/dbms/TestSparkTLP.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSparkTLP { + + @Test + public void testSparkTLPWhere() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.SPARK_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", + "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "spark", "--oracle", "TLPWhere" })); + } +} \ No newline at end of file From b4c3763d9db5b179e41c170dfd5d7fa31d4bc9eb Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 10:48:47 +0000 Subject: [PATCH 1156/1284] Fix where clause in Materialize --- src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java index d61e452f8..f7ff76305 100644 --- a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -600,6 +600,7 @@ public String generateOptimizedQueryString(MaterializeSelect select, Materialize } select.setSelectType(SelectType.ALL); } + select.setWhereClause(whereCondition); return select.asString(); } From e4e013d064232ea7f9d17b8bdbb5fb2411c035ae Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 10:55:54 +0000 Subject: [PATCH 1157/1284] Fix up Materialize CI --- .github/workflows/main.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c53192aa..026fa52bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -392,10 +392,7 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest - sleep 5 - # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 - psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -420,10 +417,7 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest - sleep 5 - # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 - psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest - name: Set up JDK 11 uses: actions/setup-java@v3 with: From dacd33c3ba671bed063d618d864c14d5bba0a4f5 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 12:50:10 +0000 Subject: [PATCH 1158/1284] Try to fix duplicate table creation in Materialize --- src/sqlancer/materialize/MaterializeProvider.java | 11 ++++++++--- .../materialize/gen/MaterializeTableGenerator.java | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java index 5b18b1dfb..e7bdb4c4f 100644 --- a/src/sqlancer/materialize/MaterializeProvider.java +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -240,12 +240,17 @@ protected void readFunctions(MaterializeGlobalState globalState) throws SQLExcep } protected void createTables(MaterializeGlobalState globalState, int numTables) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < numTables) { + int existingTables = globalState.getSchema().getDatabaseTables().size(); + int createdTables = 0; + int nextTableIndex = existingTables; + while (existingTables + createdTables < numTables) { try { - String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + String tableName = DBMSCommon.createTableName(nextTableIndex++); SQLQueryAdapter createTable = MaterializeTableGenerator.generate(tableName, globalState.getSchema(), generateOnlyKnown, globalState); - globalState.executeStatement(createTable); + if (globalState.executeStatement(createTable)) { + createdTables++; + } } catch (IgnoreMeException e) { } diff --git a/src/sqlancer/materialize/gen/MaterializeTableGenerator.java b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java index f132f7370..c6772db47 100644 --- a/src/sqlancer/materialize/gen/MaterializeTableGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java @@ -52,6 +52,7 @@ public MaterializeTableGenerator(String tableName, MaterializeSchema newSchema, errors.add("no collation was derived for partition key column"); errors.add("inherits from generated column but specifies identity"); errors.add("inherits from generated column but specifies default"); + errors.add("already exists"); MaterializeCommon.addCommonExpressionErrors(errors); MaterializeCommon.addCommonTableErrors(errors); } From 44a156e8fe7c9c913c782808ce0540beca3be342 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 14:10:22 +0000 Subject: [PATCH 1159/1284] Adapt materialize insert error message --- src/sqlancer/materialize/gen/MaterializeInsertGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java index 01015514b..7a5374b95 100644 --- a/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java @@ -26,7 +26,7 @@ public static SQLQueryAdapter insert(MaterializeGlobalState globalState) { MaterializeCommon.addCommonExpressionErrors(errors); errors.add("multiple assignments to same column"); errors.add("violates foreign key constraint"); - errors.add("value too long for type character varying"); + errors.add("value too long for type character"); errors.add("conflicting key value violates exclusion constraint"); errors.add("violates not-null constraint"); errors.add("current transaction is aborted"); From e039cbaa71e0dfabc69e816bf5714692f5c5e412 Mon Sep 17 00:00:00 2001 From: Elshaarawy-1 Date: Wed, 11 Feb 2026 11:36:08 +0200 Subject: [PATCH 1160/1284] Add DuckDBSchema.getIndexes() implementation Based on PR #1171 Addresses #1163 --- src/sqlancer/duckdb/DuckDBSchema.java | 23 ++++++++++++++++--- .../duckdb/gen/DuckDBIndexGenerator.java | 4 ++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 379576fd8..7e89da7e6 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -218,8 +218,8 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { public static class DuckDBTable extends AbstractRelationalTable { - public DuckDBTable(String tableName, List columns, boolean isView) { - super(tableName, columns, Collections.emptyList(), isView); + public DuckDBTable(String tableName, List columns, List indexes, boolean isView) { + super(tableName, columns, indexes, isView); } } @@ -233,7 +233,8 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName } List databaseColumns = getTableColumns(con, tableName); boolean isView = tableName.startsWith("v"); - DuckDBTable t = new DuckDBTable(tableName, databaseColumns, isView); + List indexes = getIndexes(con, tableName, databaseName); + DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { c.setTable(t); } @@ -243,6 +244,22 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName return new DuckDBSchema(databaseTables); } + private static List getIndexes(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT INDEX_NAME FROM duckdb_indexes() WHERE DATABASE_NAME = '%s' and TABLE_NAME = '%s';", + databaseName, tableName))) { + while (rs.next()) { + String indexName = rs.getString("INDEX_NAME"); + indexes.add(TableIndex.create(indexName)); + } + } + } + return indexes; + } + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 597fda19f..cc4114f63 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -23,7 +23,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append("UNIQUE "); } sb.append("INDEX "); - sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); @@ -40,7 +40,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } } sb.append(")"); - errors.add("already exists!"); + // errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("cannot create an index on the rowid"); } From 353edac5b4217070dd37206925c3a5823f8f62f0 Mon Sep 17 00:00:00 2001 From: Elshaarawy-1 Date: Wed, 11 Feb 2026 11:36:33 +0200 Subject: [PATCH 1161/1284] Fix database name filter in getIndexes() Remove DATABASE_NAME filter to fix index collision bug. --- src/sqlancer/duckdb/DuckDBSchema.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 7e89da7e6..8141220fa 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import sqlancer.IgnoreMeException; @@ -233,7 +232,7 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName } List databaseColumns = getTableColumns(con, tableName); boolean isView = tableName.startsWith("v"); - List indexes = getIndexes(con, tableName, databaseName); + List indexes = getIndexes(con, tableName); DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { c.setTable(t); @@ -244,13 +243,12 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName return new DuckDBSchema(databaseTables); } - private static List getIndexes(SQLConnection con, String tableName, String databaseName) - throws SQLException { + private static List getIndexes(SQLConnection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format( - "SELECT INDEX_NAME FROM duckdb_indexes() WHERE DATABASE_NAME = '%s' and TABLE_NAME = '%s';", - databaseName, tableName))) { + "SELECT index_name FROM duckdb_indexes() WHERE database_name = current_database() AND table_name = '%s';", + tableName))) { while (rs.next()) { String indexName = rs.getString("INDEX_NAME"); indexes.add(TableIndex.create(indexName)); @@ -259,7 +257,7 @@ private static List getIndexes(SQLConnection con, String tableName, } return indexes; } - + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { From 5caba27dacd843f4dd70a69f4febfc35e94d97e8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 1 Mar 2026 05:02:22 +0100 Subject: [PATCH 1162/1284] ClickHouse: add EMPTY_LIST_OF_COLUMNS_PASSED to expected errors ClickHouse PR https://github.com/ClickHouse/ClickHouse/pull/81835 added a check that forbids creating tables without insertable columns (e.g., tables with only MATERIALIZED or ALIAS columns). Since SQLancer's random table generator can produce such definitions, ClickHouse now returns EMPTY_LIST_OF_COLUMNS_PASSED, which should be treated as an expected error rather than causing an AssertionError. The table creation retry loop in ClickHouseProvider.generateDatabase already handles expected errors by regenerating the table definition. --- src/sqlancer/clickhouse/ClickHouseErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 2c4d8d1b8..09fbe5ea8 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -41,6 +41,7 @@ public static List getExpectedExpressionErrors() { "Cannot convert out of range floating point value to integer type", "Unexpected inf or nan to integer conversion", "No such name in Block::erase", // https://github.com/ClickHouse/ClickHouse/issues/42769 "EMPTY_LIST_OF_COLUMNS_QUERIED", // https://github.com/ClickHouse/ClickHouse/issues/43003 + "EMPTY_LIST_OF_COLUMNS_PASSED", // https://github.com/ClickHouse/ClickHouse/pull/81835 "cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)", "AMBIGUOUS_IDENTIFIER", "CYCLIC_ALIASES", "Positional argument numeric constant expression is not representable as", "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", From 1506cf28b48f632d29df76dd992d85f0dfa1d829 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Mar 2026 22:40:19 +0000 Subject: [PATCH 1163/1284] Bump jacoco version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2037b71ce..767fedcce 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ org.jacoco jacoco-maven-plugin - 0.8.8 + 0.8.12 From d7518297ead5d46bff05552f8ecedd042afe5058 Mon Sep 17 00:00:00 2001 From: albertZhangTJ Date: Tue, 3 Mar 2026 01:37:32 +0000 Subject: [PATCH 1164/1284] Add expected errors for postgres 18 --- src/sqlancer/postgres/gen/PostgresCommon.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 63bc885cb..992cbc880 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -48,6 +48,7 @@ public static List getCommonTableErrors() { errors.add("is not commutative"); // exclude errors.add("operator requires run-time type coercion"); // exclude + errors.add("partitioned tables cannot be unlogged"); return errors; } @@ -59,6 +60,8 @@ public static void addCommonTableErrors(ExpectedErrors errors) { public static List getCommonExpressionErrors() { ArrayList errors = new ArrayList<>(); + errors.add("for encoding \"SQL_ASCII\" does not exist"); + errors.add("invalid byte sequence for encoding"); errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); From d7b272b93b946a66e39dc64067d5ee5b21b105f5 Mon Sep 17 00:00:00 2001 From: albertZhangTJ Date: Tue, 3 Mar 2026 02:14:40 +0000 Subject: [PATCH 1165/1284] Update postgres version in CI to 18 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c53192aa..c07452653 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -504,7 +504,7 @@ jobs: - name: Set up PostgreSQL uses: harmon758/postgresql-action@v1.0.0 with: - postgresql version: '13' + postgresql version: '18' postgresql user: 'sqlancer' postgresql password: 'sqlancer' postgresql db: 'test' From a4bf0a9d65c52843d05634ca21c72f1cb1eeea72 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Sun, 22 Mar 2026 18:39:58 +0000 Subject: [PATCH 1166/1284] Disable SET_UNLOGGED_LOGGED for partitioned table for PG18; Fix NPE for PostgresBinaryComparisonOperation --- src/sqlancer/postgres/PostgresSchema.java | 24 ++++++++++++++++--- .../PostgresBinaryComparisonOperation.java | 2 +- .../gen/PostgresAlterTableGenerator.java | 3 +++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index c99c8648e..17089b011 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -164,6 +164,7 @@ public enum TableType { private final TableType tableType; private final List statistics; private final boolean isInsertable; + private final boolean isPartitioned; public PostgresTable(String tableName, List columns, List indexes, TableType tableType, List statistics, boolean isView, boolean isInsertable) { @@ -171,6 +172,18 @@ public PostgresTable(String tableName, List columns, List columns, List indexes, + TableType tableType, List statistics, boolean isView, boolean isInsertable, + boolean isPartitioned) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + this.isPartitioned = isPartitioned; } public List getStatistics() { @@ -185,6 +198,10 @@ public boolean isInsertable() { return isInsertable; } + public boolean isPartitioned() { + return isPartitioned; + } + } public static final class PostgresStatisticsObject { @@ -225,11 +242,12 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery( - "SELECT table_name, table_schema, table_type, is_insertable_into FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { + "SELECT t.table_name, t.table_schema, t.table_type, t.is_insertable_into, c.relkind FROM information_schema.tables t JOIN pg_class c ON c.relname = t.table_name JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = t.table_schema WHERE t.table_schema='public' OR t.table_schema LIKE 'pg_temp_%' ORDER BY t.table_name;")) { while (rs.next()) { String tableName = rs.getString("table_name"); String tableTypeSchema = rs.getString("table_schema"); boolean isInsertable = rs.getBoolean("is_insertable_into"); + boolean isPartitioned = "p".equals(rs.getString("relkind")); // TODO: also check insertable // TODO: insert into view? boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || @@ -240,7 +258,7 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa List indexes = getIndexes(con, tableName); List statistics = getStatistics(con); PostgresTable t = new PostgresTable(tableName, databaseColumns, indexes, tableType, statistics, - isView, isInsertable); + isView, isInsertable, isPartitioned); for (PostgresColumn c : databaseColumns) { c.setTable(t); } @@ -324,4 +342,4 @@ public String getDatabaseName() { return databaseName; } -} +} \ No newline at end of file diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 95efe5b72..daa464b77 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -126,7 +126,7 @@ public PostgresConstant getExpectedValue() { PostgresConstant leftExpectedValue = getLeft().getExpectedValue(); PostgresConstant rightExpectedValue = getRight().getExpectedValue(); if (leftExpectedValue == null || rightExpectedValue == null) { - return null; + return PostgresConstant.createNullConstant(); } return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); } diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 6e0c436ee..eb53f6df1 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -127,6 +127,9 @@ public List getActions(ExpectedErrors errors) { if (!randomTable.hasIndexes()) { action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); } + if (randomTable.isPartitioned()){ + action.remove(Action.SET_LOGGED_UNLOGGED); + } if (action.isEmpty()) { throw new IgnoreMeException(); } From 2b8796cda88cfa17856388ef57f1f8ea9989e6d6 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Sun, 22 Mar 2026 20:03:57 +0000 Subject: [PATCH 1167/1284] Add cannot drop inherited constraint expected error --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index eb53f6df1..d39006f3c 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -238,6 +238,8 @@ public SQLQueryAdapter generate() { errors.add("is in a primary key"); errors.add("is an identity column"); errors.add("is in index used as replica identity"); + // PG18 update: otherwise we need to encode contraint inheritance info in PostgreColumn + errors.add("cannot drop inherited constraint"); } break; case ALTER_COLUMN_SET_STATISTICS: From 21ec5c73f541a5cb85b488378caed188f184b240 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Mon, 23 Mar 2026 15:49:09 +0000 Subject: [PATCH 1168/1284] Format the previous commits; Update PostgresBinaryComparisonOperator null handling --- src/sqlancer/postgres/PostgresSchema.java | 2 +- .../postgres/ast/PostgresBinaryComparisonOperation.java | 3 ++- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 +- src/sqlancer/postgres/gen/PostgresCommon.java | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 17089b011..3cc8a6e4b 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -172,7 +172,7 @@ public PostgresTable(String tableName, List columns, List getActions(ExpectedErrors errors) { if (!randomTable.hasIndexes()) { action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); } - if (randomTable.isPartitioned()){ + if (randomTable.isPartitioned()) { action.remove(Action.SET_LOGGED_UNLOGGED); } if (action.isEmpty()) { diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 992cbc880..e9f27aeb2 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -61,7 +61,7 @@ public static List getCommonExpressionErrors() { ArrayList errors = new ArrayList<>(); errors.add("for encoding \"SQL_ASCII\" does not exist"); - errors.add("invalid byte sequence for encoding"); + errors.add("invalid byte sequence for encoding"); errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); From 9a3d65e3a10f974d9ccd4b125caaf05fba122c7f Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Mon, 23 Mar 2026 16:17:14 +0000 Subject: [PATCH 1169/1284] Fix checkStyle errors --- src/sqlancer/postgres/PostgresSchema.java | 3 ++- .../postgres/ast/PostgresBinaryComparisonOperation.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 3cc8a6e4b..17ef652dd 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -342,4 +342,5 @@ public String getDatabaseName() { return databaseName; } -} \ No newline at end of file +} + diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 9341d3578..3ee69ba17 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; -import sqlancer.Randomly; import sqlancer.IgnoreMeException; +import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -138,3 +138,4 @@ public PostgresDataType getExpressionType() { } } + From d4f459fcfb3127f609a59c71097f5c2ac4e48a27 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Mon, 23 Mar 2026 16:19:49 +0000 Subject: [PATCH 1170/1284] Fix checkStyle errors --- src/sqlancer/postgres/PostgresSchema.java | 1 - src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 17ef652dd..20337fea7 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -343,4 +343,3 @@ public String getDatabaseName() { } } - diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 3ee69ba17..b77060dfd 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -138,4 +138,3 @@ public PostgresDataType getExpressionType() { } } - From f2b42cef4fe10e8500cc4181f241b863f1bae1ea Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 26 Mar 2026 21:52:56 +0800 Subject: [PATCH 1171/1284] Fix Materialize CI by removing deprecated MZ_EAT_MY_DATA flag MZ_EAT_MY_DATA=1 is a deprecated env var from old Materialize versions that triggered an initialization path running CockroachDB-style SQL (SET CLUSTER SETTING sql.stats.forecasts.enabled = false), which is no longer valid in the current Materialize architecture. Also add a pg_isready health check to ensure Materialize is accepting connections before tests run. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3514d904..665394c8a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -392,7 +392,8 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest + until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -417,7 +418,8 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest + until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 uses: actions/setup-java@v3 with: From 708af09439398b662c674ce7752f954a4a0b1295 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 08:08:49 +0800 Subject: [PATCH 1172/1284] Replace hardcoded sleeps with readiness loops in CI workflows Use `until` polling loops instead of fixed `sleep` durations to wait for services to become ready, reducing flakiness and unnecessary wait time across CnosDB, ClickHouse, CockroachDB, OceanBase, Presto, TiDB, YugabyteDB, and Doris jobs. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 665394c8a..b0bd5355d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -111,7 +111,7 @@ jobs: run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - sleep 5 + until curl -sf http://127.0.0.1:8902/ping 2>/dev/null; do sleep 1; done - name: Run Tests run: | CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test @@ -137,7 +137,7 @@ jobs: run: | docker pull clickhouse/clickhouse-server:24.3.1.2672 docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:24.3.1.2672 - sleep 5 + until curl -sf http://127.0.0.1:8123/ping 2>/dev/null; do sleep 1; done - name: Run Tests run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test - name: Show fatal errors @@ -166,7 +166,7 @@ jobs: run: | wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & - sleep 10 + until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests @@ -194,7 +194,7 @@ jobs: run: | wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & - sleep 10 + until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests @@ -483,7 +483,7 @@ jobs: - name: Set up OceanBase run: | docker run -p 2881:2881 --name oceanbase-ce -e MODE=mini -d oceanbase/oceanbase-ce:4.2.1-lts - sleep 120 + until mysql -h127.1 -uroot@test -P2881 --connect-timeout=3 -Doceanbase -A -e "SELECT 1" 2>/dev/null; do sleep 5; done mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | @@ -537,13 +537,13 @@ jobs: docker pull prestodb/presto:latest echo "connector.name=memory" >> memory.properties docker run -p 8080:8080 -d -v ./memory.properties:/opt/presto-server/etc/catalog/memory.properties --name presto prestodb/presto:latest - sleep 30 + until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests run: | PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoNoREC test - docker restart presto && sleep 30 + docker restart presto && until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoTLP test sqlite: name: DBMS Tests (SQLite) @@ -607,9 +607,9 @@ jobs: run: | docker pull hawkingrei/tidb-playground:nightly-2025-09-16 docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 - sleep 10 + until mysql -h 127.0.0.1 -P 4000 -u root --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done - name: Create SQLancer user - run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: | TIDB_AVAILABLE=true mvn -Dtest=TestTiDBTLP test @@ -634,9 +634,9 @@ jobs: run: | docker pull hawkingrei/tidb-playground:nightly-2025-09-16 docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 - sleep 10 + until mysql -h 127.0.0.1 -P 4000 -u root --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done - name: Create SQLancer user - run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test @@ -659,7 +659,7 @@ jobs: run: | docker pull yugabytedb/yugabyte:latest docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false - sleep 5 + until pg_isready -h localhost -p 5433; do sleep 1; done - name: Run Tests run: | YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLNoREC test @@ -696,7 +696,7 @@ jobs: cd ../be ./bin/start_be.sh --daemon - sleep 30 + until mysql -u root -h 127.0.0.1 --port 9030 --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done IP=$(hostname -I | awk '{print $1}') mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';" mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" From 5032eae9a96fa2a37e8d6f0ff0f4f2a475a0e925 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 08:41:24 +0800 Subject: [PATCH 1173/1284] Update MySQL CI from 8.0.36 to 8.4 and fix compatibility issues - Update mysql service image in CI from 8.0.36 to 8.4 - Remove show_old_temporals system variable (removed in MySQL 8.4) - Update expected error for integer literals in ORDER BY: MySQL 8.4 changed the error context from 'order clause' to 'EXISTS subquery' Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 2 +- src/sqlancer/mysql/MySQLErrors.java | 2 +- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 1 - src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 665394c8a..34c0e7e5d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -438,7 +438,7 @@ jobs: runs-on: ubuntu-latest services: mysql: - image: mysql:8.0.36 + image: mysql:8.4 env: MYSQL_ROOT_PASSWORD: root ports: diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index f01b30e03..bec149920 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -32,7 +32,7 @@ public static List getExpressionRegexErrors() { // "00000000000000000000-0" } - errors.add(Pattern.compile("Unknown column '.*' in 'order clause'")); + errors.add(Pattern.compile("Unknown column '.*' in 'EXISTS subquery'")); return errors; } diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 79333eb36..e350685ef 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -99,7 +99,6 @@ private enum Action { SCHEMA_DEFINITION_CACHE("schema_definition_cache", (r) -> r.getLong(256, 524288), Scope.GLOBAL), // SHOW_CREATE_TABLE_VERBOSITY("show_create_table_verbosity", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - SHOW_OLD_TEMPORALS("show_old_temporals", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), /* * sort_buffer_size is commented out as a workaround for https://bugs.mysql.com/bug.php?id=95969 */ diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 28665c328..c613f2622 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -37,7 +37,7 @@ public class MySQLPivotedQuerySynthesisOracle public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { super(globalState); MySQLErrors.addExpressionErrors(errors); - errors.add("in 'order clause'"); // e.g., Unknown column '2067708013' in 'order clause' + errors.add("in 'EXISTS subquery'"); // e.g., Unknown column '2067708013' in 'EXISTS subquery' (MySQL 8.4+) } @Override From 227904af7a92ecfa69f5be07ddc8782fc7d3b2ef Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 09:01:12 +0800 Subject: [PATCH 1174/1284] Optimize CI: remove unnecessary fetch-depth and update action versions - Remove fetch-depth: 0 from all jobs (full git history is not needed for running tests) - Update actions/checkout from v3/v2 to v4 - Update actions/setup-java from v3 to v4 Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 156 +++++++++++++------------------------ 1 file changed, 52 insertions(+), 104 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 665394c8a..f0b8317e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,11 +24,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -48,11 +46,9 @@ jobs: name: DBMS Tests (Citus) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -96,11 +92,9 @@ jobs: name: DBMS Tests (CnosDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -122,11 +116,9 @@ jobs: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -151,11 +143,9 @@ jobs: name: DBMS Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -179,11 +169,9 @@ jobs: name: QPG Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -213,11 +201,9 @@ jobs: - 8000:8000 - 3307:3307 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -234,9 +220,7 @@ jobs: name: DBMS Tests (DataFusion) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Rust uses: actions-rs/toolchain@v1 with: @@ -247,7 +231,7 @@ jobs: cd src/sqlancer/datafusion/server/datafusion_server cargo run & sleep 300 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -263,11 +247,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -283,11 +265,9 @@ jobs: name: DBMS Tests (H2) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -319,11 +299,9 @@ jobs: volumes: - warehouse:/opt/hive/data/warehouse steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -337,11 +315,9 @@ jobs: name: DBMS Tests (HSQLB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -366,11 +342,9 @@ jobs: options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -386,16 +360,14 @@ jobs: name: DBMS Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Materialize run: | docker pull materialize/materialized:latest docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -412,16 +384,14 @@ jobs: name: QPG Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Materialize run: | docker pull materialize/materialized:latest docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -445,11 +415,9 @@ jobs: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -469,11 +437,9 @@ jobs: name: DBMS Tests (OceanBase) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -494,9 +460,7 @@ jobs: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up PostgreSQL uses: harmon758/postgresql-action@v1.0.0 with: @@ -505,7 +469,7 @@ jobs: postgresql password: 'sqlancer' postgresql db: 'test' - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -523,11 +487,9 @@ jobs: name: DBMS Tests (Presto) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -550,11 +512,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -573,11 +533,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -592,11 +550,9 @@ jobs: name: DBMS Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -619,11 +575,9 @@ jobs: name: QPG Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -644,11 +598,9 @@ jobs: name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -671,11 +623,9 @@ jobs: name: DBMS Tests (Apache Doris) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -713,11 +663,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' From 91d7a6ae05158c1d908a1b416b08bceafb5011d0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 09:03:37 +0800 Subject: [PATCH 1175/1284] Restore 'order clause' error alongside 'EXISTS subquery' for MySQL 8.4 MySQL 8.4 uses 'order clause' when the ORDER BY only contains an integer literal, and 'EXISTS subquery' when ORDER BY also contains an EXISTS expression. Both patterns need to be handled. Co-Authored-By: Claude Sonnet 4.6 --- src/sqlancer/mysql/MySQLErrors.java | 1 + src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index bec149920..989c8fed6 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -32,6 +32,7 @@ public static List getExpressionRegexErrors() { // "00000000000000000000-0" } + errors.add(Pattern.compile("Unknown column '.*' in 'order clause'")); errors.add(Pattern.compile("Unknown column '.*' in 'EXISTS subquery'")); return errors; diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index c613f2622..c1fe893b6 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -37,6 +37,7 @@ public class MySQLPivotedQuerySynthesisOracle public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { super(globalState); MySQLErrors.addExpressionErrors(errors); + errors.add("in 'order clause'"); // e.g., Unknown column '2067708013' in 'order clause' errors.add("in 'EXISTS subquery'"); // e.g., Unknown column '2067708013' in 'EXISTS subquery' (MySQL 8.4+) } From 8c143a3d4951bf85d891a0fbbefcfa0f08ff38d9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 09:09:47 +0800 Subject: [PATCH 1176/1284] Fix CnosDB readiness check to use TCP port probe The /ping endpoint does not exist on CnosDB, causing curl -f to loop forever. Switch to nc -z which just checks that the port is open. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0bd5355d..6bd47d3e1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -111,7 +111,7 @@ jobs: run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - until curl -sf http://127.0.0.1:8902/ping 2>/dev/null; do sleep 1; done + until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done - name: Run Tests run: | CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test From 6837dd1d05e7da8ef49a577a6b58498c03da9325 Mon Sep 17 00:00:00 2001 From: Carmen Kwan Date: Thu, 2 Apr 2026 01:12:40 +0200 Subject: [PATCH 1177/1284] Fix CONTRIBUTING.md Stop referencing files that don't exist --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5db3bdc69..ea5baea1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ If you do not find an option to import Maven projects, you might need to install ## Implementing Support for a New DBMS -The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see `DuckDBNoRECOracle` or `DuckDBQueryPartitioningWhereTester` for TLP). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). +The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see enum value `NOREC` in `DuckDBOracleFactory`). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). Please consider the following suggestions when creating a PR to contribute a new DBMS: * Ensure that `mvn verify -DskipTests=true` does not result in style violations. From 8787427efc005358f83913ace3fc1beaf2350c2e Mon Sep 17 00:00:00 2001 From: Aman Bihari <161295600+codebreaker32@users.noreply.github.com> Date: Fri, 3 Apr 2026 06:16:57 +0000 Subject: [PATCH 1178/1284] Remove outdated comment and wrong errors --- src/sqlancer/spark/SparkErrors.java | 2 -- src/sqlancer/spark/SparkProvider.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java index 97c8056a3..83ba84bb3 100644 --- a/src/sqlancer/spark/SparkErrors.java +++ b/src/sqlancer/spark/SparkErrors.java @@ -35,8 +35,6 @@ public static List getExpressionErrors() { errors.add("expression is neither present in the group by"); errors.add("is not a valid grouping expression"); errors.add("is not contained in either an aggregate function or the GROUP BY clause"); - errors.add("PARSE_SYNTAX_ERROR"); - errors.add("Syntax error"); return errors; } diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java index f53ca10a8..89f24a738 100644 --- a/src/sqlancer/spark/SparkProvider.java +++ b/src/sqlancer/spark/SparkProvider.java @@ -28,7 +28,7 @@ public SparkProvider() { } public enum Action implements AbstractAction { - INSERT(SparkInsertGenerator::getQuery); // You will need to create this class + INSERT(SparkInsertGenerator::getQuery); private final SQLQueryProvider sqlQueryProvider; From 787fcf3225a313059c92dadf3cab25ea6cbc120e Mon Sep 17 00:00:00 2001 From: Aman Date: Sat, 4 Apr 2026 15:05:35 +0530 Subject: [PATCH 1179/1284] Format spark/* --- src/sqlancer/Main.java | 2 +- src/sqlancer/spark/SparkErrors.java | 2 +- src/sqlancer/spark/SparkGlobalState.java | 2 +- src/sqlancer/spark/SparkOptions.java | 2 +- src/sqlancer/spark/SparkProvider.java | 5 ++-- src/sqlancer/spark/SparkSchema.java | 24 ++++++++++++------- src/sqlancer/spark/SparkToStringVisitor.java | 5 ++-- .../spark/ast/SparkBetweenOperation.java | 2 +- .../spark/ast/SparkBinaryOperation.java | 2 +- .../spark/ast/SparkCaseOperation.java | 2 +- .../spark/ast/SparkCastOperation.java | 2 +- .../spark/ast/SparkColumnReference.java | 2 +- src/sqlancer/spark/ast/SparkConstant.java | 2 +- src/sqlancer/spark/ast/SparkExpression.java | 2 +- src/sqlancer/spark/ast/SparkInOperation.java | 2 +- src/sqlancer/spark/ast/SparkJoin.java | 2 +- src/sqlancer/spark/ast/SparkOrderingTerm.java | 2 +- src/sqlancer/spark/ast/SparkSelect.java | 2 +- .../spark/ast/SparkTableReference.java | 2 +- .../spark/ast/SparkUnaryPostfixOperation.java | 2 +- .../spark/ast/SparkUnaryPrefixOperation.java | 2 +- .../spark/gen/SparkExpressionGenerator.java | 2 +- .../spark/gen/SparkInsertGenerator.java | 2 +- .../spark/gen/SparkTableGenerator.java | 8 ++++--- 24 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index f778bd7da..faf35e3c9 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -44,11 +44,11 @@ import sqlancer.postgres.PostgresProvider; import sqlancer.presto.PrestoProvider; import sqlancer.questdb.QuestDBProvider; +import sqlancer.spark.SparkProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; import sqlancer.yugabyte.ysql.YSQLProvider; -import sqlancer.spark.SparkProvider; public final class Main { diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java index 83ba84bb3..a3a96061f 100644 --- a/src/sqlancer/spark/SparkErrors.java +++ b/src/sqlancer/spark/SparkErrors.java @@ -62,4 +62,4 @@ public static List getInsertErrors() { public static void addInsertErrors(ExpectedErrors errors) { errors.addAll(getInsertErrors()); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkGlobalState.java b/src/sqlancer/spark/SparkGlobalState.java index e79826332..d78c737e5 100644 --- a/src/sqlancer/spark/SparkGlobalState.java +++ b/src/sqlancer/spark/SparkGlobalState.java @@ -8,4 +8,4 @@ public class SparkGlobalState extends SQLGlobalState protected SparkSchema readSchema() throws Exception { return SparkSchema.fromConnection(getConnection(), getDatabaseName()); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkOptions.java b/src/sqlancer/spark/SparkOptions.java index c9422a910..7b347ceef 100644 --- a/src/sqlancer/spark/SparkOptions.java +++ b/src/sqlancer/spark/SparkOptions.java @@ -40,4 +40,4 @@ public TestOracle create(SparkGlobalState globalState) throws public List getTestOracleFactory() { return oracle; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java index 89f24a738..817a92471 100644 --- a/src/sqlancer/spark/SparkProvider.java +++ b/src/sqlancer/spark/SparkProvider.java @@ -109,7 +109,8 @@ public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLExce con = DriverManager.getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName), username, password); try (Statement s = con.createStatement()) { - // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the generator loves to do. + // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the + // generator loves to do. s.execute("SET spark.sql.ansi.enabled=false"); } return new SQLConnection(con); @@ -119,4 +120,4 @@ public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLExce public String getDBMSName() { return "spark"; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java index 849652b19..8d988e31e 100644 --- a/src/sqlancer/spark/SparkSchema.java +++ b/src/sqlancer/spark/SparkSchema.java @@ -85,8 +85,9 @@ private static List getTableColumns(SQLConnection con, String table String columnName = rs.getString("col_name"); String dataType = rs.getString("data_type"); // Filter out Spark partition info or comments usually at bottom of describe - if (columnName.startsWith("#") || columnName.isEmpty()) + if (columnName.startsWith("#") || columnName.isEmpty()) { continue; + } columns.add(new SparkColumn(columnName, null, getColumnType(dataType))); } @@ -97,18 +98,25 @@ private static List getTableColumns(SQLConnection con, String table private static SparkDataType getColumnType(String typeString) { String upper = typeString.toUpperCase(); - if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) + if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) { return SparkDataType.STRING; - if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) + } + if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) { return SparkDataType.INTEGER; - if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) + } + if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) { return SparkDataType.DOUBLE; - if (upper.startsWith("BOOLEAN")) + } + if (upper.startsWith("BOOLEAN")) { return SparkDataType.BOOLEAN; - if (upper.startsWith("TIMESTAMP")) + } + if (upper.startsWith("TIMESTAMP")) { return SparkDataType.TIMESTAMP; - if (upper.startsWith("DATE")) + } + if (upper.startsWith("DATE")) { return SparkDataType.DATE; + } return SparkDataType.STRING; // Fallback } -} \ No newline at end of file + +} diff --git a/src/sqlancer/spark/SparkToStringVisitor.java b/src/sqlancer/spark/SparkToStringVisitor.java index 91f47e32c..0777c86a6 100644 --- a/src/sqlancer/spark/SparkToStringVisitor.java +++ b/src/sqlancer/spark/SparkToStringVisitor.java @@ -63,7 +63,8 @@ private void visit(SparkSelect select) { sb.append(" LIMIT "); visit(select.getLimitClause()); } - // Spark supports OFFSET, though strictly usually with LIMIT or in newer versions + // Spark supports OFFSET, though strictly usually with LIMIT or in newer + // versions if (select.getOffsetClause() != null) { sb.append(" OFFSET "); visit(select.getOffsetClause()); @@ -117,4 +118,4 @@ public static String asString(SparkExpression expr) { visitor.visit(expr); return visitor.get(); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkBetweenOperation.java b/src/sqlancer/spark/ast/SparkBetweenOperation.java index f229c1c7c..59297ba8f 100644 --- a/src/sqlancer/spark/ast/SparkBetweenOperation.java +++ b/src/sqlancer/spark/ast/SparkBetweenOperation.java @@ -7,4 +7,4 @@ public class SparkBetweenOperation extends NewBetweenOperatorNode public SparkBinaryOperation(SparkExpression left, SparkExpression right, Operator op) { super(left, right, op); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkCaseOperation.java b/src/sqlancer/spark/ast/SparkCaseOperation.java index fb1ee0cd8..995fd7f52 100644 --- a/src/sqlancer/spark/ast/SparkCaseOperation.java +++ b/src/sqlancer/spark/ast/SparkCaseOperation.java @@ -10,4 +10,4 @@ public SparkCaseOperation(SparkExpression switchCondition, List List expressions, SparkExpression elseExpr) { super(switchCondition, conditions, expressions, elseExpr); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkCastOperation.java b/src/sqlancer/spark/ast/SparkCastOperation.java index 3bc5eb30d..547551285 100644 --- a/src/sqlancer/spark/ast/SparkCastOperation.java +++ b/src/sqlancer/spark/ast/SparkCastOperation.java @@ -22,4 +22,4 @@ public SparkExpression getExpression() { public SparkDataType getType() { return type; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkColumnReference.java b/src/sqlancer/spark/ast/SparkColumnReference.java index 75e92d267..ccd1b7855 100644 --- a/src/sqlancer/spark/ast/SparkColumnReference.java +++ b/src/sqlancer/spark/ast/SparkColumnReference.java @@ -8,4 +8,4 @@ public class SparkColumnReference extends ColumnReferenceNode { -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkInOperation.java b/src/sqlancer/spark/ast/SparkInOperation.java index 37a80e3ff..430d9b5c2 100644 --- a/src/sqlancer/spark/ast/SparkInOperation.java +++ b/src/sqlancer/spark/ast/SparkInOperation.java @@ -9,4 +9,4 @@ public class SparkInOperation extends NewInOperatorNode impleme public SparkInOperation(SparkExpression left, List right, boolean isNegated) { super(left, right, isNegated); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkJoin.java b/src/sqlancer/spark/ast/SparkJoin.java index 44da7fba4..a59eaff48 100644 --- a/src/sqlancer/spark/ast/SparkJoin.java +++ b/src/sqlancer/spark/ast/SparkJoin.java @@ -43,4 +43,4 @@ public SparkExpression getOnClause() { public void setOnClause(SparkExpression onClause) { this.onClause = onClause; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkOrderingTerm.java b/src/sqlancer/spark/ast/SparkOrderingTerm.java index 824801c00..870c8239b 100644 --- a/src/sqlancer/spark/ast/SparkOrderingTerm.java +++ b/src/sqlancer/spark/ast/SparkOrderingTerm.java @@ -7,4 +7,4 @@ public class SparkOrderingTerm extends NewOrderingTerm implemen public SparkOrderingTerm(SparkExpression expr, Ordering ordering) { super(expr, ordering); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkSelect.java b/src/sqlancer/spark/ast/SparkSelect.java index 0986ce0a6..8b59f5513 100644 --- a/src/sqlancer/spark/ast/SparkSelect.java +++ b/src/sqlancer/spark/ast/SparkSelect.java @@ -39,4 +39,4 @@ public String asString() { return SparkToStringVisitor.asString(this); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkTableReference.java b/src/sqlancer/spark/ast/SparkTableReference.java index 92a59ad3d..5bcbb5d03 100644 --- a/src/sqlancer/spark/ast/SparkTableReference.java +++ b/src/sqlancer/spark/ast/SparkTableReference.java @@ -10,4 +10,4 @@ public SparkTableReference(SparkSchema.SparkTable table) { super(table); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java index f1082a655..3dd9d28e2 100644 --- a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java +++ b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java @@ -10,4 +10,4 @@ public SparkUnaryPostfixOperation(SparkExpression expr, Operator op) { super(expr, op); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java index d1bd94ab4..5c1a8e4c6 100644 --- a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java +++ b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java @@ -9,4 +9,4 @@ public SparkUnaryPrefixOperation(SparkExpression expr, Operator op) { super(expr, op); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/gen/SparkExpressionGenerator.java b/src/sqlancer/spark/gen/SparkExpressionGenerator.java index faf8a07f0..3708f314a 100644 --- a/src/sqlancer/spark/gen/SparkExpressionGenerator.java +++ b/src/sqlancer/spark/gen/SparkExpressionGenerator.java @@ -333,4 +333,4 @@ public int getNrArgs() { } } } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java index 29232fdb2..b1755a848 100644 --- a/src/sqlancer/spark/gen/SparkInsertGenerator.java +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -44,4 +44,4 @@ private SQLQueryAdapter generate() { SparkErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, false, false); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java index 68cafdafb..937e52248 100644 --- a/src/sqlancer/spark/gen/SparkTableGenerator.java +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -19,8 +19,10 @@ public class SparkTableGenerator { private enum ColumnConstraints { NOT_NULL, DEFAULT - // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources (Parquet/ORC) - // without specific catalogs (like Delta/Iceberg), so we limit to constraints Spark SQL widely accepts. + // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources + // (Parquet/ORC) + // without specific catalogs (like Delta/Iceberg), so we limit to constraints + // Spark SQL widely accepts. } private final SparkGlobalState globalState; @@ -97,4 +99,4 @@ private void appendColumnConstraint() { throw new AssertionError(constraint); } } -} \ No newline at end of file +} From 22d5c1335d705fb9992ce347728ce90b7423a1cb Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:04:14 +0800 Subject: [PATCH 1180/1284] Fix DataFusion CI: replace blind sleep with build step and readiness poll The DataFusion CI job was flaky because it backgrounded `cargo run` with a fixed 300s sleep that raced against compilation time. Split into explicit build, start, and readiness-check steps so tests only run once the server is actually listening on port 50051. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 34c0e7e5d..d22c46ff9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -242,10 +242,14 @@ jobs: with: toolchain: stable override: true - - name: Run DataFusion Server + - name: Build DataFusion Server run: | cd src/sqlancer/datafusion/server/datafusion_server - cargo run & sleep 300 + cargo build + - name: Start DataFusion Server + run: | + cd src/sqlancer/datafusion/server/datafusion_server + cargo run & - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -254,6 +258,18 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true + - name: Wait for DataFusion Server + run: | + for i in $(seq 1 30); do + if nc -z 127.0.0.1 50051 2>/dev/null; then + echo "DataFusion server is ready" + exit 0 + fi + echo "Waiting for DataFusion server... ($i/30)" + sleep 10 + done + echo "DataFusion server failed to start within 300s" + exit 1 - name: Run Tests run: | DATAFUSION_AVAILABLE=true mvn test -Pdatafusion-tests From 9d26d70264ea85379830ec8a6907321ccbea6be1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:12:16 +0800 Subject: [PATCH 1181/1284] Pin chrono <0.4.40 to fix arrow-arith 52.2.0 build conflict chrono 0.4.40+ added `Datelike::quarter()` which conflicts with `ChronoDateExt::quarter()` in arrow-arith 52.2.0, causing ambiguous method resolution. Pin chrono below 0.4.40 until arrow dependencies are upgraded. Co-Authored-By: Claude Sonnet 4.6 --- src/sqlancer/datafusion/server/datafusion_server/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml index cd8b85e1d..332a88e30 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml +++ b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml @@ -16,7 +16,7 @@ arrow-schema = { version = "52.1.0", default-features = false } arrow-string = { version = "52.1.0", default-features = false } async-trait = "0.1.73" bytes = "1.4" -chrono = { version = "0.4.34", default-features = false } +chrono = { version = ">=0.4.34, <0.4.40", default-features = false } dashmap = "5.5.0" # This version is for SQLancer CI run datafusion = { version = "40.0.0" } From c548689d99bc3d9f5290d2ca28ec5965964c1405 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:32:02 +0800 Subject: [PATCH 1182/1284] Fix DataFusion server memory leak: clear state on database reset The server's DashMaps (contexts, statements, results) were never cleaned up between SQLancer rounds, causing unbounded memory growth that eventually stalled query throughput to 0/s. Co-Authored-By: Claude Sonnet 4.6 --- src/sqlancer/datafusion/server/datafusion_server/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs index 13ec73e96..057c34883 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs +++ b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs @@ -215,6 +215,11 @@ impl FlightSqlService for FlightSqlServiceImpl { let mut ctx_guard = self.ctx.lock().await; // Use `lock()` for async Mutex *ctx_guard = new_ctx; + + // Clear leaked state from previous round + self.statements.clear(); + self.results.clear(); + self.contexts.clear(); } // no authentication actually takes place here // see Ballista implementation for example of basic auth From 5b4f606ad4a9c83ff638e4329b8b53144651f738 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:43:34 +0800 Subject: [PATCH 1183/1284] Fix HSQLDB test: resolve SLF4J/Log4j version conflicts The HSQLDB test failed with NoClassDefFoundError on FrameworkLogger due to incompatible logging dependencies from Hive: - log4j-slf4j-impl (SLF4J 1.x bridge) conflicting with slf4j-api 2.0.6 - log4j-api 2.10.0 and log4j-core 2.18.0 version mismatch Fix by excluding log4j-slf4j-impl from all Hive dependencies, aligning Log4j2 at 2.24.3, and adding log4j-slf4j2-impl (SLF4J 2.x compatible). Also upgrade HSQLDB from 2.7.1 to 2.7.4. Co-Authored-By: Claude Sonnet 4.6 --- pom.xml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0fe88c17d..7c9a1106b 100644 --- a/pom.xml +++ b/pom.xml @@ -360,7 +360,7 @@ org.hsqldb hsqldb - 2.7.1 + 2.7.4 runtime @@ -382,16 +382,49 @@ org.apache.hive hive-jdbc 3.1.2 + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.hive hive-serde 4.0.1 + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.hive hive-cli 4.0.1 + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + org.apache.logging.log4j + log4j-api + 2.24.3 + + + org.apache.logging.log4j + log4j-core + 2.24.3 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.24.3 org.apache.hadoop From f928e7c98038097fcbcbde1cff85cf5d3549240a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 14:41:36 +0800 Subject: [PATCH 1184/1284] Fix CI job names: typos, copy-paste errors, and creation-only labels - CnosDB: fix step name "Set up ClickHouse" -> "Set up CnosDB" - CnosDB: add "creation only" (both NoREC and TLP use --num-queries 0) - HSQLDB: fix typo "HSQLB" -> "HSQLDB" - MySQL: note "CERT creation only" (CERT uses --num-queries 0) - TiDB: note "TLP creation only" (TLP uses --num-queries 0) Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f2e493f9..ed6769c34 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,7 +89,7 @@ jobs: run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test cnosdb: - name: DBMS Tests (CnosDB) + name: DBMS Tests (CnosDB, creation only) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -101,7 +101,7 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up ClickHouse + - name: Set up CnosDB run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest @@ -366,7 +366,7 @@ jobs: run: SPARK_AVAILABLE=true mvn -Dtest=TestSparkTLP test hsqldb: - name: DBMS Tests (HSQLB) + name: DBMS Tests (HSQLDB) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -458,7 +458,7 @@ jobs: MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQueryPlan mysql: - name: DBMS Tests (MySQL) + name: DBMS Tests (MySQL, CERT creation only) runs-on: ubuntu-latest services: mysql: @@ -601,7 +601,7 @@ jobs: mvn -Dtest=TestSQLiteQPG test tidb: - name: DBMS Tests (TiDB) + name: DBMS Tests (TiDB, TLP creation only) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 6a352e88c489f145c617c58f64eb1f4180b9a717 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 23:45:44 +0800 Subject: [PATCH 1185/1284] Remove unnecessary "does not exist" expected error from PostgresReindexGenerator Following the REINDEX syntax fix in PR #1283 (89a48dd2), the "does not exist" expected error is no longer needed. It was only masking the previous bug where multiple index names were concatenated without delimiters (e.g., REINDEX INDEX i0i1i2), causing PostgreSQL to report that the concatenated name does not exist. With the fix now selecting a single valid index, this error cannot legitimately occur since getIndexes() only returns indexes that exist in the schema. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresReindexGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index dc0d2cf34..d22ffe53e 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -58,7 +58,6 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { throw new AssertionError(scope); } errors.add("already contains data"); // FIXME bug report - errors.add("does not exist"); // internal index errors.add("REINDEX is not yet implemented for partitioned indexes"); return new SQLQueryAdapter(sb.toString(), errors); } From ce741c9eaa9f28130af5479a1e95f979a097d391 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 6 Apr 2026 09:53:54 +0800 Subject: [PATCH 1186/1284] Add EXPLAIN with randomized options for Postgres Based on PR #1265, which adds GENERIC_PLAN support for EXPLAIN. This reimplements the feature with the following bugs fixed: - EXPLAIN options were each wrapped in separate parentheses (e.g. EXPLAIN (ANALYZE) (FORMAT JSON) ...) instead of a single comma-separated list (EXPLAIN (ANALYZE, FORMAT JSON) ...) - BUFFERS and TIMING were generated without ANALYZE, which is required - GENERIC_PLAN and ANALYZE were not treated as mutually exclusive Closes #1044 point 3 Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/PostgresProvider.java | 4 ++ .../gen/PostgresExplainGenerator.java | 68 ++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index acd20a184..ec7978216 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -127,6 +127,7 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // + EXPLAIN(PostgresExplainGenerator::create), // CREATE_VIEW(PostgresViewGenerator::create), // CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); @@ -201,6 +202,9 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; + case EXPLAIN: + nrPerformed = r.getInteger(0, 1); + break; default: throw new AssertionError(a); } diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index d3039394b..e4359e5aa 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -1,16 +1,82 @@ package sqlancer.postgres.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTables; +import sqlancer.postgres.ast.PostgresSelect; + public final class PostgresExplainGenerator { private PostgresExplainGenerator() { } - public static String explain(String selectStr) throws Exception { + public static String explain(String selectStr) { StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN (FORMAT JSON) "); sb.append(selectStr); return sb.toString(); } + public static String explainGeneral(String selectStr) { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + + List options = new ArrayList<>(); + boolean analyze = Randomly.getBoolean(); + boolean genericPlan = !analyze && Randomly.getBoolean(); + if (analyze) { + options.add("ANALYZE"); + } + if (genericPlan) { + options.add("GENERIC_PLAN"); + } + if (Randomly.getBoolean()) { + options.add("FORMAT " + Randomly.fromOptions("TEXT", "XML", "JSON", "YAML")); + } + if (Randomly.getBoolean()) { + options.add("VERBOSE"); + } + if (Randomly.getBoolean()) { + options.add("COSTS"); + } + if (analyze && Randomly.getBoolean()) { + options.add("BUFFERS"); + } + if (analyze && Randomly.getBoolean()) { + options.add("TIMING"); + } + if (Randomly.getBoolean()) { + options.add("SUMMARY"); + } + if (!options.isEmpty()) { + sb.append("("); + sb.append(String.join(", ", options)); + sb.append(") "); + } + + sb.append(selectStr); + return sb.toString(); + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) throws Exception { + PostgresSchema.PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + gen.setTablesAndColumns(new PostgresTables(Arrays.asList(table))); + PostgresSelect select = gen.generateSelect(); + select.setFromList(gen.getTableRefs()); + select.setFetchColumns(gen.generateFetchColumns(false)); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PostgresDataType.BOOLEAN)); + } + return new SQLQueryAdapter(explainGeneral(select.asString())); + } + } From c67c7380ee39fa685895eea7839929aad41d44e4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Apr 2026 09:32:27 +0800 Subject: [PATCH 1187/1284] PostgreSQL: Add expected error for PRIMARY KEY on column with DROP NOT NULL When a multi-action ALTER TABLE combines ADD CONSTRAINT ... PRIMARY KEY with ALTER ... DROP NOT NULL, PostgreSQL returns "primary key column is not marked NOT NULL". Add this to expected errors in both ADD_TABLE_CONSTRAINT and ADD_TABLE_CONSTRAINT_USING_INDEX cases. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 5598c3f9f..69b509f60 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -304,6 +304,7 @@ public SQLQueryAdapter generate() { errors.add("multiple primary keys for table"); errors.add("could not create unique index"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("cannot cast type"); errors.add("unsupported PRIMARY KEY constraint with partition key definition"); errors.add("unsupported UNIQUE constraint with partition key definition"); @@ -342,6 +343,7 @@ public SQLQueryAdapter generate() { errors.add("appears twice in unique constraint"); errors.add("appears twice in primary key constraint"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("insufficient columns in PRIMARY KEY constraint definition"); errors.add("which is part of the partition key"); break; From 3db5ffa9c2153bcf5c36ea6278177e20063538b7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Apr 2026 09:34:42 +0800 Subject: [PATCH 1188/1284] PostgreSQL: Add expected error for PRIMARY KEY on column with DROP NOT NULL When a multi-action ALTER TABLE combines ADD CONSTRAINT ... PRIMARY KEY with ALTER ... DROP NOT NULL, PostgreSQL returns "primary key column is not marked NOT NULL". Add this to expected errors in both ADD_TABLE_CONSTRAINT and ADD_TABLE_CONSTRAINT_USING_INDEX cases. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 5598c3f9f..69b509f60 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -304,6 +304,7 @@ public SQLQueryAdapter generate() { errors.add("multiple primary keys for table"); errors.add("could not create unique index"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("cannot cast type"); errors.add("unsupported PRIMARY KEY constraint with partition key definition"); errors.add("unsupported UNIQUE constraint with partition key definition"); @@ -342,6 +343,7 @@ public SQLQueryAdapter generate() { errors.add("appears twice in unique constraint"); errors.add("appears twice in primary key constraint"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("insufficient columns in PRIMARY KEY constraint definition"); errors.add("which is part of the partition key"); break; From d8d90bbfefccd567270bda1de81b3dd0ff00f5ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 8 Apr 2026 00:35:01 +0800 Subject: [PATCH 1189/1284] Remove commented-out code in the DuckDB index generator --- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index cc4114f63..6c50b204d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -40,7 +40,6 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } } sb.append(")"); - // errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("cannot create an index on the rowid"); } From 6d8685090321a11648e2dd2f8538aadb02f937e4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 8 Apr 2026 09:55:17 +0800 Subject: [PATCH 1190/1284] Materialize: Update expected query plan for QPG test The latest Materialize version changed the EXPLAIN OPTIMIZED PLAN output format: the With block now appears before Return, and column name annotations shifted positions. Update the expected string to match. Co-Authored-By: Claude Opus 4.6 --- test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java index f282af42e..4d26bc08e 100644 --- a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java +++ b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java @@ -42,7 +42,7 @@ void testMaterializeQueryPlan() throws Exception { String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); assertEquals( - "Return // { arity: 3 };Union // { arity: 3 };Get l0 // { arity: 3 };Project (#2{c}, #3, #0) // { arity: 3 };Union // { arity: 1 };Negate // { arity: 1 };Project (#2) // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };With;ReadStorage queryplan.public.t1 // { arity: 2 };ReadStorage queryplan.public.t2 // { arity: 1 };;Source queryplan.public.t1;Source queryplan.public.t2;;Target cluster: quickstart;", + "With;ReadStorage queryplan.public.t1 // { arity: 2 };ReadStorage queryplan.public.t2 // { arity: 1 };Return // { arity: 3 };Union // { arity: 3 };Get l0 // { arity: 3 };Project (#2, #3, #0{c}) // { arity: 3 };Union // { arity: 1 };Negate // { arity: 1 };Project (#2{c}) // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };;Source queryplan.public.t1;Source queryplan.public.t2;;Target cluster: quickstart;", queryPlan); } From 9a9b5f345aa26571a815928ff96a24e41ffeee85 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Apr 2026 00:09:46 +0800 Subject: [PATCH 1191/1284] Use AbstractSchema.matchesViewName() for view detection everywhere Replace inlined tableName.startsWith("v") with the existing matchesViewName() helper across all 15 schema classes, ensuring a single source of truth for view name detection. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/clickhouse/ClickHouseSchema.java | 2 +- src/sqlancer/cockroachdb/CockroachDBSchema.java | 2 +- src/sqlancer/databend/DatabendSchema.java | 2 +- src/sqlancer/datafusion/DataFusionSchema.java | 2 +- src/sqlancer/doris/DorisSchema.java | 2 +- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- src/sqlancer/hive/HiveSchema.java | 2 +- src/sqlancer/hsqldb/HSQLDBSchema.java | 2 +- src/sqlancer/postgres/PostgresSchema.java | 2 +- src/sqlancer/presto/PrestoSchema.java | 2 +- src/sqlancer/questdb/QuestDBSchema.java | 2 +- src/sqlancer/spark/SparkSchema.java | 2 +- src/sqlancer/tidb/TiDBSchema.java | 2 +- src/sqlancer/yugabyte/ycql/YCQLSchema.java | 2 +- src/sqlancer/yugabyte/ysql/YSQLSchema.java | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index 97f4534a1..8f8f906ec 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -216,7 +216,7 @@ public static ClickHouseSchema fromConnection(SQLConnection con, String database for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); List indexes = Collections.emptyList(); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); ClickHouseTable t = new ClickHouseTable(tableName, databaseColumns, indexes, isView); for (ClickHouseColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 708afef92..c0eeedd2b 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -292,7 +292,7 @@ public static CockroachDBSchema fromConnection(SQLConnection con, String databas for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); List indexes = getIndexes(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); CockroachDBTable t = new CockroachDBTable(tableName, databaseColumns, indexes, isView); for (CockroachDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index af9a3e0ce..f6bf30757 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -321,7 +321,7 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa List tableNames = getTableNames(con, databaseName); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName, databaseName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); for (DatabendColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/datafusion/DataFusionSchema.java b/src/sqlancer/datafusion/DataFusionSchema.java index b9f00a02e..d02e80c30 100644 --- a/src/sqlancer/datafusion/DataFusionSchema.java +++ b/src/sqlancer/datafusion/DataFusionSchema.java @@ -37,7 +37,7 @@ public static DataFusionSchema fromConnection(SQLConnection con, String database for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); DataFusionTable t = new DataFusionTable(tableName, databaseColumns, isView); for (DataFusionColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/doris/DorisSchema.java b/src/sqlancer/doris/DorisSchema.java index 76697fad7..70a61ee62 100644 --- a/src/sqlancer/doris/DorisSchema.java +++ b/src/sqlancer/doris/DorisSchema.java @@ -569,7 +569,7 @@ public static DorisSchema fromConnection(SQLConnection con, String databaseName) continue; } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); DorisTable t = new DorisTable(tableName, databaseColumns, isView); for (DorisColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 8141220fa..e4b760221 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -231,7 +231,7 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName continue; // TODO: unexpected? } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); List indexes = getIndexes(con, tableName); DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java index 822eea163..8733d5caa 100644 --- a/src/sqlancer/hive/HiveSchema.java +++ b/src/sqlancer/hive/HiveSchema.java @@ -59,7 +59,7 @@ public static HiveSchema fromConnection(SQLConnection con, String databaseName) List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); HiveTable t = new HiveTable(tableName, databaseColumns, isView); for (HiveColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/hsqldb/HSQLDBSchema.java b/src/sqlancer/hsqldb/HSQLDBSchema.java index e1e1cb94b..2d41df83f 100644 --- a/src/sqlancer/hsqldb/HSQLDBSchema.java +++ b/src/sqlancer/hsqldb/HSQLDBSchema.java @@ -29,7 +29,7 @@ public static HSQLDBSchema fromConnection(SQLConnection connection, String datab continue; // TODO: unexpected? } List databaseColumns = getTableColumns(connection, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); HSQLDBSchema.HSQLDBTable t = new HSQLDBSchema.HSQLDBTable(tableName, databaseColumns, isView); for (HSQLDBSchema.HSQLDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 20337fea7..2c4607418 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -250,7 +250,7 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa boolean isPartitioned = "p".equals(rs.getString("relkind")); // TODO: also check insertable // TODO: insert into view? - boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || + boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || // tableTypeStr.contains("LOCAL TEMPORARY") && // !isInsertable; PostgresTable.TableType tableType = getTableType(tableTypeSchema); diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java index 112da7ef2..439615950 100644 --- a/src/sqlancer/presto/PrestoSchema.java +++ b/src/sqlancer/presto/PrestoSchema.java @@ -27,7 +27,7 @@ public static PrestoSchema fromConnection(SQLConnection con, String databaseName List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, databaseName, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); PrestoTable t = new PrestoTable(tableName, databaseColumns, isView); for (PrestoColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index 8253ec82c..55ee01aab 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -268,7 +268,7 @@ public static QuestDBSchema fromConnection(SQLConnection con, String databaseNam continue; // TODO: unexpected? } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); QuestDBTable t = new QuestDBTable(tableName, databaseColumns, isView); for (QuestDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java index 8d988e31e..9b3666916 100644 --- a/src/sqlancer/spark/SparkSchema.java +++ b/src/sqlancer/spark/SparkSchema.java @@ -54,7 +54,7 @@ public static SparkSchema fromConnection(SQLConnection con, String databaseName) List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.toLowerCase().startsWith("v"); + boolean isView = matchesViewName(tableName); SparkTable t = new SparkTable(tableName, databaseColumns, isView); for (SparkColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 32e00504b..f8439c734 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -313,7 +313,7 @@ public static TiDBSchema fromConnection(SQLConnection con, String databaseName) continue; } List indexes = getIndexes(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); TiDBTable t = new TiDBTable(tableName, databaseColumns, indexes, isView); for (TiDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java index 1534b9b56..41762d453 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLSchema.java +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -216,7 +216,7 @@ public static YCQLSchema fromConnection(SQLConnection con, String databaseName) continue; } List databaseColumns = getTableColumns(con, databaseName, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); YCQLTable t = new YCQLTable(tableName, databaseColumns, isView); for (YCQLColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/yugabyte/ysql/YSQLSchema.java b/src/sqlancer/yugabyte/ysql/YSQLSchema.java index 400a34e21..c75322af9 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLSchema.java +++ b/src/sqlancer/yugabyte/ysql/YSQLSchema.java @@ -81,7 +81,7 @@ public static YSQLSchema fromConnection(SQLConnection con, String databaseName) boolean isInsertable = rs.getBoolean("is_insertable_into"); // TODO: also check insertable // TODO: insert into view? - boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || + boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || // tableTypeStr.contains("LOCAL TEMPORARY") && // !isInsertable; YSQLTable.TableType tableType = getTableType(tableTypeSchema); From 5cb7193514b16a3668cb6b3bfdbb17ee8d8e384f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Apr 2026 00:11:55 +0800 Subject: [PATCH 1192/1284] Use getFreeViewName() in view generators instead of inline logic PostgresViewGenerator, MaterializeViewGenerator, and YSQLViewGenerator had their own inline "v" + i++ loops to find free view names. Replace with the existing AbstractSchema.getFreeViewName() which provides the same logic plus randomized start indices for better fuzzing variety. Co-Authored-By: Claude Opus 4.6 --- .../materialize/gen/MaterializeViewGenerator.java | 14 +++----------- .../postgres/gen/PostgresViewGenerator.java | 14 +++----------- .../yugabyte/ysql/gen/YSQLViewGenerator.java | 14 +++----------- 3 files changed, 9 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/materialize/gen/MaterializeViewGenerator.java b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java index c8cd0e93f..e3cb8ff29 100644 --- a/src/sqlancer/materialize/gen/MaterializeViewGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java @@ -30,19 +30,11 @@ public static SQLQueryAdapter create(MaterializeGlobalState globalState) { materialized = false; } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index b0a2a8b9d..10992ece6 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -35,19 +35,11 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { materialized = false; } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java index f70891cc2..3005d49f8 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -28,19 +28,11 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { } } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } From e6a46c5803e5021a6f34d35f9982719e09cc6178 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Apr 2026 09:12:41 +0800 Subject: [PATCH 1193/1284] Fix formatting in PostgresSchema after matchesViewName refactoring Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/PostgresSchema.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 2c4607418..8abd286f0 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -251,8 +251,8 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa // TODO: also check insertable // TODO: insert into view? boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || - // tableTypeStr.contains("LOCAL TEMPORARY") && - // !isInsertable; + // tableTypeStr.contains("LOCAL TEMPORARY") && + // !isInsertable; PostgresTable.TableType tableType = getTableType(tableTypeSchema); List databaseColumns = getTableColumns(con, tableName); List indexes = getIndexes(con, tableName); From 01d848de3ce1681b2fcc6d7393087e03bf606461 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Apr 2026 22:13:08 +0800 Subject: [PATCH 1194/1284] CI: Run reducer tests in Misc Tests job instead of separate job Combine the standalone reducer job into the existing Misc Tests step to reduce CI overhead. The reducer tests are lightweight and don't need their own runner. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed6769c34..d35bf55b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**,!sqlancer.reducer.**' test + run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - name: Set up Python uses: actions/setup-python@v4 with: @@ -711,21 +711,3 @@ jobs: DORIS_AVAILABLE=true mvn -Dtest=TestDorisNoREC test DORIS_AVAILABLE=true mvn -Dtest=TestDorisPQS test DORIS_AVAILABLE=true mvn -Dtest=TestDorisTLP test - - reducer: - name: Reducer Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - - name: Build - run: mvn -B package -DskipTests=true - - name: Run Tests - run: | - mvn -Dtest=TestStatementReducer test From 3cdb5c51ce006d3b72e324977c8c44cd51b46fa6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Apr 2026 22:18:43 +0800 Subject: [PATCH 1195/1284] Pull up isPrimaryKey() and hasPrimaryKey() into abstract base classes Add isPrimaryKey() to AbstractTableColumn (default false) and hasPrimaryKey() to AbstractTable, replacing duplicate implementations in MySQL, OceanBase, and TiDB. Add @Override to all database-specific isPrimaryKey() methods. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/cockroachdb/CockroachDBSchema.java | 1 + src/sqlancer/common/schema/AbstractTable.java | 4 ++++ src/sqlancer/common/schema/AbstractTableColumn.java | 4 ++++ src/sqlancer/databend/DatabendSchema.java | 1 + src/sqlancer/duckdb/DuckDBSchema.java | 1 + src/sqlancer/mariadb/MariaDBSchema.java | 1 + src/sqlancer/mysql/MySQLSchema.java | 5 +---- src/sqlancer/oceanbase/OceanBaseSchema.java | 5 +---- src/sqlancer/presto/PrestoSchema.java | 1 + src/sqlancer/sqlite3/schema/SQLite3Schema.java | 1 + src/sqlancer/tidb/TiDBSchema.java | 5 +---- src/sqlancer/yugabyte/ycql/YCQLSchema.java | 1 + 12 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index c0eeedd2b..cbde577cb 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -182,6 +182,7 @@ public CockroachDBColumn(String name, CockroachDBCompositeDataType columnType, b this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 89d568867..58154681c 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -95,6 +95,10 @@ public boolean isView() { return isView; } + public boolean hasPrimaryKey() { + return columns.stream().anyMatch(c -> c.isPrimaryKey()); + } + public String getFreeColumnName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/common/schema/AbstractTableColumn.java b/src/sqlancer/common/schema/AbstractTableColumn.java index e519bea97..a2f5fb1b3 100644 --- a/src/sqlancer/common/schema/AbstractTableColumn.java +++ b/src/sqlancer/common/schema/AbstractTableColumn.java @@ -12,6 +12,10 @@ public AbstractTableColumn(String name, T table, U type) { this.type = type; } + public boolean isPrimaryKey() { + return false; + } + public String getName() { return name; } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index f6bf30757..89738a1f3 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -148,6 +148,7 @@ public DatabendColumn(String name, DatabendCompositeDataType columnType, boolean this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index e4b760221..857b1e008 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -132,6 +132,7 @@ public DuckDBColumn(String name, DuckDBCompositeDataType columnType, boolean isP this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index 808653b25..7f7656d76 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -51,6 +51,7 @@ public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 0384f34df..c8a30614f 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -76,6 +76,7 @@ public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -194,10 +195,6 @@ public MySQLEngine getEngine() { return engine; } - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - } public static final class MySQLIndex extends TableIndex { diff --git a/src/sqlancer/oceanbase/OceanBaseSchema.java b/src/sqlancer/oceanbase/OceanBaseSchema.java index 7de2457ee..7b5b5954f 100644 --- a/src/sqlancer/oceanbase/OceanBaseSchema.java +++ b/src/sqlancer/oceanbase/OceanBaseSchema.java @@ -81,6 +81,7 @@ public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -195,10 +196,6 @@ public OceanBaseTable(String tableName, List columns, List c.isPrimaryKey()); - } - } public static final class OceanBaseIndex extends TableIndex { diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java index 439615950..2e668969d 100644 --- a/src/sqlancer/presto/PrestoSchema.java +++ b/src/sqlancer/presto/PrestoSchema.java @@ -453,6 +453,7 @@ public PrestoColumn(String name, PrestoCompositeDataType columnType, boolean isP this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 586d7e776..fc97929d3 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -78,6 +78,7 @@ public SQLite3Column(String rowId, SQLite3DataType columnType, boolean isInteger this.generated = generated; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index f8439c734..4ce7306e5 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -170,6 +170,7 @@ public TiDBColumn(String name, TiDBCompositeDataType columnType, boolean isPrima this.hasDefault = hasDefault; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -297,10 +298,6 @@ public TiDBTable(String tableName, List columns, List in super(tableName, columns, indexes, isView); } - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - } public static TiDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java index 41762d453..736247364 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLSchema.java +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -126,6 +126,7 @@ public YCQLColumn(String name, YCQLCompositeDataType columnType, boolean isPrima this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } From d13a1e4fc10956b14d6a580ee4aa99e876a6cd5c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Apr 2026 22:44:09 +0800 Subject: [PATCH 1196/1284] PostgreSQL: expect to_char "EEEE must be the last pattern used" error Random format strings fed to to_char (e.g. via md5(...)) can contain "eeee" followed by other characters, which PostgreSQL rejects with ERROR: "EEEE" must be the last pattern used. Add this to the known to_char expected errors so NoREC and similar oracles do not trip on it. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index e9f27aeb2..eeb160a56 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -122,6 +122,7 @@ private static List getToCharFunctionErrors() { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + errors.add("\"EEEE\" must be the last pattern used"); return errors; } From c9bc432f20d999edd73a9e9c3fe393c22a6ff2ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Apr 2026 23:40:04 +0800 Subject: [PATCH 1197/1284] CI: upgrade Citus from PostgreSQL 15 + Citus 11.1 to PostgreSQL 17 + Citus 13.0 ubuntu-latest is now Ubuntu 24.04 (Noble), which has no Citus packages. Patch citusdata_community.list to use jammy after the curl installer creates it, then re-run apt-get update before installing. Fixes: https://github.com/citusdata/citus/issues/7692 Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d35bf55b0..45055c819 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,9 +59,11 @@ jobs: run: | echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list curl https://install.citusdata.com/community/deb.sh | sudo bash - sudo apt-get -y install postgresql-15-citus-11.1 + sudo sed -i 's/noble/jammy/g' /etc/apt/sources.list.d/citusdata_community.list # https://github.com/citusdata/citus/issues/7692 + sudo apt-get update + sudo apt-get -y install postgresql-17-citus-13.0 sudo chown -R $USER:$USER /var/run/postgresql - export PATH=/usr/lib/postgresql/15/bin:$PATH + export PATH=/usr/lib/postgresql/17/bin:$PATH cd ~ mkdir -p citus/coordinator citus/worker1 citus/worker2 initdb -D citus/coordinator From 9780e5f58715421f989e0885966da30b08838df5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Apr 2026 00:24:20 +0800 Subject: [PATCH 1198/1284] Fix Citus CI: handle reg* OID types in PostgresSchema PostgreSQL 17 + Citus 13.0 system tables expose columns with OID alias types (regnamespace, regrole, regtype, regproc, etc.) that were not handled by getColumnType(), causing an AssertionError. Map them to TEXT since they are textually representable, like the existing regclass entry. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/PostgresSchema.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 8abd286f0..82937557c 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -125,6 +125,12 @@ public static PostgresDataType getColumnType(String typeString) { case "character varying": case "name": case "regclass": + case "regnamespace": + case "regrole": + case "regtype": + case "regproc": + case "regprocedure": + case "regoper": return PostgresDataType.TEXT; case "numeric": return PostgresDataType.DECIMAL; From a2019cc2f2358cbeb769024ffe081d5b98f99e38 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Apr 2026 00:25:27 +0800 Subject: [PATCH 1199/1284] Fix Citus CI: filter citus_* system views and update error strings for Citus 13.0 - CitusSchema.fromConnection now skips all tables/views starting with "citus_" (not just "citus_tables") to handle the citus_schemas view added in Citus 12.0, which was causing unexpected SQL errors when SQLancer tried to DELETE/ALTER it after the regnamespace fix exposed it. - Update columnar table index error string from the old (incorrect) "indexes not supported for columnar tables" to the actual Citus error "unsupported access method for the index on columnar table", and add "BRIN indexes on columnar tables are not supported". - Add "alter table command is currently unsupported" for the Citus restriction on certain ALTER TABLE operations (e.g., multi-subcommand ALTERs, type changes) on distributed tables. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/citus/CitusSchema.java | 4 ++-- src/sqlancer/citus/gen/CitusCommon.java | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index 560191c05..b2550bdce 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -63,8 +63,8 @@ public static CitusSchema fromConnection(SQLConnection con, String databaseName) "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { while (rs.next()) { String tableName = rs.getString("table_name"); - /* citus_tables is a helper view, we don't need to test with it so we let's ignore it */ - if (tableName.equals("citus_tables")) { + /* skip Citus-managed views in the public schema (citus_tables, citus_schemas, etc.) */ + if (tableName.startsWith("citus_")) { continue; } String distributionColumnName = rs.getString("dist_col_name"); diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 7a9f9c659..35aee85fa 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -23,10 +23,10 @@ public static List getCitusErrors() { errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); errors.add("functions used in UPDATE queries on distributed tables must not be VOLATILE"); errors.add("STABLE functions used in UPDATE queries cannot be called with column references"); - errors.add( - "functions used in the WHERE clause of modification queries on distributed tables must not be VOLATILE"); + errors.add("of modification queries on distributed tables must not be VOLATILE"); errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); errors.add("cannot execute ALTER TABLE command involving partition column"); + errors.add("alter table command is currently unsupported"); errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); errors.add("is not a regular, foreign or partitioned table"); errors.add("must be a distributed table or a reference table"); @@ -53,7 +53,8 @@ public static List getCitusErrors() { errors.add("direct joins between distributed and local tables are not supported"); errors.add("unlogged columnar tables are not supported"); errors.add("UPDATE and CTID scans not supported for ColumnarScan"); - errors.add("indexes not supported for columnar tables"); + errors.add("unsupported access method for the index on columnar table"); + errors.add("BRIN indexes on columnar tables are not supported"); errors.add("invalid byte sequence for encoding \"UTF8\": 0x00"); errors.add("columnar_tuple_insert_speculative not implemented"); errors.add("row field count is 1, expected 2"); From aed787157599f706c4d0d705d01b1b2fa7299efa Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Apr 2026 00:25:51 +0800 Subject: [PATCH 1200/1284] Fix Citus CI: handle OID 0 errors for columnar temporary tables Citus 13.0 raises "could not open relation with OID 0" when operating on columnar temporary tables (USING columnar ON COMMIT DROP) during VACUUM, DISCARD TEMPORARY, and some INSERT operations where Citus cannot resolve the relation OID. Collect the error into getColumnarOidErrors() and wire it through Citus-specific generator wrappers: - CitusVacuumGenerator wraps PostgresVacuumGenerator so VACUUM accepts the OID 0 error (the VACUUM action previously used the Postgres generator directly). - CitusDiscardGenerator does the same for DISCARD. - INSERT already routes through CitusInsertGenerator, which picks up the error via addCitusErrors() in getCitusErrors(). Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/citus/CitusProvider.java | 16 +++++++-------- src/sqlancer/citus/gen/CitusCommon.java | 15 ++++++++++++++ .../citus/gen/CitusDiscardGenerator.java | 20 +++++++++++++++++++ .../citus/gen/CitusReindexGenerator.java | 20 +++++++++++++++++++ .../citus/gen/CitusTruncateGenerator.java | 20 +++++++++++++++++++ .../citus/gen/CitusVacuumGenerator.java | 20 +++++++++++++++++++ 6 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/citus/gen/CitusDiscardGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusReindexGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusTruncateGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusVacuumGenerator.java diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 747f72216..a11424f18 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -21,11 +21,15 @@ import sqlancer.citus.gen.CitusAlterTableGenerator; import sqlancer.citus.gen.CitusCommon; import sqlancer.citus.gen.CitusDeleteGenerator; +import sqlancer.citus.gen.CitusDiscardGenerator; import sqlancer.citus.gen.CitusIndexGenerator; import sqlancer.citus.gen.CitusInsertGenerator; +import sqlancer.citus.gen.CitusReindexGenerator; import sqlancer.citus.gen.CitusSetGenerator; import sqlancer.citus.gen.CitusTableGenerator; +import sqlancer.citus.gen.CitusTruncateGenerator; import sqlancer.citus.gen.CitusUpdateGenerator; +import sqlancer.citus.gen.CitusVacuumGenerator; import sqlancer.citus.gen.CitusViewGenerator; import sqlancer.common.DBMSCommon; import sqlancer.common.oracle.CompositeTestOracle; @@ -44,15 +48,11 @@ import sqlancer.postgres.gen.PostgresAnalyzeGenerator; import sqlancer.postgres.gen.PostgresClusterGenerator; import sqlancer.postgres.gen.PostgresCommentGenerator; -import sqlancer.postgres.gen.PostgresDiscardGenerator; import sqlancer.postgres.gen.PostgresDropIndexGenerator; import sqlancer.postgres.gen.PostgresNotifyGenerator; -import sqlancer.postgres.gen.PostgresReindexGenerator; import sqlancer.postgres.gen.PostgresSequenceGenerator; import sqlancer.postgres.gen.PostgresStatisticsGenerator; import sqlancer.postgres.gen.PostgresTransactionGenerator; -import sqlancer.postgres.gen.PostgresTruncateGenerator; -import sqlancer.postgres.gen.PostgresVacuumGenerator; @AutoService(DatabaseProvider.class) public class CitusProvider extends PostgresProvider { @@ -82,13 +82,13 @@ public enum Action implements AbstractAction { CREATE_STATISTICS(PostgresStatisticsGenerator::insert), // DROP_STATISTICS(PostgresStatisticsGenerator::remove), // DELETE(CitusDeleteGenerator::create), // - DISCARD(PostgresDiscardGenerator::create), // + DISCARD(CitusDiscardGenerator::create), // DROP_INDEX(PostgresDropIndexGenerator::create), // INSERT(CitusInsertGenerator::insert), // UPDATE(CitusUpdateGenerator::create), // - TRUNCATE(PostgresTruncateGenerator::create), // - VACUUM(PostgresVacuumGenerator::create), // - REINDEX(PostgresReindexGenerator::create), // + TRUNCATE(CitusTruncateGenerator::create), // + VACUUM(CitusVacuumGenerator::create), // + REINDEX(CitusReindexGenerator::create), // SET(CitusSetGenerator::create), // CREATE_INDEX(CitusIndexGenerator::generate), // SET_CONSTRAINTS((g) -> { diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 35aee85fa..ba8936aaa 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -27,6 +27,7 @@ public static List getCitusErrors() { errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); errors.add("cannot execute ALTER TABLE command involving partition column"); errors.add("alter table command is currently unsupported"); + errors.add("on distributed partitioned tables are not supported"); errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); errors.add("is not a regular, foreign or partitioned table"); errors.add("must be a distributed table or a reference table"); @@ -61,6 +62,7 @@ public static List getCitusErrors() { errors.add("incorrect binary data format"); errors.add("invalid sign in external \"numeric\" value"); errors.add("Foreign keys and AFTER ROW triggers are not supported for columnar tables"); + errors.addAll(getColumnarOidErrors()); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { @@ -88,6 +90,19 @@ public static List getCitusErrors() { return errors; } + /** + * Citus can fail with "could not open relation with OID 0" when operating on columnar temporary tables (e.g., USING + * columnar ON COMMIT DROP), during VACUUM, DISCARD TEMPORARY, or INSERT operations where Citus cannot resolve the + * relation OID. + * + * @return the list of expected error substrings for columnar OID resolution failures. + */ + public static List getColumnarOidErrors() { + List errors = new ArrayList<>(); + errors.add("could not open relation with OID 0"); + return errors; + } + public static void addCitusErrors(ExpectedErrors errors) { errors.addAll(getCitusErrors()); } diff --git a/src/sqlancer/citus/gen/CitusDiscardGenerator.java b/src/sqlancer/citus/gen/CitusDiscardGenerator.java new file mode 100644 index 000000000..f4a1b3240 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusDiscardGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresDiscardGenerator; + +public final class CitusDiscardGenerator { + + private CitusDiscardGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter discardQuery = PostgresDiscardGenerator.create(globalState); + ExpectedErrors errors = discardQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return discardQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusReindexGenerator.java b/src/sqlancer/citus/gen/CitusReindexGenerator.java new file mode 100644 index 000000000..6f37cbe06 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusReindexGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresReindexGenerator; + +public final class CitusReindexGenerator { + + private CitusReindexGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter reindexQuery = PostgresReindexGenerator.create(globalState); + ExpectedErrors errors = reindexQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return reindexQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusTruncateGenerator.java b/src/sqlancer/citus/gen/CitusTruncateGenerator.java new file mode 100644 index 000000000..cf36ce9c2 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusTruncateGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresTruncateGenerator; + +public final class CitusTruncateGenerator { + + private CitusTruncateGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter truncateQuery = PostgresTruncateGenerator.create(globalState); + ExpectedErrors errors = truncateQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return truncateQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusVacuumGenerator.java b/src/sqlancer/citus/gen/CitusVacuumGenerator.java new file mode 100644 index 000000000..ae73dbf82 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusVacuumGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresVacuumGenerator; + +public final class CitusVacuumGenerator { + + private CitusVacuumGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter vacuumQuery = PostgresVacuumGenerator.create(globalState); + ExpectedErrors errors = vacuumQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return vacuumQuery; + } + +} From f88317414049f709632a88efe442d660bc7f7002 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 15 Apr 2026 09:49:49 +0800 Subject: [PATCH 1201/1284] CI: skip JaCoCo report for test runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JaCoCo report goal intermittently fails with EOFException when the forked test JVM is force-killed before the agent can finish flushing jacoco.exec. This has been especially visible on QPG/Materialize and other remote-DBMS jobs where long-lived worker threads continue past test completion. The report isn't uploaded or consumed anywhere in CI — it's generated and discarded — so skip it with -Djacoco.skip=true. Developers can still produce coverage locally by running mvn test without the flag. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 108 ++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45055c819..40306d847 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test + run: mvn -Djacoco.skip=true -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - name: Set up Python uses: actions/setup-python@v4 with: @@ -88,7 +88,7 @@ jobs: psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test - name: Run Tests - run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test + run: CITUS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCitus test cnosdb: name: DBMS Tests (CnosDB, creation only) @@ -110,9 +110,9 @@ jobs: until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done - name: Run Tests run: | - CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test + CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBNoREC test sleep 20 - CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBTLP test + CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBTLP test clickhouse: name: DBMS Tests (ClickHouse) @@ -133,7 +133,7 @@ jobs: docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:24.3.1.2672 until curl -sf http://127.0.0.1:8123/ping 2>/dev/null; do sleep 1; done - name: Run Tests - run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test + run: CLICKHOUSE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test - name: Show fatal errors run: docker exec clickhouse-server grep Fatal /var/log/clickhouse-server/clickhouse-server.log || echo No Fatal Errors found - name: Teardown ClickHouse server @@ -163,9 +163,9 @@ jobs: run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: | - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBNoREC test - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBTLP test - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBCERT test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBNoREC test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBTLP test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBCERT test cockroachdb-qpg: name: QPG Tests (CockroachDB) @@ -188,7 +188,7 @@ jobs: - name: Create SQLancer user run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests - run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test + run: COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBQPG test databend: name: DBMS Tests (Databend) @@ -214,9 +214,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendTLP test - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendNoREC test - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendPQS test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendTLP test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendNoREC test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendPQS test datafusion: name: DBMS Tests (DataFusion) @@ -258,7 +258,7 @@ jobs: exit 1 - name: Run Tests run: | - DATAFUSION_AVAILABLE=true mvn test -Pdatafusion-tests + DATAFUSION_AVAILABLE=true mvn -Djacoco.skip=true test -Pdatafusion-tests duckdb: name: DBMS Tests (DuckDB) @@ -276,8 +276,8 @@ jobs: run: mvn -B package -DskipTests=true - name: DuckDB Tests run: | - mvn -Dtest=TestDuckDBTLP test - mvn -Dtest=TestDuckDBNoREC test + mvn -Djacoco.skip=true -Dtest=TestDuckDBTLP test + mvn -Djacoco.skip=true -Dtest=TestDuckDBNoREC test h2: name: DBMS Tests (H2) @@ -293,7 +293,7 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: mvn -Dtest=TestH2 test + run: mvn -Djacoco.skip=true -Dtest=TestH2 test hive: name: DBMS Tests (Hive) @@ -327,7 +327,7 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: HIVE_AVAILABLE=true mvn -Dtest=TestHiveTLP test + run: HIVE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestHiveTLP test spark: name: DBMS Tests (Spark) @@ -365,7 +365,7 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests - run: SPARK_AVAILABLE=true mvn -Dtest=TestSparkTLP test + run: SPARK_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestSparkTLP test hsqldb: name: DBMS Tests (HSQLDB) @@ -382,8 +382,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - mvn -Dtest=TestHSQLDBNoREC test - mvn -Dtest=TestHSQLDBTLP test + mvn -Djacoco.skip=true -Dtest=TestHSQLDBNoREC test + mvn -Djacoco.skip=true -Dtest=TestHSQLDBTLP test mariadb: name: DBMS Tests (MariaDB) @@ -410,7 +410,7 @@ jobs: - name: Create SQLancer User run: sudo mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests - run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test + run: MARIADB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestMariaDB test materialize: name: DBMS Tests (Materialize) @@ -432,9 +432,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeNoREC - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeTLP - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializePQS + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeNoREC + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeTLP + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializePQS materialize-qpg: name: QPG Tests (Materialize) @@ -456,8 +456,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQPG - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQueryPlan + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeQPG + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeQueryPlan mysql: name: DBMS Tests (MySQL, CERT creation only) @@ -484,10 +484,10 @@ jobs: run: mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLCERT - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLDQE + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLPQS + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLTLP + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLCERT + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLDQE oceanbase: name: DBMS Tests (OceanBase) @@ -509,9 +509,9 @@ jobs: mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | - OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseNoREC - OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBasePQS - OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseTLP + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBaseNoREC + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBasePQS + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBaseTLP postgres: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest @@ -534,10 +534,10 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresPQS test - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresTLP test - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresNoREC test - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresCERT test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresPQS test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresTLP test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresNoREC test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresCERT test presto: name: DBMS Tests (Presto) @@ -560,9 +560,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoNoREC test + PRESTO_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPrestoNoREC test docker restart presto && until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done - PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoTLP test + PRESTO_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPrestoTLP test sqlite: name: DBMS Tests (SQLite) runs-on: ubuntu-latest @@ -579,10 +579,10 @@ jobs: run: mvn -B package -DskipTests=true - name: SQLite Tests run: | - mvn -Dtest=TestSQLitePQS test - mvn -Dtest=TestSQLiteTLP test - mvn -Dtest=TestSQLiteNoREC test - mvn -Dtest=TestSQLiteCODDTest test + mvn -Djacoco.skip=true -Dtest=TestSQLitePQS test + mvn -Djacoco.skip=true -Dtest=TestSQLiteTLP test + mvn -Djacoco.skip=true -Dtest=TestSQLiteNoREC test + mvn -Djacoco.skip=true -Dtest=TestSQLiteCODDTest test sqlite-qpg: name: QPG Tests (SQLite) @@ -600,7 +600,7 @@ jobs: run: mvn -B package -DskipTests=true - name: SQLite Tests for QPG run: | - mvn -Dtest=TestSQLiteQPG test + mvn -Djacoco.skip=true -Dtest=TestSQLiteQPG test tidb: name: DBMS Tests (TiDB, TLP creation only) @@ -624,8 +624,8 @@ jobs: run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: | - TIDB_AVAILABLE=true mvn -Dtest=TestTiDBTLP test - TIDB_AVAILABLE=true mvn -Dtest=TestTiDBCERT test + TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBTLP test + TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBCERT test tidb-qpg: name: QPG Tests (TiDB) @@ -648,7 +648,7 @@ jobs: - name: Create SQLancer user run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests - run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test + run: TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBQPG test yugabyte: name: DBMS Tests (YugabyteDB) @@ -670,10 +670,10 @@ jobs: until pg_isready -h localhost -p 5433; do sleep 1; done - name: Run Tests run: | - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLNoREC test - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLTLP test - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLPQS test - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYCQL test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLNoREC test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLTLP test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLPQS test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYCQL test doris: name: DBMS Tests (Apache Doris) @@ -710,6 +710,6 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - DORIS_AVAILABLE=true mvn -Dtest=TestDorisNoREC test - DORIS_AVAILABLE=true mvn -Dtest=TestDorisPQS test - DORIS_AVAILABLE=true mvn -Dtest=TestDorisTLP test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisNoREC test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisPQS test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisTLP test From 8804b3f1d627426a64db40ca5d221e04d91872b3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 16 Apr 2026 22:38:19 +0800 Subject: [PATCH 1202/1284] Fix YugabyteDB CI: use DROP DATABASE WITH (FORCE) DROP DATABASE could fail with "database is being accessed by other users" when a prior iteration's session had not yet been released, causing tests to exit with -1. Adding WITH (FORCE) terminates lingering sessions as part of the drop. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/yugabyte/ysql/YSQLProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java index e72790344..efcc7ec22 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLProvider.java +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -204,11 +204,11 @@ private void createDatabaseSync(YSQLGlobalState globalState, String entryDatabas Connection con = createConnectionSafely(entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)"); createDatabaseCommand = getCreateDatabaseCommand(globalState); globalState.getState().logStatement(createDatabaseCommand); try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + s.execute("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)"); } try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); From 0c477bbfb553ce4e158f4f49b516e990f7b10b68 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 16 Apr 2026 23:16:43 +0800 Subject: [PATCH 1203/1284] Fix Citus CI: add distribution-column filter error for recursive CTEs as expected error Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/citus/gen/CitusCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index ba8936aaa..58b1b7c16 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -18,6 +18,7 @@ public static List getCitusErrors() { errors.add("cannot perform an INSERT without a partition column value"); errors.add("cannot perform an INSERT with NULL in the partition column"); errors.add("recursive CTEs are not supported in distributed queries"); + errors.add("recursive CTEs are only supported when they contain a filter on the distribution column"); errors.add("could not run distributed query with GROUPING SETS, CUBE, or ROLLUP"); errors.add("Subqueries in HAVING cannot refer to outer query"); errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); From e44eba43304e7ea0d4f65c639fa9a13f06bfc5d9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 17 Apr 2026 23:06:03 +0800 Subject: [PATCH 1204/1284] Fix Databend CI: upgrade to v1.2.896-nightly and handle bug #19738 The old Databend image (v1.2.687-nightly) had a server hang triggered by SELECT DISTINCT with large LIMIT values, which caused PQS tests to fail on every CI run. Upgrade to v1.2.896-nightly (which fixes the hang) and work around https://github.com/databendlabs/databend/issues/19738, where SELECT AVG(constant) over a cross join causes an internal assertion failure (UnwindError with Decimal precision mismatch). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/DatabendBugs.java | 1 + src/sqlancer/databend/DatabendErrors.java | 4 ++++ .../tlp/DatabendQueryPartitioningAggregateTester.java | 11 ++++++++--- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 40306d847..ee4186650 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.687-nightly + image: datafuselabs/databend:v1.2.896-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index dd11512d8..a058ccc32 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -19,6 +19,7 @@ public final class DatabendBugs { public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 + public static boolean bug19738 = true; // https://github.com/databendlabs/databend/issues/19738 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index fdd8a3a69..746a4e848 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -47,6 +47,10 @@ public static List getExpressionErrors() { if (DatabendBugs.bug15568) { errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); } + if (DatabendBugs.bug19738) { + errors.add("UnwindError"); + errors.add("unable to cast `NULL`"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 0ba0d9c86..6d52caea3 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -10,6 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -44,9 +45,13 @@ public DatabendQueryPartitioningAggregateTester(DatabendGlobalState state) { @Override public void check() throws SQLException { super.check(); - DatabendAggregateFunction aggregateFunction = Randomly.fromOptions(DatabendAggregateFunction.MAX, - DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT, - DatabendAggregateFunction.AVG/* , DatabendAggregateFunction.STDDEV_POP */); + List aggregateFunctions = new ArrayList<>(List.of(DatabendAggregateFunction.MAX, + DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT + /* , DatabendAggregateFunction.STDDEV_POP */)); + if (!DatabendBugs.bug19738) { + aggregateFunctions.add(DatabendAggregateFunction.AVG); + } + DatabendAggregateFunction aggregateFunction = Randomly.fromList(aggregateFunctions); DatabendFunctionOperation aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); List fetchColumns = new ArrayList<>(); From d6533d2e4c4453ce8947f3f9420862e5892e0750 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 00:07:58 +0800 Subject: [PATCH 1205/1284] Fix Presto CI: handle bytecode compiler bugs #27608 and #27609 Walk the exception cause chain in ComparatorHelper (consistent with SQLQueryAdapter.checkException) so wrapped errors like the JDBC driver's RuntimeException are matched against expected errors. Add VerifyError (#27608) and Compiler failed (#27609) as expected Presto bugs triggered by complex CASE expressions on Presto 0.297. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/ComparatorHelper.java | 11 ++++++----- src/sqlancer/presto/PrestoBugs.java | 6 ++++++ src/sqlancer/presto/PrestoErrors.java | 7 +++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 5da635de2..cee290924 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -70,11 +70,12 @@ public static List getResultSetFirstColumnAsString(String queryString, E throw e; } - if (e.getMessage() == null) { - throw new AssertionError(queryString, e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); + Throwable current = e; + while (current != null) { + if (current.getMessage() != null && errors.errorIsExpected(current.getMessage())) { + throw new IgnoreMeException(); + } + current = current.getCause(); } throw new AssertionError(queryString, e); } finally { diff --git a/src/sqlancer/presto/PrestoBugs.java b/src/sqlancer/presto/PrestoBugs.java index f5e888df1..b0eb3fe57 100644 --- a/src/sqlancer/presto/PrestoBugs.java +++ b/src/sqlancer/presto/PrestoBugs.java @@ -8,6 +8,12 @@ public final class PrestoBugs { // https://github.com/prestodb/presto/issues/23613 public static boolean bug23613 = true; + // https://github.com/prestodb/presto/issues/27608 + public static boolean bugVerifyError = true; + + // https://github.com/prestodb/presto/issues/27609 + public static boolean bugCompilerFailed = true; + private PrestoBugs() { } diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 2296223d6..dd2931976 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -47,6 +47,13 @@ public static List getExpressionErrors() { } errors.add("Cannot cast java.lang.String to java.util.List"); errors.add("Unexpected subquery expression in logical plan"); + if (PrestoBugs.bugVerifyError) { + errors.add("VerifyError"); + } + if (PrestoBugs.bugCompilerFailed) { + errors.add("Compiler failed"); + errors.add("Error processing class definition"); + } // 9223372036854775808 errors.add("Invalid numeric literal"); From e5ea4affea8271210966c04b89367e1f3d506b30 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 00:23:07 +0800 Subject: [PATCH 1206/1284] Fix Hive CI: skip unary prefix operators to work around negation nullability bug Hive incorrectly evaluates IS NULL for negated expressions involving string concatenation with column references (e.g., -(c || 'x') IS NULL returns false instead of true). The optimizer's nullability inference for GenericUDFOPNegative does not account for runtime NULL from non-null input. Affects Hive 4.0.1 and 4.2.0. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/hive/HiveBugs.java | 18 ++++++++++++++++++ .../hive/gen/HiveExpressionGenerator.java | 9 ++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/hive/HiveBugs.java diff --git a/src/sqlancer/hive/HiveBugs.java b/src/sqlancer/hive/HiveBugs.java new file mode 100644 index 000000000..01b772bb8 --- /dev/null +++ b/src/sqlancer/hive/HiveBugs.java @@ -0,0 +1,18 @@ +package sqlancer.hive; + +// do not make the fields final to avoid warnings +public final class HiveBugs { + + // Incorrect IS NULL evaluation for negation of string concatenation involving column references. + // -(c || 'x') evaluates to NULL at runtime, but IS NULL incorrectly returns false. + // The optimizer's nullability inference for GenericUDFOPNegative does not account for + // runtime conversion failures producing NULL from non-null input. + // Reproduce: CREATE TABLE t(c DOUBLE); INSERT INTO t VALUES(1.0); + // SELECT (-(c || 'x')) IS NULL FROM t; -- returns false, expected true + // Affects: 4.0.1, 4.2.0 + public static boolean bugNegationNullability = true; + + private HiveBugs() { + } + +} diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index 9f3de2514..a31420899 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -5,12 +5,14 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; +import sqlancer.hive.HiveBugs; import sqlancer.hive.HiveGlobalState; import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveDataType; @@ -80,7 +82,12 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr Expression expr = Randomly.fromList(possibleOptions); switch (expr) { case UNARY_PREFIX: - return new HiveUnaryPrefixOperation(generateExpression(depth + 1), HiveUnaryPrefixOperator.getRandom()); + HiveUnaryPrefixOperator prefixOp = HiveUnaryPrefixOperator.getRandom(); + if (HiveBugs.bugNegationNullability + && (prefixOp == HiveUnaryPrefixOperator.MINUS || prefixOp == HiveUnaryPrefixOperator.PLUS)) { + throw new IgnoreMeException(); + } + return new HiveUnaryPrefixOperation(generateExpression(depth + 1), prefixOp); case UNARY_POSTFIX: return new HiveUnaryPostfixOperation(generateExpression(depth + 1), HiveUnaryPostfixOperator.getRandom()); case BINARY_COMPARISON: From 8ee570f530b01890d77e7f101ec34ee47904bef7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 00:44:50 +0800 Subject: [PATCH 1207/1284] Fix OceanBase CI: add "value is out of range" as expected expression error Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/oceanbase/OceanBaseErrors.java | 1 + src/sqlancer/oceanbase/OceanBaseOracleFactory.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index c539ab129..7a5bd8a77 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -15,6 +15,7 @@ public static List getExpressionErrors() { ArrayList errors = new ArrayList<>(); errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) + errors.add("value is out of range"); errors.add("is not valid for CHARACTER SET"); errors.add("The observer or zone is not the master"); errors.add("Incorrect integer value"); diff --git a/src/sqlancer/oceanbase/OceanBaseOracleFactory.java b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java index b7b115f37..b1ab1cb5b 100644 --- a/src/sqlancer/oceanbase/OceanBaseOracleFactory.java +++ b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java @@ -17,7 +17,7 @@ public enum OceanBaseOracleFactory implements OracleFactory create(OceanBaseGlobalState globalState) throws SQLException { OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) - .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).with("value is out of range").build(); + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).build(); return new TLPWhereOracle<>(globalState, gen, expectedErrors); } From 220137f84371f2ebf64c1649f1f741d96c7fb056 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 01:12:55 +0800 Subject: [PATCH 1208/1284] Fix Hive CI: skip additional expression types triggering Hive evaluation bugs Add bug flags for three more Hive bugs: - bugNonBooleanWhereClause: CAST/FLOOR/ROUND/arithmetic silently return 0 rows when used as WHERE predicates instead of erroring - bugInBooleanEvaluation: IN with boolean/IS NULL sub-expressions returns 0 rows for all TLP partitions - bugBetweenMixedTypes: BETWEEN with mixed boolean/numeric types loses rows in TLP partitions Skip CAST, FUNC, BINARY_ARITHMETIC, IN, and BETWEEN expression types in the generator when the corresponding bug flags are enabled. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/hive/HiveBugs.java | 23 +++++++++++++++++++ .../hive/gen/HiveExpressionGenerator.java | 12 +++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/hive/HiveBugs.java b/src/sqlancer/hive/HiveBugs.java index 01b772bb8..43d42ce0b 100644 --- a/src/sqlancer/hive/HiveBugs.java +++ b/src/sqlancer/hive/HiveBugs.java @@ -12,6 +12,29 @@ public final class HiveBugs { // Affects: 4.0.1, 4.2.0 public static boolean bugNegationNullability = true; + // Non-boolean expressions (CAST to non-boolean, FLOOR, ROUND, arithmetic) silently + // return 0 rows for all three TLP partitions when used as WHERE predicates. + // Hive requires BOOLEAN in WHERE but does not error; instead it returns empty results. + // Reproduce: CREATE TABLE t(c INT); INSERT INTO t VALUES(1); + // SELECT * FROM t WHERE FLOOR(1); -- returns 0 rows, expected 1 + // Affects: 4.0.1, 4.2.0 + public static boolean bugNonBooleanWhereClause = true; + + // IN operator with boolean sub-expressions involving IS NULL evaluates incorrectly, + // returning 0 rows for all three TLP partitions. + // Reproduce: CREATE TABLE t(c BOOLEAN); INSERT INTO t VALUES(true),(false); + // SELECT * FROM t WHERE (c != c) IN ((false) IS NULL); -- returns 0, expected 2 + // Affects: 4.0.1, 4.2.0 + public static boolean bugInBooleanEvaluation = true; + + // BETWEEN with mixed boolean/numeric types has incorrect TLP evaluation. + // The IS NULL partition misses rows due to wrong nullability inference. + // Reproduce: CREATE TABLE t(c DOUBLE); INSERT INTO t VALUES(0.5),(1.5); + // SELECT * FROM t WHERE (c NOT IN (true)) NOT BETWEEN 0.01 AND c; + // -- TLP partitions lose rows + // Affects: 4.0.1, 4.2.0 + public static boolean bugBetweenMixedTypes = true; + private HiveBugs() { } diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index a31420899..92154873c 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -77,7 +77,17 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - // TODO: remove some of the possible expression types according to options. + if (HiveBugs.bugNonBooleanWhereClause) { + possibleOptions.remove(Expression.CAST); + possibleOptions.remove(Expression.FUNC); + possibleOptions.remove(Expression.BINARY_ARITHMETIC); + } + if (HiveBugs.bugInBooleanEvaluation) { + possibleOptions.remove(Expression.IN); + } + if (HiveBugs.bugBetweenMixedTypes) { + possibleOptions.remove(Expression.BETWEEN); + } Expression expr = Randomly.fromList(possibleOptions); switch (expr) { From e81ffd1ddbf326f485a79ed9dd28906263fd7cb1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 13:56:04 +0800 Subject: [PATCH 1209/1284] Fix Materialize CI: retry readSchema to handle eventual consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Materialize's information_schema is eventually consistent — tables and columns may not be visible immediately after creation. This caused IndexOutOfBoundsException when generators called getRandomTable() on an empty or incomplete schema. Retry readSchema() until the snapshot has no tables with empty columns, the table count has not regressed, and — after the initial read — the table list is not suspiciously empty. Co-Authored-By: Claude Opus 4.6 --- .../materialize/MaterializeGlobalState.java | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java index 77cbe5d14..8a66a7819 100644 --- a/src/sqlancer/materialize/MaterializeGlobalState.java +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -27,6 +27,8 @@ public class MaterializeGlobalState extends SQLGlobalState functionsAndTypes = new HashMap<>(); private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + private int lastKnownTableCount; + private int readSchemaCallCount; @Override public void setConnection(SQLConnection con) { @@ -266,7 +268,30 @@ public String getRandomTableAccessMethod() { @Override public MaterializeSchema readSchema() throws SQLException { - return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + // Materialize's information_schema is eventually consistent: tables and columns + // may not be visible immediately after creation. Retry until the snapshot is + // consistent. + readSchemaCallCount++; + for (int tries = 0; tries < 30; tries++) { + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() + .anyMatch(t -> t.getColumns().isEmpty()); + boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; + boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); + if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; } public void addFunctionAndType(String functionName, Character functionType) { From c7e8b04ebd169bc914e590b4407417138159799a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 21 Apr 2026 09:15:55 +0800 Subject: [PATCH 1210/1284] Refactor: extract common INSERT generation logic into AbstractInsertGenerator Move shared boilerplate (INSERT INTO, column list, ExpectedErrors field) from 12 concrete InsertGenerator subclasses into AbstractInsertGenerator, reducing ~79 lines of duplicated code. Co-Authored-By: Claude Opus 4.6 --- .../gen/ClickHouseInsertGenerator.java | 10 +--------- .../common/gen/AbstractInsertGenerator.java | 20 ++++++++++++++++++- .../databend/gen/DatabendInsertGenerator.java | 12 +---------- .../gen/DataFusionInsertGenerator.java | 15 +------------- .../doris/gen/DorisInsertGenerator.java | 11 +--------- .../duckdb/gen/DuckDBInsertGenerator.java | 11 +--------- src/sqlancer/h2/H2InsertGenerator.java | 6 +----- .../hive/gen/HiveInsertGenerator.java | 2 -- .../hsqldb/gen/HSQLDBInsertGenerator.java | 12 +---------- .../presto/gen/PrestoInsertGenerator.java | 11 +--------- .../questdb/gen/QuestDBInsertGenerator.java | 13 +----------- .../spark/gen/SparkInsertGenerator.java | 2 -- .../ycql/gen/YCQLInsertGenerator.java | 12 +---------- 13 files changed, 29 insertions(+), 108 deletions(-) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 9286185a7..593dec7c6 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -11,13 +11,11 @@ import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; public class ClickHouseInsertGenerator extends AbstractInsertGenerator { private final ClickHouseGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final ClickHouseExpressionGenerator gen; public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { @@ -42,13 +40,7 @@ private SQLQueryAdapter get() { columns = table.getRandomNonEmptyColumnSubset().stream().filter(c -> !c.isAlias() && !c.isMaterialized()) .collect(Collectors.toList()); } - sb.append("INSERT INTO "); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/common/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java index 1a0b2a997..7426656b8 100644 --- a/src/sqlancer/common/gen/AbstractInsertGenerator.java +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -1,12 +1,30 @@ package sqlancer.common.gen; import java.util.List; +import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; -public abstract class AbstractInsertGenerator { +public abstract class AbstractInsertGenerator> { protected StringBuilder sb = new StringBuilder(); + protected ExpectedErrors errors = new ExpectedErrors(); + + protected void appendColumnList(List columns) { + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + } + + protected void buildInsertInto(String tableName, List columns) { + sb.append("INSERT INTO "); + sb.append(tableName); + appendColumnList(columns); + sb.append(" VALUES "); + insertColumns(columns); + } protected void insertColumns(List columns) { for (int nrRows = 0; nrRows < Randomly.smallNumber() + 1; nrRows++) { diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 340e494ef..6a3a4c51f 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -1,12 +1,9 @@ package sqlancer.databend.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -16,7 +13,6 @@ public class DatabendInsertGenerator extends AbstractInsertGenerator { private final DatabendGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DatabendInsertGenerator(DatabendGlobalState globalState) { this.globalState = globalState; @@ -27,15 +23,9 @@ public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); DatabendErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java index 1ee00dd50..43a340731 100644 --- a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.datafusion.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; @@ -15,7 +13,6 @@ public class DataFusionInsertGenerator extends AbstractInsertGenerator { private final DataFusionGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DataFusionInsertGenerator(DataFusionGlobalState globalState) { this.globalState = globalState; @@ -26,21 +23,11 @@ public static SQLQueryAdapter getQuery(DataFusionGlobalState globalState, DataFu } private SQLQueryAdapter generate(DataFusionTable targetTable) { - // `sb` is a global `StringBuilder` for current insert query - sb.append("INSERT INTO "); - if (targetTable.getColumns().isEmpty()) { throw new IgnoreMeException(); } List columns = targetTable.getRandomNonEmptyColumnSubset(); - - sb.append(targetTable.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); // will finally call `insertValue()` to generate random value - + buildInsertInto(targetTable.getName(), columns); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java index 50dc5cdec..9407ebf43 100644 --- a/src/sqlancer/doris/gen/DorisInsertGenerator.java +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.doris.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; @@ -16,7 +14,6 @@ public class DorisInsertGenerator extends AbstractInsertGenerator { private final DorisGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DorisInsertGenerator(DorisGlobalState globalState) { this.globalState = globalState; @@ -27,15 +24,9 @@ public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyInsertColumns(); - sb.append(table.getName()); - sb.append(" ("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); DorisErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 6793d2b51..65a9c222f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.duckdb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -16,7 +14,6 @@ public class DuckDBInsertGenerator extends AbstractInsertGenerator { private final DuckDBGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DuckDBInsertGenerator(DuckDBGlobalState globalState) { this.globalState = globalState; @@ -27,15 +24,9 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); DuckDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index c4e559b0e..be715315d 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; @@ -14,7 +13,6 @@ public class H2InsertGenerator extends AbstractInsertGenerator { private final H2GlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final H2ExpressionGenerator gen; public H2InsertGenerator(H2GlobalState globalState) { @@ -39,9 +37,7 @@ private SQLQueryAdapter generate() { H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); + appendColumnList(columns); if (mergeInto && Randomly.getBoolean()) { sb.append(" KEY("); sb.append(table.getRandomNonEmptyColumnSubset().stream().map(c -> c.getName()) diff --git a/src/sqlancer/hive/gen/HiveInsertGenerator.java b/src/sqlancer/hive/gen/HiveInsertGenerator.java index 963fafbce..cd0e11df7 100644 --- a/src/sqlancer/hive/gen/HiveInsertGenerator.java +++ b/src/sqlancer/hive/gen/HiveInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hive.HiveErrors; import sqlancer.hive.HiveGlobalState; @@ -14,7 +13,6 @@ public class HiveInsertGenerator extends AbstractInsertGenerator { private final HiveGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final HiveExpressionGenerator gen; public HiveInsertGenerator(HiveGlobalState globalState) { diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index 1cc132190..16020c508 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -1,10 +1,8 @@ package sqlancer.hsqldb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; @@ -14,7 +12,6 @@ public class HSQLDBInsertGenerator extends AbstractInsertGenerator { private final HSQLDBProvider.HSQLDBGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public HSQLDBInsertGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { this.globalState = globalState; @@ -25,16 +22,9 @@ public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalSt } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); - // HSQLDBErrors.addInsertErrors(errors); + buildInsertInto(table.getName(), columns); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index 072a22ae0..e9fa06f7b 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -1,10 +1,8 @@ package sqlancer.presto.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; @@ -26,16 +24,9 @@ public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); - ExpectedErrors errors = new ExpectedErrors(); + buildInsertInto(table.getName(), columns); PrestoErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, false, false); } diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java index 754a0d2e5..e3a4dc35d 100644 --- a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -1,12 +1,9 @@ package sqlancer.questdb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; import sqlancer.questdb.QuestDBErrors; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; @@ -17,22 +14,14 @@ public class QuestDBInsertGenerator extends AbstractInsertGenerator columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); QuestDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java index b1755a848..c1404f509 100644 --- a/src/sqlancer/spark/gen/SparkInsertGenerator.java +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.spark.SparkErrors; import sqlancer.spark.SparkGlobalState; @@ -14,7 +13,6 @@ public class SparkInsertGenerator extends AbstractInsertGenerator { private final SparkGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final SparkExpressionGenerator gen; public SparkInsertGenerator(SparkGlobalState globalState) { diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java index a1159d310..d53c70d29 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -1,12 +1,9 @@ package sqlancer.yugabyte.ycql.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ycql.YCQLErrors; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; @@ -16,7 +13,6 @@ public class YCQLInsertGenerator extends AbstractInsertGenerator { private final YCQLGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public YCQLInsertGenerator(YCQLGlobalState globalState) { this.globalState = globalState; @@ -27,15 +23,9 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getColumns(); - sb.append(globalState.getDatabaseName()).append(".").append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(globalState.getDatabaseName() + "." + table.getName(), columns); errors.add("Invalid Arguments"); errors.add("Null Argument for Primary Key"); From 38d14b423a31a46adb113c938fbce777854ac968 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 21 Apr 2026 10:52:45 +0800 Subject: [PATCH 1211/1284] Unify INSERT/UPDATE/DELETE generators under AbstractGenerator hierarchy Introduce AbstractDeleteGenerator extending AbstractGenerator, and make AbstractInsertGenerator and AbstractUpdateGenerator also extend it, eliminating duplicate sb/errors field declarations across all generator classes. Convert all 15 DELETE generators (CockroachDB, Databend, Doris, DuckDB, H2, MariaDB, Materialize, MySQL, OceanBase, Postgres, Presto, SQLite3, TiDB, YCQL, YSQL) from static-utility classes to the instance-based buildStatement() pattern. Add canonicalizeString field to AbstractGenerator (defaulting to true) so generators like Presto/Hive/Spark can opt out of semicolon canonicalization. Co-Authored-By: Claude Sonnet 4.6 --- .../gen/ClickHouseInsertGenerator.java | 9 +++--- .../gen/CockroachDBDeleteGenerator.java | 17 ++++++---- .../gen/CockroachDBIndexGenerator.java | 2 +- .../gen/CockroachDBTableGenerator.java | 2 +- .../gen/CockroachDBUpdateGenerator.java | 6 ++-- .../common/gen/AbstractDeleteGenerator.java | 5 +++ .../common/gen/AbstractGenerator.java | 5 +-- .../common/gen/AbstractInsertGenerator.java | 6 +--- .../common/gen/AbstractUpdateGenerator.java | 6 +--- .../databend/gen/DatabendDeleteGenerator.java | 18 +++++++---- .../databend/gen/DatabendInsertGenerator.java | 6 ++-- .../gen/DataFusionInsertGenerator.java | 10 +++--- .../doris/gen/DorisDeleteGenerator.java | 18 +++++++---- .../doris/gen/DorisInsertGenerator.java | 8 ++--- .../doris/gen/DorisUpdateGenerator.java | 6 ++-- .../duckdb/gen/DuckDBDeleteGenerator.java | 18 +++++++---- .../duckdb/gen/DuckDBInsertGenerator.java | 6 ++-- .../duckdb/gen/DuckDBUpdateGenerator.java | 6 ++-- src/sqlancer/h2/H2DeleteGenerator.java | 18 +++++++---- src/sqlancer/h2/H2InsertGenerator.java | 6 ++-- src/sqlancer/h2/H2UpdateGenerator.java | 6 ++-- .../hive/gen/HiveInsertGenerator.java | 7 ++-- .../hsqldb/gen/HSQLDBInsertGenerator.java | 6 ++-- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 6 ++-- .../mariadb/gen/MariaDBDeleteGenerator.java | 25 +++++++++------ .../gen/MaterializeDeleteGenerator.java | 18 +++++++---- .../gen/MaterializeUpdateGenerator.java | 8 ++--- .../mysql/gen/MySQLDeleteGenerator.java | 12 +++---- .../mysql/gen/MySQLUpdateGenerator.java | 10 +++--- .../gen/OceanBaseDeleteGenerator.java | 12 +++---- .../gen/OceanBaseUpdateGenerator.java | 7 ++-- .../postgres/gen/PostgresDeleteGenerator.java | 18 +++++++---- .../postgres/gen/PostgresUpdateGenerator.java | 8 ++--- .../presto/gen/PrestoDeleteGenerator.java | 19 +++++++---- .../presto/gen/PrestoInsertGenerator.java | 7 ++-- .../presto/gen/PrestoUpdateGenerator.java | 7 ++-- .../questdb/gen/QuestDBInsertGenerator.java | 12 +++---- .../spark/gen/SparkInsertGenerator.java | 7 ++-- .../gen/dml/SQLite3DeleteGenerator.java | 32 ++++++++++++------- .../gen/dml/SQLite3UpdateGenerator.java | 15 +++++---- .../tidb/gen/TiDBDeleteGenerator.java | 23 +++++++------ .../tidb/gen/TiDBUpdateGenerator.java | 10 +++--- .../ycql/gen/YCQLDeleteGenerator.java | 19 +++++++---- .../ycql/gen/YCQLInsertGenerator.java | 6 ++-- .../ycql/gen/YCQLUpdateGenerator.java | 6 ++-- .../ysql/gen/YSQLDeleteGenerator.java | 18 +++++++---- .../ysql/gen/YSQLUpdateGenerator.java | 8 ++--- 47 files changed, 291 insertions(+), 219 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractDeleteGenerator.java diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 593dec7c6..3951f4b55 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -1,6 +1,5 @@ package sqlancer.clickhouse.gen; -import java.sql.SQLException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -29,11 +28,12 @@ public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { ClickHouseErrors.addExpectedExpressionErrors(errors); } - public static SQLQueryAdapter getQuery(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseInsertGenerator(globalState).get(); + public static SQLQueryAdapter getQuery(ClickHouseGlobalState globalState) { + return new ClickHouseInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter get() { + @Override + public void buildStatement() { ClickHouseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = Collections.emptyList(); while (columns.isEmpty()) { @@ -41,7 +41,6 @@ private SQLQueryAdapter get() { .collect(Collectors.toList()); } buildInsertInto(table.getName(), columns); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index 23fbae389..f0048d278 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -6,17 +6,23 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; -public final class CockroachDBDeleteGenerator { +public final class CockroachDBDeleteGenerator extends AbstractDeleteGenerator { - private CockroachDBDeleteGenerator() { + private final CockroachDBGlobalState globalState; + + private CockroachDBDeleteGenerator(CockroachDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + return new CockroachDBDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append("DELETE FROM "); sb.append(table.getName()); @@ -30,7 +36,6 @@ public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { } errors.add("foreign key violation"); CockroachDBErrors.addTransactionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 4db3588ac..da6d3c609 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -21,7 +21,7 @@ public static SQLQueryAdapter create(CockroachDBGlobalState s) { if (s.getSchema().getIndexCount() >= s.getDbmsSpecificOptions().maxNumIndexes) { throw new IgnoreMeException(); } - return new CockroachDBIndexGenerator(s).getQuery(); + return new CockroachDBIndexGenerator(s).getStatement(); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index c8bfa3c6a..8678aff4d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -32,7 +32,7 @@ public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { throw new IgnoreMeException(); } - return new CockroachDBTableGenerator(globalState).getQuery(); + return new CockroachDBTableGenerator(globalState).getStatement(); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index 8dcd605d7..b367f5c59 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -22,10 +22,11 @@ private CockroachDBUpdateGenerator(CockroachDBGlobalState globalState) { } public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { - return new CockroachDBUpdateGenerator(globalState).generate(); + return new CockroachDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); @@ -51,7 +52,6 @@ private SQLQueryAdapter generate() { errors.add("cannot write directly to computed column"); CockroachDBErrors.addExpressionErrors(errors); CockroachDBErrors.addTransactionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java new file mode 100644 index 000000000..47df974e6 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -0,0 +1,5 @@ +package sqlancer.common.gen; + +public abstract class AbstractDeleteGenerator extends AbstractGenerator { + +} diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java index c478610c4..5d13fc746 100644 --- a/src/sqlancer/common/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -8,10 +8,11 @@ public abstract class AbstractGenerator { protected final ExpectedErrors errors = new ExpectedErrors(); protected final StringBuilder sb = new StringBuilder(); protected boolean canAffectSchema; + protected boolean canonicalizeString = true; - public SQLQueryAdapter getQuery() { + public SQLQueryAdapter getStatement() { buildStatement(); - return new SQLQueryAdapter(sb.toString(), errors, canAffectSchema); + return new SQLQueryAdapter(sb.toString(), errors, canAffectSchema, canonicalizeString); } public abstract void buildStatement(); diff --git a/src/sqlancer/common/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java index 7426656b8..1a1b36b67 100644 --- a/src/sqlancer/common/gen/AbstractInsertGenerator.java +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -4,13 +4,9 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; -public abstract class AbstractInsertGenerator> { - - protected StringBuilder sb = new StringBuilder(); - protected ExpectedErrors errors = new ExpectedErrors(); +public abstract class AbstractInsertGenerator> extends AbstractGenerator { protected void appendColumnList(List columns) { sb.append("("); diff --git a/src/sqlancer/common/gen/AbstractUpdateGenerator.java b/src/sqlancer/common/gen/AbstractUpdateGenerator.java index f130c15a5..52b716bff 100644 --- a/src/sqlancer/common/gen/AbstractUpdateGenerator.java +++ b/src/sqlancer/common/gen/AbstractUpdateGenerator.java @@ -2,13 +2,9 @@ import java.util.List; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; -public abstract class AbstractUpdateGenerator> { - - protected final ExpectedErrors errors = new ExpectedErrors(); - protected StringBuilder sb = new StringBuilder(); +public abstract class AbstractUpdateGenerator> extends AbstractGenerator { protected void updateColumns(List columns) { for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index 256030409..5adabf175 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -1,21 +1,28 @@ package sqlancer.databend.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; -public final class DatabendDeleteGenerator { +public final class DatabendDeleteGenerator extends AbstractDeleteGenerator { - private DatabendDeleteGenerator() { + private final DatabendGlobalState globalState; + + private DatabendDeleteGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(DatabendGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new DatabendDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); @@ -23,7 +30,6 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN))); DatabendErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 6a3a4c51f..13ee80e95 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -19,15 +19,15 @@ public DatabendInsertGenerator(DatabendGlobalState globalState) { } public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - return new DatabendInsertGenerator(globalState).generate(); + return new DatabendInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); DatabendErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java index 43a340731..36a178791 100644 --- a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java @@ -13,22 +13,24 @@ public class DataFusionInsertGenerator extends AbstractInsertGenerator { private final DataFusionGlobalState globalState; + private final DataFusionTable targetTable; - public DataFusionInsertGenerator(DataFusionGlobalState globalState) { + public DataFusionInsertGenerator(DataFusionGlobalState globalState, DataFusionTable targetTable) { this.globalState = globalState; + this.targetTable = targetTable; } public static SQLQueryAdapter getQuery(DataFusionGlobalState globalState, DataFusionTable targetTable) { - return new DataFusionInsertGenerator(globalState).generate(targetTable); + return new DataFusionInsertGenerator(globalState, targetTable).getStatement(); } - private SQLQueryAdapter generate(DataFusionTable targetTable) { + @Override + public void buildStatement() { if (targetTable.getColumns().isEmpty()) { throw new IgnoreMeException(); } List columns = targetTable.getRandomNonEmptyColumnSubset(); buildInsertInto(targetTable.getName(), columns); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/doris/gen/DorisDeleteGenerator.java b/src/sqlancer/doris/gen/DorisDeleteGenerator.java index 27f369aec..66deaea8f 100644 --- a/src/sqlancer/doris/gen/DorisDeleteGenerator.java +++ b/src/sqlancer/doris/gen/DorisDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.doris.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; @@ -9,14 +9,21 @@ import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.visitor.DorisToStringVisitor; -public final class DorisDeleteGenerator { +public final class DorisDeleteGenerator extends AbstractDeleteGenerator { - private DorisDeleteGenerator() { + private final DorisGlobalState globalState; + + private DorisDeleteGenerator(DorisGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(DorisGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new DorisDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -25,7 +32,6 @@ public static SQLQueryAdapter generate(DorisGlobalState globalState) { .setColumns(table.getColumns()).generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java index 9407ebf43..e05bc85fd 100644 --- a/src/sqlancer/doris/gen/DorisInsertGenerator.java +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -20,15 +20,15 @@ public DorisInsertGenerator(DorisGlobalState globalState) { } public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { - return new DorisInsertGenerator(globalState).generate(); + return new DorisInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyInsertColumns(); + List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); DorisErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java index 906173921..e0db2b7b4 100644 --- a/src/sqlancer/doris/gen/DorisUpdateGenerator.java +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -23,10 +23,11 @@ private DorisUpdateGenerator(DorisGlobalState globalState) { } public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { - return new DorisUpdateGenerator(globalState).generate(); + return new DorisUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new DorisNewExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -37,7 +38,6 @@ private SQLQueryAdapter generate() { sb.append(" WHERE "); sb.append(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index 5fadce30a..42695a9f6 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -1,21 +1,28 @@ package sqlancer.duckdb.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBToStringVisitor; -public final class DuckDBDeleteGenerator { +public final class DuckDBDeleteGenerator extends AbstractDeleteGenerator { - private DuckDBDeleteGenerator() { + private final DuckDBGlobalState globalState; + + private DuckDBDeleteGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new DuckDBDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -24,7 +31,6 @@ public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } DuckDBErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 65a9c222f..e8b122a51 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -20,15 +20,15 @@ public DuckDBInsertGenerator(DuckDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - return new DuckDBInsertGenerator(globalState).generate(); + return new DuckDBInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); buildInsertInto(table.getName(), columns); DuckDBErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index b4ffd0140..8e2ddd047 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -22,10 +22,11 @@ private DuckDBUpdateGenerator(DuckDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - return new DuckDBUpdateGenerator(globalState).generate(); + return new DuckDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -34,7 +35,6 @@ private SQLQueryAdapter generate() { sb.append(" SET "); updateColumns(columns); DuckDBErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java index 58afdcb48..dd0fdff34 100644 --- a/src/sqlancer/h2/H2DeleteGenerator.java +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -1,19 +1,26 @@ package sqlancer.h2; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; -public final class H2DeleteGenerator { +public final class H2DeleteGenerator extends AbstractDeleteGenerator { - private H2DeleteGenerator() { + private final H2GlobalState globalState; + + private H2DeleteGenerator(H2GlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new H2DeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -27,7 +34,6 @@ public static SQLQueryAdapter getQuery(H2GlobalState globalState) { } H2Errors.addExpressionErrors(errors); H2Errors.addDeleteErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index be715315d..3bd7552d8 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -21,10 +21,11 @@ public H2InsertGenerator(H2GlobalState globalState) { } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - return new H2InsertGenerator(globalState).generate(); + return new H2InsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { boolean mergeInto = false; // Randomly.getBooleanWithRatherLowProbability(); if (mergeInto) { sb.append("MERGE INTO "); @@ -48,7 +49,6 @@ private SQLQueryAdapter generate() { insertColumns(columns); H2Errors.addInsertErrors(errors); H2Errors.addExpressionErrors(errors); // generated columns - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index 158621409..fe63c7a4e 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -19,10 +19,11 @@ private H2UpdateGenerator(H2GlobalState globalState) { } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - return new H2UpdateGenerator(globalState).generate(); + return new H2UpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new H2ExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -37,7 +38,6 @@ private SQLQueryAdapter generate() { sb.append(H2ToStringVisitor.asString(gen.generateExpression())); } H2Errors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/hive/gen/HiveInsertGenerator.java b/src/sqlancer/hive/gen/HiveInsertGenerator.java index cd0e11df7..8c23e4456 100644 --- a/src/sqlancer/hive/gen/HiveInsertGenerator.java +++ b/src/sqlancer/hive/gen/HiveInsertGenerator.java @@ -18,10 +18,11 @@ public class HiveInsertGenerator extends AbstractInsertGenerator { public HiveInsertGenerator(HiveGlobalState globalState) { this.globalState = globalState; this.gen = new HiveExpressionGenerator(globalState); + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(HiveGlobalState globalState) { - return new HiveInsertGenerator(globalState).generate(); + return new HiveInsertGenerator(globalState).getStatement(); } @Override @@ -29,7 +30,8 @@ protected void insertValue(HiveColumn column) { sb.append(HiveToStringVisitor.asString(gen.generateConstant())); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { // Inserting values into tables from SQL. sb.append("INSERT INTO "); HiveTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); @@ -45,6 +47,5 @@ private SQLQueryAdapter generate() { insertColumns(columns); HiveErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index 16020c508..00d99be38 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -18,14 +18,14 @@ public HSQLDBInsertGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { - return new HSQLDBInsertGenerator(globalState).generate(); + return new HSQLDBInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index e639e21b3..54380214f 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -24,10 +24,11 @@ private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { - return new HSQLDBUpdateGenerator(globalState).generate(); + return new HSQLDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -42,7 +43,6 @@ private SQLQueryAdapter generate() { errors.add("data type of expression is not boolean"); HSQLDBErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index 6d85eb891..2992f569d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -3,7 +3,7 @@ import java.util.Collections; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTables; import sqlancer.mariadb.MariaDBSchema; @@ -11,12 +11,22 @@ import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.ast.MariaDBVisitor; -public final class MariaDBDeleteGenerator { +public final class MariaDBDeleteGenerator extends AbstractDeleteGenerator { - private MariaDBDeleteGenerator() { + private final MariaDBSchema schema; + private final Randomly r; + + private MariaDBDeleteGenerator(MariaDBSchema schema, Randomly r) { + this.schema = schema; + this.r = r; } public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { + return new MariaDBDeleteGenerator(schema, r).getStatement(); + } + + @Override + public void buildStatement() { MariaDBTable table = schema.getRandomTable(); MariaDBExpressionGenerator expressionGenerator = new MariaDBExpressionGenerator(r); @@ -25,15 +35,13 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { Collections.singletonList(table)); expressionGenerator.setTablesAndColumns(tablesAndColumns); - ExpectedErrors errors = new ExpectedErrors(); - errors.add("foreign key constraint fails"); errors.add("cannot delete or update a parent row"); errors.add("Data truncated"); errors.add("Division by 0"); errors.add("Incorrect value"); - StringBuilder sb = new StringBuilder("DELETE"); + sb.append("DELETE"); if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" LOW_PRIORITY"); @@ -81,13 +89,10 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { } } - String query = sb.toString(); - if (query.contains("RLIKE") || query.contains("REGEXP")) { + if (sb.toString().contains("RLIKE") || sb.toString().contains("REGEXP")) { errors.add("Regex error"); errors.add("quantifier does not follow a repeatable item"); errors.add("Got error"); } - - return new SQLQueryAdapter(query, errors); } } diff --git a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java index ded3e53f4..2aceddcc9 100644 --- a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java @@ -1,25 +1,32 @@ package sqlancer.materialize.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.materialize.MaterializeGlobalState; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; import sqlancer.materialize.MaterializeSchema.MaterializeTable; import sqlancer.materialize.MaterializeVisitor; -public final class MaterializeDeleteGenerator { +public final class MaterializeDeleteGenerator extends AbstractDeleteGenerator { - private MaterializeDeleteGenerator() { + private final MaterializeGlobalState globalState; + + private MaterializeDeleteGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + return new MaterializeDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { MaterializeTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append("DELETE FROM"); sb.append(" "); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -32,7 +39,6 @@ public static SQLQueryAdapter create(MaterializeGlobalState globalState) { errors.add("does not support casting"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java index 1b7e69208..7d338c027 100644 --- a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java @@ -20,6 +20,7 @@ public final class MaterializeUpdateGenerator extends AbstractUpdateGenerator t.isInsertable()); List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); @@ -55,8 +57,6 @@ private SQLQueryAdapter generate() { randomTable.getColumns(), MaterializeDataType.BOOLEAN); sb.append(MaterializeVisitor.asString(where)); } - - return new SQLQueryAdapter(sb.toString(), errors, true); } @Override diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index f12c23a72..048ef335f 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -3,16 +3,15 @@ import java.util.Arrays; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; -public class MySQLDeleteGenerator { +public class MySQLDeleteGenerator extends AbstractDeleteGenerator { - private final StringBuilder sb = new StringBuilder(); private final MySQLGlobalState globalState; public MySQLDeleteGenerator(MySQLGlobalState globalState) { @@ -20,13 +19,13 @@ public MySQLDeleteGenerator(MySQLGlobalState globalState) { } public static SQLQueryAdapter delete(MySQLGlobalState globalState) { - return new MySQLDeleteGenerator(globalState).generate(); + return new MySQLDeleteGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { MySQLTable randomTable = globalState.getSchema().getRandomTable(); MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()); - ExpectedErrors errors = new ExpectedErrors(); sb.append("DELETE"); if (Randomly.getBoolean()) { sb.append(" LOW_PRIORITY"); @@ -51,7 +50,6 @@ private SQLQueryAdapter generate() { */, "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index")); // TODO: support ORDER BY - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java index 55ba3dd45..1aca84580 100644 --- a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java @@ -1,6 +1,5 @@ package sqlancer.mysql.gen; -import java.sql.SQLException; import java.util.List; import sqlancer.Randomly; @@ -21,11 +20,12 @@ public MySQLUpdateGenerator(MySQLGlobalState globalState) { this.globalState = globalState; } - public static SQLQueryAdapter create(MySQLGlobalState globalState) throws SQLException { - return new MySQLUpdateGenerator(globalState).generate(); + public static SQLQueryAdapter create(MySQLGlobalState globalState) { + return new MySQLUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() throws SQLException { + @Override + public void buildStatement() { MySQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new MySQLExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -40,8 +40,6 @@ private SQLQueryAdapter generate() throws SQLException { } MySQLErrors.addInsertUpdateErrors(errors); errors.add("doesn't have this option"); - - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index ea1cb36e8..c46ace304 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -3,16 +3,15 @@ import java.util.Arrays; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; import sqlancer.oceanbase.OceanBaseVisitor; -public class OceanBaseDeleteGenerator { +public class OceanBaseDeleteGenerator extends AbstractDeleteGenerator { - private final StringBuilder sb = new StringBuilder(); private final OceanBaseGlobalState globalState; private final Randomly r; @@ -22,14 +21,14 @@ public OceanBaseDeleteGenerator(OceanBaseGlobalState globalState) { } public static SQLQueryAdapter delete(OceanBaseGlobalState globalState) { - return new OceanBaseDeleteGenerator(globalState).generate(); + return new OceanBaseDeleteGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { OceanBaseTable randomTable = globalState.getSchema().getRandomTable(); OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState) .setColumns(randomTable.getColumns()); - ExpectedErrors errors = new ExpectedErrors(); sb.append("DELETE"); if (Randomly.getBoolean()) { sb.append(" /*+parallel(" + r.getLong(0, 10) + ") enable_parallel_dml*/ "); @@ -45,7 +44,6 @@ private SQLQueryAdapter generate() { "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index", "Incorrect value", "Out of range value for column", "Data truncation:")); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index 950317bd2..44fdf39fe 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -23,10 +23,11 @@ public OceanBaseUpdateGenerator(OceanBaseGlobalState globalState) { } public static SQLQueryAdapter update(OceanBaseGlobalState globalState) { - return new OceanBaseUpdateGenerator(globalState).generate(); + return new OceanBaseUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -45,8 +46,6 @@ private SQLQueryAdapter generate() { } errors.add("Duplicated primary key"); OceanBaseErrors.addInsertErrors(errors); - - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index f827331c0..452f5cdf4 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -1,25 +1,32 @@ package sqlancer.postgres.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; -public final class PostgresDeleteGenerator { +public final class PostgresDeleteGenerator extends AbstractDeleteGenerator { - private PostgresDeleteGenerator() { + private final PostgresGlobalState globalState; + + private PostgresDeleteGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter create(PostgresGlobalState globalState) { + return new PostgresDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append("DELETE FROM"); if (Randomly.getBoolean()) { sb.append(" ONLY"); } @@ -40,7 +47,6 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { errors.add("cannot cast"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 7ce7fe882..1effad582 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -20,6 +20,7 @@ public final class PostgresUpdateGenerator extends AbstractUpdateGenerator t.isInsertable()); List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); @@ -55,8 +57,6 @@ private SQLQueryAdapter generate() { randomTable.getColumns(), PostgresDataType.BOOLEAN); sb.append(PostgresVisitor.asString(where)); } - - return new SQLQueryAdapter(sb.toString(), errors, true); } @Override diff --git a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java index 9f869c241..59b7b4174 100644 --- a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java +++ b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.presto.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; @@ -9,14 +9,22 @@ import sqlancer.presto.PrestoSchema.PrestoTable; import sqlancer.presto.PrestoToStringVisitor; -public final class PrestoDeleteGenerator { +public final class PrestoDeleteGenerator extends AbstractDeleteGenerator { - private PrestoDeleteGenerator() { + private final PrestoGlobalState globalState; + + private PrestoDeleteGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canonicalizeString = false; } public static SQLQueryAdapter generate(PrestoGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new PrestoDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -26,7 +34,6 @@ public static SQLQueryAdapter generate(PrestoGlobalState globalState) { .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()))); } PrestoErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } } diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index e9fa06f7b..15d5f3543 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -17,18 +17,19 @@ public class PrestoInsertGenerator extends AbstractInsertGenerator public PrestoInsertGenerator(PrestoGlobalState globalState) { this.globalState = globalState; + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { - return new PrestoInsertGenerator(globalState).generate(); + return new PrestoInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); PrestoErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } @Override diff --git a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java index a8afcc578..3c197f0b9 100644 --- a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java +++ b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java @@ -19,13 +19,15 @@ public final class PrestoUpdateGenerator extends AbstractUpdateGenerator !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -34,7 +36,6 @@ private SQLQueryAdapter generate() { sb.append(" SET "); updateColumns(columns); PrestoErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } @Override diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java index e3a4dc35d..41ba72406 100644 --- a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -18,16 +18,16 @@ public QuestDBInsertGenerator(QuestDBGlobalState globalState) { this.globalState = globalState; } - private SQLQueryAdapter generate() { + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + return new QuestDBInsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { QuestDBTable table = globalState.getSchema().getRandomTable(); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); QuestDBErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); - } - - public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { - return new QuestDBInsertGenerator(globalState).generate(); } @Override diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java index c1404f509..c43584315 100644 --- a/src/sqlancer/spark/gen/SparkInsertGenerator.java +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -18,10 +18,11 @@ public class SparkInsertGenerator extends AbstractInsertGenerator { public SparkInsertGenerator(SparkGlobalState globalState) { this.globalState = globalState; this.gen = new SparkExpressionGenerator(globalState); + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(SparkGlobalState globalState) { - return new SparkInsertGenerator(globalState).generate(); + return new SparkInsertGenerator(globalState).getStatement(); } @Override @@ -29,7 +30,8 @@ protected void insertValue(SparkColumn column) { sb.append(SparkToStringVisitor.asString(gen.generateConstant())); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { sb.append("INSERT INTO "); SparkTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); @@ -40,6 +42,5 @@ private SQLQueryAdapter generate() { insertColumns(columns); SparkErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 4c86417d0..51b23eaa4 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -3,7 +3,7 @@ import java.util.Arrays; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; @@ -11,26 +11,35 @@ import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public final class SQLite3DeleteGenerator { +public final class SQLite3DeleteGenerator extends AbstractDeleteGenerator { - private SQLite3DeleteGenerator() { + private final SQLite3GlobalState globalState; + private final SQLite3Table table; + + private SQLite3DeleteGenerator(SQLite3GlobalState globalState, SQLite3Table table) { + this.globalState = globalState; + this.table = table; + this.canAffectSchema = true; } public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState) { - SQLite3Table tableName = globalState.getSchema().getRandomTable(t -> !t.isView() && !t.isReadOnly()); - return deleteContent(globalState, tableName); + SQLite3Table table = globalState.getSchema().getRandomTable(t -> !t.isView() && !t.isReadOnly()); + return deleteContent(globalState, table); + } + + public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLite3Table table) { + return new SQLite3DeleteGenerator(globalState, table).getStatement(); } - public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLite3Table tableName) { - StringBuilder sb = new StringBuilder(); + @Override + public void buildStatement() { sb.append("DELETE FROM "); - sb.append(tableName.getName()); + sb.append(table.getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); - sb.append(SQLite3Visitor.asString(new SQLite3ExpressionGenerator(globalState) - .setColumns(tableName.getColumns()).generateExpression())); + sb.append(SQLite3Visitor.asString( + new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } - ExpectedErrors errors = new ExpectedErrors(); SQLite3Errors.addExpectedExpressionErrors(errors); errors.addAll(Arrays.asList("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", "[SQLITE_CONSTRAINT] Abort due to constraint violation ", @@ -40,7 +49,6 @@ public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLi "cannot INSERT into generated column", "A table in the database is locked", "load_extension() prohibited in triggers and views", "The database file is locked")); SQLite3Errors.addDeleteErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 5a17ad339..9a2be3150 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -18,10 +18,13 @@ public class SQLite3UpdateGenerator extends AbstractUpdateGenerator columnsToUpdate = Randomly.nonEmptySubsetPotentialDuplicates(table.getColumns()); sb.append("UPDATE "); if (Randomly.getBoolean()) { @@ -98,8 +101,6 @@ private SQLQueryAdapter generate(SQLite3Table table) { SQLite3Errors.addInsertNowErrors(errors); SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addDeleteErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true /* column could have an ON UPDATE clause */); - } @Override diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index c3986f8d9..cbe5fc721 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -1,10 +1,9 @@ package sqlancer.tidb.gen; -import java.sql.SQLException; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; @@ -12,16 +11,24 @@ import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.visitor.TiDBVisitor; -public final class TiDBDeleteGenerator { +public final class TiDBDeleteGenerator extends AbstractDeleteGenerator { - private TiDBDeleteGenerator() { + private final TiDBGlobalState globalState; + + private TiDBDeleteGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + return new TiDBDeleteGenerator(globalState).getStatement(); } - public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); + @Override + public void buildStatement() { + errors.addAll(TiDBErrors.getExpressionErrors()); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); - StringBuilder sb = new StringBuilder("DELETE "); + sb.append("DELETE "); if (Randomly.getBooleanWithSmallProbability()) { sb.append("LOW_PRIORITY "); } @@ -55,8 +62,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx errors.add("is not valid for CHARACTER SET"); errors.add("Division by 0"); errors.add("error parsing regexp"); - return new SQLQueryAdapter(sb.toString(), errors); - } } diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 241ee3321..af6430b48 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -1,6 +1,5 @@ package sqlancer.tidb.gen; -import java.sql.SQLException; import java.util.List; import sqlancer.Randomly; @@ -22,11 +21,12 @@ private TiDBUpdateGenerator(TiDBGlobalState globalState) { this.globalState = globalState; } - public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - return new TiDBUpdateGenerator(globalState).generate(); + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + return new TiDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() throws SQLException { + @Override + public void buildStatement() { TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -40,8 +40,6 @@ private SQLQueryAdapter generate() throws SQLException { sb.append(TiDBVisitor.asString(gen.generateExpression())); } TiDBErrors.addInsertErrors(errors); - - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java index 108cd1be9..6ebe0db5f 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java @@ -1,31 +1,36 @@ package sqlancer.yugabyte.ycql.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLErrors; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; import sqlancer.yugabyte.ycql.YCQLToStringVisitor; -public final class YCQLDeleteGenerator { +public final class YCQLDeleteGenerator extends AbstractDeleteGenerator { - private YCQLDeleteGenerator() { + private final YCQLGlobalState globalState; + + private YCQLDeleteGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(YCQLGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new YCQLDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append("DELETE FROM "); sb.append(table.getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append(YCQLToStringVisitor.asString( new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } - YCQLErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java index d53c70d29..167f5d237 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -19,10 +19,11 @@ public YCQLInsertGenerator(YCQLGlobalState globalState) { } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - return new YCQLInsertGenerator(globalState).generate(); + return new YCQLInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getColumns(); buildInsertInto(globalState.getDatabaseName() + "." + table.getName(), columns); @@ -31,7 +32,6 @@ private SQLQueryAdapter generate() { errors.add("Null Argument for Primary Key"); YCQLErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java index a6c855cf3..eee49bee4 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -22,10 +22,11 @@ private YCQLUpdateGenerator(YCQLGlobalState globalState) { } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - return new YCQLUpdateGenerator(globalState).generate(); + return new YCQLUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -41,7 +42,6 @@ private SQLQueryAdapter generate() { errors.add("Missing Argument for Primary Key"); YCQLErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java index e0128707b..6e35c6862 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.yugabyte.ysql.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; @@ -9,18 +9,25 @@ import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; import sqlancer.yugabyte.ysql.YSQLVisitor; -public final class YSQLDeleteGenerator { +public final class YSQLDeleteGenerator extends AbstractDeleteGenerator { - private YSQLDeleteGenerator() { + private final YSQLGlobalState globalState; + + private YSQLDeleteGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter create(YSQLGlobalState globalState) { + return new YSQLDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { YSQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append("DELETE FROM"); if (Randomly.getBoolean()) { sb.append(" ONLY"); } @@ -41,7 +48,6 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { errors.add("cannot cast"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index bc7b00d79..6c5fd4144 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -21,6 +21,7 @@ public final class YSQLUpdateGenerator extends AbstractUpdateGenerator columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); @@ -57,8 +59,6 @@ private SQLQueryAdapter generate() { YSQLDataType.BOOLEAN); sb.append(YSQLVisitor.asString(where)); } - - return new SQLQueryAdapter(sb.toString(), errors, true); } @Override From 5d2e48a932d0fa17f0fc32ab4bd088635ca426b7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Apr 2026 00:35:56 +0800 Subject: [PATCH 1212/1284] Address review feedback: fix comment and add MaterializeBugs Remove incorrect claim about eventual consistency in readSchema comment. Add MaterializeBugs class with bugSchemaReadIncomplete flag to guard the retry logic, following the MySQLBugs pattern. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/materialize/MaterializeBugs.java | 12 ++++++++++++ src/sqlancer/materialize/MaterializeGlobalState.java | 8 +++++--- 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 src/sqlancer/materialize/MaterializeBugs.java diff --git a/src/sqlancer/materialize/MaterializeBugs.java b/src/sqlancer/materialize/MaterializeBugs.java new file mode 100644 index 000000000..a7611a925 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeBugs.java @@ -0,0 +1,12 @@ +package sqlancer.materialize; + +// do not make the fields final to avoid warnings +public final class MaterializeBugs { + + // Tables or columns may be missing when reading information_schema shortly after creation + public static boolean bugSchemaReadIncomplete = true; + + private MaterializeBugs() { + } + +} diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java index 8a66a7819..46977529b 100644 --- a/src/sqlancer/materialize/MaterializeGlobalState.java +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -268,10 +268,12 @@ public String getRandomTableAccessMethod() { @Override public MaterializeSchema readSchema() throws SQLException { - // Materialize's information_schema is eventually consistent: tables and columns - // may not be visible immediately after creation. Retry until the snapshot is - // consistent. + // Workaround for a suspected Materialize bug where tables or columns may be + // missing when reading the schema; retry until stable. readSchemaCallCount++; + if (!MaterializeBugs.bugSchemaReadIncomplete) { + return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + } for (int tries = 0; tries < 30; tries++) { MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() From 82612b1d0f310cc131661d2b92a8d8571453a2f9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Apr 2026 22:16:13 +0800 Subject: [PATCH 1213/1284] Wrap readSchema workaround in bugSchemaReadIncomplete guard Co-Authored-By: Claude Opus 4.7 --- .../materialize/MaterializeGlobalState.java | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java index 46977529b..7ab963c03 100644 --- a/src/sqlancer/materialize/MaterializeGlobalState.java +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -268,32 +268,32 @@ public String getRandomTableAccessMethod() { @Override public MaterializeSchema readSchema() throws SQLException { - // Workaround for a suspected Materialize bug where tables or columns may be - // missing when reading the schema; retry until stable. - readSchemaCallCount++; - if (!MaterializeBugs.bugSchemaReadIncomplete) { - return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); - } - for (int tries = 0; tries < 30; tries++) { - MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); - boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() - .anyMatch(t -> t.getColumns().isEmpty()); - boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; - boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); - if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { - lastKnownTableCount = schema.getDatabaseTables().size(); - return schema; - } - try { - Thread.sleep(100); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; + if (MaterializeBugs.bugSchemaReadIncomplete) { + // Workaround for a suspected Materialize bug where tables or columns may be + // missing when reading the schema; retry until stable. + readSchemaCallCount++; + for (int tries = 0; tries < 30; tries++) { + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() + .anyMatch(t -> t.getColumns().isEmpty()); + boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; + boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); + if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } } + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; } - MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); - lastKnownTableCount = schema.getDatabaseTables().size(); - return schema; + return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); } public void addFunctionAndType(String functionName, Character functionType) { From 0eea0212a9a07ec7f6b3f9bf5ae1fe5a101826dd Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Apr 2026 00:49:07 +0800 Subject: [PATCH 1214/1284] Fix checkstyle: declare SQLite3UpdateGenerator as final Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/common/gen/AbstractDeleteGenerator.java | 3 +++ src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java index 47df974e6..8dfb0b0c6 100644 --- a/src/sqlancer/common/gen/AbstractDeleteGenerator.java +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -2,4 +2,7 @@ public abstract class AbstractDeleteGenerator extends AbstractGenerator { + protected AbstractDeleteGenerator() { + } + } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 9a2be3150..05f4b9a2e 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -14,7 +14,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3UpdateGenerator extends AbstractUpdateGenerator { +public final class SQLite3UpdateGenerator extends AbstractUpdateGenerator { private final SQLite3GlobalState globalState; private final Randomly r; From d4ed473fb7d9ba1237a168144b22b6aeb5c9cf6f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Apr 2026 10:17:37 +0800 Subject: [PATCH 1215/1284] Refactor: extract common CREATE INDEX logic into AbstractIndexGenerator Introduce AbstractIndexGenerator with appendCreateIndex(boolean) and appendIndexColumnList(List, boolean) helpers, and convert nine concrete index generators (DuckDB, Doris, Presto, YCQL, TiDB, MariaDB, Materialize, Postgres, YSQL) from static-utility classes to instance-based subclasses whose logic lives in buildStatement(). Public entry points (getQuery/generate) are preserved so callers in the provider classes don't need to change. --- .../common/gen/AbstractIndexGenerator.java | 33 +++++++++++++ .../doris/gen/DorisIndexGenerator.java | 31 ++++++------ .../duckdb/gen/DuckDBIndexGenerator.java | 25 ++++++---- .../mariadb/gen/MariaDBIndexGenerator.java | 48 +++++++------------ .../gen/MaterializeIndexGenerator.java | 24 ++++++---- .../postgres/gen/PostgresIndexGenerator.java | 23 ++++----- .../presto/gen/PrestoIndexGenerator.java | 26 ++++++---- src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 26 +++++----- .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 37 +++++++------- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 23 ++++----- 10 files changed, 167 insertions(+), 129 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractIndexGenerator.java diff --git a/src/sqlancer/common/gen/AbstractIndexGenerator.java b/src/sqlancer/common/gen/AbstractIndexGenerator.java new file mode 100644 index 000000000..bfa62a36c --- /dev/null +++ b/src/sqlancer/common/gen/AbstractIndexGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractIndexGenerator> extends AbstractGenerator { + + protected void appendCreateIndex(boolean unique) { + sb.append("CREATE "); + if (unique) { + sb.append("UNIQUE "); + } + sb.append("INDEX "); + } + + protected void appendIndexColumnList(List columns, boolean allowOrdering) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + if (allowOrdering && Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/doris/gen/DorisIndexGenerator.java b/src/sqlancer/doris/gen/DorisIndexGenerator.java index 5e56bb192..308c09c3f 100644 --- a/src/sqlancer/doris/gen/DorisIndexGenerator.java +++ b/src/sqlancer/doris/gen/DorisIndexGenerator.java @@ -5,42 +5,45 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisTable; -public final class DorisIndexGenerator { +public class DorisIndexGenerator extends AbstractIndexGenerator { - private DorisIndexGenerator() { + private final DorisGlobalState globalState; + + public DorisIndexGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(DorisGlobalState globalState) throws SQLException { if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { throw new IgnoreMeException(); } - ExpectedErrors errors = new ExpectedErrors(); + return new DorisIndexGenerator(globalState).getStatement(); + } + @Override + public void buildStatement() { DorisTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - String indexName = globalState.getSchema().getFreeIndexName(); - StringBuilder sb = new StringBuilder("CREATE "); - sb.append("INDEX "); + appendCreateIndex(false); if (Randomly.getBoolean()) { sb.append("IF NOT EXISTS "); } - sb.append(indexName); + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); sb.append(randomTable.getName()); - sb.append("("); - int nr = 1; // Doris Only support CREATE_INDEX on single column and index type is BITMAP; - List subset = Randomly.extractNrRandomColumns(randomTable.getColumns(), nr); - sb.append(subset.get(0).getName()); - sb.append(") "); + // Doris only supports CREATE INDEX on a single column; index type is BITMAP + List subset = Randomly.extractNrRandomColumns(randomTable.getColumns(), 1); + appendIndexColumnList(subset, false); + sb.append(" "); if (Randomly.getBoolean()) { sb.append("USING BITMAP "); } - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 6c50b204d..bd42b64bd 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -3,26 +3,32 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -public final class DuckDBIndexGenerator { +public class DuckDBIndexGenerator extends AbstractIndexGenerator { - private DuckDBIndexGenerator() { + private final DuckDBGlobalState globalState; + + public DuckDBIndexGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { + return new DuckDBIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Data contains duplicates on indexed column(s)"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); @@ -43,7 +49,6 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("cannot create an index on the rowid"); } - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 1ba3fbd4d..1fb6f10b3 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -1,31 +1,36 @@ package sqlancer.mariadb.gen; -import java.util.List; - import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -public final class MariaDBIndexGenerator { +public class MariaDBIndexGenerator extends AbstractIndexGenerator { + + private final MariaDBSchema schema; - private MariaDBIndexGenerator() { + public MariaDBIndexGenerator(MariaDBSchema schema) { + this.schema = schema; + this.canAffectSchema = true; } public static SQLQueryAdapter generate(MariaDBSchema s) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder("CREATE "); + return new MariaDBIndexGenerator(s).getStatement(); + } + + @Override + public void buildStatement() { errors.add("Key/Index cannot be defined on a virtual generated column"); errors.add("Specified key was too long"); - if (Randomly.getBoolean()) { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Duplicate entry"); errors.add("Key/Index cannot be defined on a virtual generated column"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append("i"); sb.append(DBMSCommon.createColumnName(Randomly.smallNumber())); if (Randomly.getBoolean()) { @@ -34,28 +39,9 @@ public static SQLQueryAdapter generate(MariaDBSchema s) { } sb.append(" ON "); - MariaDBTable randomTable = s.getRandomTable(); + MariaDBTable randomTable = schema.getRandomTable(); sb.append(randomTable.getName()); - sb.append("("); - List columns = Randomly.nonEmptySubset(randomTable.getColumns()); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - if (Randomly.getBoolean()) { - sb.append(" "); - sb.append(Randomly.fromOptions("ASC", "DESC")); - } - } - sb.append(")"); - // if (Randomly.getBoolean()) { - // sb.append(" ALGORITHM="); - // sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY", "NOCOPY", "INSTANT")); - // errors.add("is not supported for this operation"); - // } - - return new SQLQueryAdapter(sb.toString(), errors, true); + appendIndexColumnList(Randomly.nonEmptySubset(randomTable.getColumns()), true); } } diff --git a/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java index 9d7a91b50..a63353f1f 100644 --- a/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java @@ -1,14 +1,18 @@ package sqlancer.materialize.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeTable; -public final class MaterializeIndexGenerator { +public class MaterializeIndexGenerator extends AbstractIndexGenerator { - private MaterializeIndexGenerator() { + private final MaterializeGlobalState globalState; + + public MaterializeIndexGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; } public enum IndexType { @@ -16,17 +20,18 @@ public enum IndexType { } public static SQLQueryAdapter generate(MaterializeGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - sb.append(" INDEX "); + return new MaterializeIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(false); MaterializeTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized // views sb.append(MaterializeCommon.getFreeIndexName(globalState.getSchema())); sb.append(" ON "); sb.append(randomTable.getName()); - IndexType method; - method = IndexType.BTREE; + IndexType method = IndexType.BTREE; sb.append("("); if (method == IndexType.HASH) { @@ -75,6 +80,5 @@ public static SQLQueryAdapter generate(MaterializeGlobalState globalState) { errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); MaterializeCommon.addCommonExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 4cb2b8e3a..bf70c32d4 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -15,9 +15,12 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public final class PostgresIndexGenerator { +public class PostgresIndexGenerator extends AbstractIndexGenerator { - private PostgresIndexGenerator() { + private final PostgresGlobalState globalState; + + public PostgresIndexGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; } public enum IndexType { @@ -25,13 +28,12 @@ public enum IndexType { } public static SQLQueryAdapter generate(PostgresGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - if (Randomly.getBoolean()) { - sb.append(" UNIQUE"); - } - sb.append(" INDEX "); + return new PostgresIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(Randomly.getBoolean()); /* * Commented out as a workaround for https://www.postgresql.org/message-id/CA%2Bu7OA4XYhc- * qyCgJqwwgMGZDWAyeH821oa5oMzm_HEifZ4BeA%40mail.gmail.com @@ -136,7 +138,6 @@ public static SQLQueryAdapter generate(PostgresGlobalState globalState) { errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); PostgresCommon.addCommonExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } private static String getNewIndexName(PostgresTable randomTable) { diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java index c76283ee4..5ec23c773 100644 --- a/src/sqlancer/presto/gen/PrestoIndexGenerator.java +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -3,7 +3,7 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; @@ -12,20 +12,27 @@ import sqlancer.presto.PrestoToStringVisitor; import sqlancer.presto.ast.PrestoExpression; -public final class PrestoIndexGenerator { +public class PrestoIndexGenerator extends AbstractIndexGenerator { - private PrestoIndexGenerator() { + private final PrestoGlobalState globalState; + + public PrestoIndexGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { + return new PrestoIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information sb.append(" ON "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); @@ -50,7 +57,6 @@ public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { sb.append(PrestoToStringVisitor.asString(expr)); } errors.add("already exists!"); - return new SQLQueryAdapter(sb.toString(), errors, true, false); } } diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index 1be2753b6..64b4c808d 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -5,34 +5,39 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; import sqlancer.tidb.TiDBSchema.TiDBTable; -public final class TiDBIndexGenerator { +public class TiDBIndexGenerator extends AbstractIndexGenerator { - private TiDBIndexGenerator() { + private final TiDBGlobalState globalState; + + public TiDBIndexGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { throw new IgnoreMeException(); } - ExpectedErrors errors = new ExpectedErrors(); + return new TiDBIndexGenerator(globalState).getStatement(); + } + @Override + public void buildStatement() { TiDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - String indexName = globalState.getSchema().getFreeIndexName(); - StringBuilder sb = new StringBuilder("CREATE "); - if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append("UNIQUE "); + boolean unique = Randomly.getBooleanWithRatherLowProbability(); + if (unique) { errors.add("Duplicate for key"); errors.add("Duplicate entry "); errors.add("A UNIQUE INDEX must include all columns in the table's partitioning function"); } - sb.append("INDEX "); - sb.append(indexName); + appendCreateIndex(unique); + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); sb.append(randomTable.getName()); sb.append("("); @@ -63,7 +68,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx errors.add("index already exist"); errors.add("Data truncation"); errors.add("key was too long"); - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java index ab03316cc..66c59a968 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -1,9 +1,7 @@ package sqlancer.yugabyte.ycql.gen; -import java.util.List; - import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; @@ -11,33 +9,31 @@ import sqlancer.yugabyte.ycql.YCQLToStringVisitor; import sqlancer.yugabyte.ycql.ast.YCQLExpression; -public final class YCQLIndexGenerator { +public class YCQLIndexGenerator extends AbstractIndexGenerator { + + private final YCQLGlobalState globalState; - private YCQLIndexGenerator() { + public YCQLIndexGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { + return new YCQLIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); sb.append(" ON "); YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); - sb.append("("); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - } - sb.append(")"); + appendIndexColumnList(table.getRandomNonEmptyColumnSubset(), false); if (Randomly.getBoolean()) { sb.append(" WHERE "); YCQLExpression expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) @@ -49,7 +45,6 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { errors.add("Invalid CQL Statement"); errors.add( "Invalid Table Definition. Transactions cannot be enabled in an index of a table without transactions enabled."); - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index 6077dcb1e..ac5b3242f 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ysql.YSQLErrors; @@ -17,19 +17,21 @@ import sqlancer.yugabyte.ysql.YSQLVisitor; import sqlancer.yugabyte.ysql.ast.YSQLExpression; -public final class YSQLIndexGenerator { +public class YSQLIndexGenerator extends AbstractIndexGenerator { - private YSQLIndexGenerator() { + private final YSQLGlobalState globalState; + + public YSQLIndexGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(YSQLGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - if (Randomly.getBoolean()) { - sb.append(" UNIQUE"); - } - sb.append(" INDEX "); + return new YSQLIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(Randomly.getBoolean()); YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized // views String indexName = getNewIndexName(randomTable); @@ -122,7 +124,6 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); YSQLErrors.addCommonExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } private static String getNewIndexName(YSQLTable randomTable) { From 037e9e7abb2c81179218d34e550507dd9330341e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Apr 2026 23:42:25 +0800 Subject: [PATCH 1216/1284] CI: cache Rust build and pinned tarball downloads Three caches that avoid repeated network work, keyed on pinned versions: - Swatinem/rust-cache for the DataFusion Rust build (cargo registry + target) - actions/cache for the CockroachDB tarball (cockroachdb + cockroachdb-qpg) - actions/cache for the Doris tarball Tarballs are cached as raw .tgz/.tar.gz files rather than extracted dirs, so runtime state from previous runs is not persisted. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee4186650..9df326bba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,9 +154,15 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true + - name: Cache CockroachDB tarball + uses: actions/cache@v4 + with: + path: cockroach-v24.2.0.linux-amd64.tgz + key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz + tar xzf cockroach-v24.2.0.linux-amd64.tgz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user @@ -180,9 +186,15 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true + - name: Cache CockroachDB tarball + uses: actions/cache@v4 + with: + path: cockroach-v24.2.0.linux-amd64.tgz + key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz + tar xzf cockroach-v24.2.0.linux-amd64.tgz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user @@ -228,6 +240,10 @@ jobs: with: toolchain: stable override: true + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + workspaces: src/sqlancer/datafusion/server/datafusion_server - name: Build DataFusion Server run: | cd src/sqlancer/datafusion/server/datafusion_server @@ -690,10 +706,15 @@ jobs: run: | sudo apt update sudo apt install mysql-client --assume-yes + - name: Cache Apache Doris tarball + uses: actions/cache@v4 + with: + path: apache-doris-2.1.4-bin-x64.tar.gz + key: apache-doris-2.1.4-bin-x64-tarball - name: Set up Apache Doris run: | sudo sysctl -w vm.max_map_count=2000000 - wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz + [ -f apache-doris-2.1.4-bin-x64.tar.gz ] || wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz tar zxf apache-doris-2.1.4-bin-x64.tar.gz mv apache-doris-2.1.4-bin-x64 apache-doris sudo swapoff -a From ee60e918d06f0f11105a9182a9852a6fa1d3209c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 10:20:56 +0800 Subject: [PATCH 1217/1284] CI: drop CockroachDB tarball cache (no measured benefit) Warm-cache rerun showed CockroachDB setup at 11s vs 10s baseline - binaries.cockroachdb.com is fast enough that cache restore offers no saving. Keep the Rust build (-90s) and Doris (-216s) caches. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9df326bba..e85c5c219 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,15 +154,9 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Cache CockroachDB tarball - uses: actions/cache@v4 - with: - path: cockroach-v24.2.0.linux-amd64.tgz - key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz - tar xzf cockroach-v24.2.0.linux-amd64.tgz + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user @@ -186,15 +180,9 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Cache CockroachDB tarball - uses: actions/cache@v4 - with: - path: cockroach-v24.2.0.linux-amd64.tgz - key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz - tar xzf cockroach-v24.2.0.linux-amd64.tgz + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user From b2df941f170bb87a975b3a55cc14692ef9b68514 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 00:07:52 +0800 Subject: [PATCH 1218/1284] Refactor: extract common CREATE TABLE logic into AbstractTableGenerator Introduce AbstractTableGenerator with appendCreateTable header helpers and appendColumnDefinitions / appendColumnDefinitionList iteration helpers that delegate to an overridable appendColumnDefinition(C) (default: "name type"). Convert four concrete table generators (QuestDB, HSQLDB, Databend, DuckDB) from plain classes to instance-based subclasses whose logic lives in buildStatement(). Public entry points (getQuery) are preserved so callers in the provider classes don't need to change. Generators with DB-specific shape (temporary/unlogged modifiers, in-place column building, partitioning, custom return types, etc.) are left alone. --- .../common/gen/AbstractTableGenerator.java | 42 +++++++++ .../databend/gen/DatabendTableGenerator.java | 65 +++++++------ .../duckdb/gen/DuckDBTableGenerator.java | 92 ++++++++++--------- .../hsqldb/gen/HSQLDBTableGenerator.java | 60 ++++++------ .../questdb/gen/QuestDBTableGenerator.java | 41 ++++----- 5 files changed, 178 insertions(+), 122 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractTableGenerator.java diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java new file mode 100644 index 000000000..e1cf34832 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -0,0 +1,42 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractTableGenerator> extends AbstractGenerator { + + protected void appendCreateTable(String tableName) { + appendCreateTable(tableName, false); + } + + protected void appendCreateTable(String tableName, boolean ifNotExists) { + sb.append("CREATE TABLE "); + if (ifNotExists) { + sb.append("IF NOT EXISTS "); + } + sb.append(tableName); + } + + protected void appendColumnDefinitions(List columns) { + sb.append("("); + appendColumnDefinitionList(columns); + sb.append(")"); + } + + protected void appendColumnDefinitionList(List columns) { + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + appendColumnDefinition(columns.get(i)); + } + } + + protected void appendColumnDefinition(C column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 514d740d0..2c3416538 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -4,8 +4,8 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.gen.TypedExpressionGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; @@ -15,44 +15,49 @@ import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendExpression; -public class DatabendTableGenerator { +public class DatabendTableGenerator extends AbstractTableGenerator { + + private DatabendGlobalState globalState; + private TypedExpressionGenerator gen; + + public DatabendTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { DatabendErrors.addExpressionErrors(errors); - StringBuilder sb = new StringBuilder(); String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName); List columns = getNewColumns(); - TypedExpressionGenerator gen = new DatabendNewExpressionGenerator( - globalState).setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); + gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + appendColumnDefinitions(columns); + } - if (globalState.getDbmsSpecificOptions().testNotNullConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" NOT NULL"); - } else { - sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 - } + @Override + protected void appendColumnDefinition(DatabendColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); - if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { - sb.append(" DEFAULT("); - sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 - gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); - sb.append(")"); - } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } else { + sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 } - sb.append(")"); - return new SQLQueryAdapter(sb.toString(), errors, true); + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 + gen.generateConstant(column.getType().getPrimitiveDataType()))); + sb.append(")"); + } } private static List getNewColumns() { diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index ea6d3537f..c8ac6f28d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -5,8 +5,8 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -16,50 +16,28 @@ import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBTableGenerator { +public class DuckDBTableGenerator extends AbstractTableGenerator { + + private DuckDBGlobalState globalState; + private UntypedExpressionGenerator gen; + + public DuckDBTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName); List columns = getNewColumns(); - UntypedExpressionGenerator gen = new DuckDBExpressionGenerator(globalState) - .setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() - && columns.get(i).getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { - sb.append(" COLLATE "); - sb.append(getRandomCollate()); - } - if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" UNIQUE"); - } - if (globalState.getDbmsSpecificOptions().testNotNullConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" NOT NULL"); - } - if (globalState.getDbmsSpecificOptions().testCheckConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" CHECK("); - sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); - DuckDBErrors.addExpressionErrors(errors); - sb.append(")"); - } - if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { - sb.append(" DEFAULT("); - sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); - sb.append(")"); - } - } + gen = new DuckDBExpressionGenerator(globalState).setColumns(columns); + sb.append("("); + appendColumnDefinitionList(columns); if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { errors.add("Invalid type for index"); List primaryKeyColumns = Randomly.nonEmptySubset(columns); @@ -68,7 +46,37 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(")"); } sb.append(")"); - return new SQLQueryAdapter(sb.toString(), errors, true); + } + + @Override + protected void appendColumnDefinition(DuckDBColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + && column.getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { + sb.append(" COLLATE "); + sb.append(getRandomCollate()); + } + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" UNIQUE"); + } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } + if (globalState.getDbmsSpecificOptions().testCheckConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" CHECK("); + sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); + DuckDBErrors.addExpressionErrors(errors); + sb.append(")"); + } + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + } } public static String getRandomCollate() { diff --git a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java index 48606e9bf..30249e780 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java @@ -5,44 +5,48 @@ import javax.annotation.Nullable; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; -public class HSQLDBTableGenerator { +public class HSQLDBTableGenerator extends AbstractTableGenerator { + + private HSQLDBProvider.HSQLDBGlobalState globalState; + private String tableName; + + public HSQLDBTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @Nullable String tableName) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + this.tableName = tableName; + return getStatement(); + } + + @Override + public void buildStatement() { String name = tableName; - if (tableName == null) { + if (name == null) { name = globalState.getSchema().getFreeTableName(); } - sb.append("CREATE TABLE "); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(name); - sb.append("("); - List columns = getNewColumns(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType().getType().name()); - if (columns.get(i).getType().getSize() > 0) { - // Cannot specify size for non composite data types - sb.append("("); - sb.append(columns.get(i).getType().getSize()); - sb.append(")"); - } - } - sb.append(")"); + appendCreateTable(name, Randomly.getBoolean()); + appendColumnDefinitions(getNewColumns()); sb.append(";"); - return new SQLQueryAdapter(sb.toString(), errors, true); + } + + @Override + protected void appendColumnDefinition(HSQLDBSchema.HSQLDBColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType().getType().name()); + if (column.getType().getSize() > 0) { + // Cannot specify size for non composite data types + sb.append("("); + sb.append(column.getType().getSize()); + sb.append(")"); + } } private static List getNewColumns() { diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java index a308b8d10..d4b17e76b 100644 --- a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -5,40 +5,37 @@ import javax.annotation.Nullable; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; -public class QuestDBTableGenerator { +public class QuestDBTableGenerator extends AbstractTableGenerator { + + private QuestDBGlobalState globalState; + private String tableName; + + public QuestDBTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(QuestDBGlobalState globalState, @Nullable String tableName) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + this.tableName = tableName; + return getStatement(); + } + + @Override + public void buildStatement() { String name = tableName; - if (tableName == null) { + if (name == null) { name = globalState.getSchema().getFreeTableName(); } - sb.append("CREATE TABLE "); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(name); - sb.append("("); - List columns = getNewColumns(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - } - sb.append(")"); + appendCreateTable(name, Randomly.getBoolean()); + appendColumnDefinitions(getNewColumns()); sb.append(";"); errors.add("table already exists"); - return new SQLQueryAdapter(sb.toString(), errors, true); } private static List getNewColumns() { From 8d53aabc5080095dc0350e452ed1bee8f04be149 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 00:24:33 +0800 Subject: [PATCH 1219/1284] Refactor: migrate Presto, YCQL, Spark table generators to AbstractTableGenerator Follow-up to the initial four migrations. Presto and YCQL fit the canonical CREATE TABLE [IF NOT EXISTS] (col type[, ...]) shape directly. Spark needs the columns pre-built before iteration so that appendColumnDefinitions can call back into the overridden appendColumnDefinition; this is safe because Spark's DEFAULT clause uses generateConstant, which does not reference other columns. Hive is left alone: its CHECK/DEFAULT constraints use generateExpression which reads the columnsToBeAdded list incrementally, so pre-building would change expression-generation behavior. --- .../presto/gen/PrestoTableGenerator.java | 70 +++++++------------ .../spark/gen/SparkTableGenerator.java | 45 +++++------- .../yugabyte/ycql/gen/YCQLTableGenerator.java | 37 +++++----- 3 files changed, 60 insertions(+), 92 deletions(-) diff --git a/src/sqlancer/presto/gen/PrestoTableGenerator.java b/src/sqlancer/presto/gen/PrestoTableGenerator.java index 1d7df2ee6..49e1346ad 100644 --- a/src/sqlancer/presto/gen/PrestoTableGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTableGenerator.java @@ -4,13 +4,35 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema.PrestoColumn; import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; -public class PrestoTableGenerator { +public class PrestoTableGenerator extends AbstractTableGenerator { + + private PrestoGlobalState globalState; + + public PrestoTableGenerator() { + this.canAffectSchema = true; + this.canonicalizeString = false; + } + + public SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { + String catalog = globalState.getDbmsSpecificOptions().catalog; + String schema = globalState.getDatabaseName(); + String tableName = globalState.getSchema().getFreeTableName(); + String qualifiedName = catalog + "." + schema + "." + tableName; + appendCreateTable(qualifiedName); + appendColumnDefinitions(getNewColumns()); + } private static List getNewColumns() { List columns = new ArrayList<>(); @@ -22,48 +44,4 @@ private static List getNewColumns() { return columns; } - public SQLQueryAdapter getQuery(PrestoGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - String catalog = globalState.getDbmsSpecificOptions().catalog; - String schema = globalState.getDatabaseName(); - - sb.append(catalog).append("."); - sb.append(schema).append("."); - - sb.append(tableName); - sb.append("("); - List columns = getNewColumns(); - // TypedExpressionGenerator, PrestoColumn, PrestoCompositeDataType> - // typedExpressionGenerator = new PrestoTypedExpressionGenerator(globalState).setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - PrestoColumn column = columns.get(i); - sb.append(column.getName()); - sb.append(" "); - sb.append(column.getType()); - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" UNIQUE"); - // } - // if (globalState.getDbmsSpecificOptions().testNotNullConstraints - // && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" NOT NULL"); - // } - } - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { - // errors.add("Invalid type for index"); - // List primaryKeyColumns = Randomly.nonEmptySubset(columns); - // sb.append(", PRIMARY KEY("); - // sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - // sb.append(")"); - // } - sb.append(")"); - - return new SQLQueryAdapter(sb.toString(), errors, true, false); - } - } diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java index 937e52248..2c26ea34c 100644 --- a/src/sqlancer/spark/gen/SparkTableGenerator.java +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.spark.SparkErrors; import sqlancer.spark.SparkGlobalState; @@ -15,7 +15,7 @@ import sqlancer.spark.SparkSchema.SparkTable; import sqlancer.spark.SparkToStringVisitor; -public class SparkTableGenerator { +public class SparkTableGenerator extends AbstractTableGenerator { private enum ColumnConstraints { NOT_NULL, DEFAULT @@ -27,7 +27,6 @@ private enum ColumnConstraints { private final SparkGlobalState globalState; private final String tableName; - private final StringBuilder sb = new StringBuilder(); private final SparkExpressionGenerator gen; private final SparkTable table; private final List columnsToBeAdded = new ArrayList<>(); @@ -37,28 +36,25 @@ public SparkTableGenerator(SparkGlobalState globalState, String tableName) { this.globalState = globalState; this.table = new SparkTable(tableName, columnsToBeAdded, false); this.gen = new SparkExpressionGenerator(globalState).setColumns(columnsToBeAdded); + this.canAffectSchema = true; + this.canonicalizeString = false; } public static SQLQueryAdapter generate(SparkGlobalState globalState, String tableName) { - SparkTableGenerator generator = new SparkTableGenerator(globalState, tableName); - return generator.create(); + return new SparkTableGenerator(globalState, tableName).getStatement(); } - private SQLQueryAdapter create() { - ExpectedErrors errors = new ExpectedErrors(); - - sb.append("CREATE TABLE "); - sb.append(globalState.getDatabaseName()); - sb.append("."); - sb.append(tableName); - sb.append(" ("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - if (i != 0) { - sb.append(", "); - } - appendColumn(i); + @Override + public void buildStatement() { + int columnCount = Randomly.smallNumber() + 1; + for (int i = 0; i < columnCount; i++) { + String columnName = DBMSCommon.createColumnName(i); + SparkDataType type = SparkSchema.SparkDataType.getRandomType(); + columnsToBeAdded.add(new SparkColumn(columnName, table, type)); } - sb.append(")"); + appendCreateTable(globalState.getDatabaseName() + "." + tableName); + sb.append(" "); + appendColumnDefinitions(columnsToBeAdded); sb.append(" USING PARQUET"); // TODO: implement PARTITION BY clause @@ -67,16 +63,13 @@ private SQLQueryAdapter create() { // TODO: randomly add some predefined TABLEPROPERTIES SparkErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true, false); } - private void appendColumn(int columnId) { - String columnName = DBMSCommon.createColumnName(columnId); - sb.append(columnName); + @Override + protected void appendColumnDefinition(SparkColumn column) { + sb.append(column.getName()); sb.append(" "); - SparkDataType randType = SparkSchema.SparkDataType.getRandomType(); - sb.append(randType); - columnsToBeAdded.add(new SparkColumn(columnName, table, randType)); + sb.append(column.getType()); appendColumnConstraint(); } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java index 148a1cdbc..c57f94287 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java @@ -5,35 +5,33 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLCompositeDataType; -public class YCQLTableGenerator { +public class YCQLTableGenerator extends AbstractTableGenerator { + + private YCQLGlobalState globalState; + + public YCQLTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName, Randomly.getBoolean()); List columns = getNewColumns(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - // todo PK, STATIC - } + sb.append("("); + appendColumnDefinitionList(columns); errors.add("Query timed out after PT2S"); errors.add("Invalid type for index"); List primaryKeyColumns = Randomly.nonEmptySubset(columns); @@ -41,7 +39,6 @@ public SQLQueryAdapter getQuery(YCQLGlobalState globalState) { sb.append(primaryKeyColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); sb.append(")"); sb.append(")"); - return new SQLQueryAdapter(sb.toString(), errors, true); } private static List getNewColumns() { From db0d9e62bc82abfa387272a969b20bf5061739a6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 00:31:33 +0800 Subject: [PATCH 1220/1284] Document AbstractTableGenerator helpers with example output Add short Javadoc to each helper showing what SQL fragment it emits, so subclass authors can pick the right entry point at a glance. --- .../common/gen/AbstractTableGenerator.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java index e1cf34832..c6ca55cbc 100644 --- a/src/sqlancer/common/gen/AbstractTableGenerator.java +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -6,10 +6,12 @@ public abstract class AbstractTableGenerator> extends AbstractGenerator { + /** Appends {@code CREATE TABLE }. */ protected void appendCreateTable(String tableName) { appendCreateTable(tableName, false); } + /** Appends {@code CREATE TABLE [IF NOT EXISTS ]}. */ protected void appendCreateTable(String tableName, boolean ifNotExists) { sb.append("CREATE TABLE "); if (ifNotExists) { @@ -18,12 +20,20 @@ protected void appendCreateTable(String tableName, boolean ifNotExists) { sb.append(tableName); } + /** + * Appends a parenthesized, comma-separated column definition list, e.g. {@code (c0 INT, c1 TEXT)}. Delegates each + * column's rendering to {@link #appendColumnDefinition(AbstractTableColumn)}. + */ protected void appendColumnDefinitions(List columns) { sb.append("("); appendColumnDefinitionList(columns); sb.append(")"); } + /** + * Appends a comma-separated column definition list without enclosing parentheses, e.g. {@code c0 INT, c1 TEXT}. + * Useful when subclasses also emit table-level constraints (e.g. {@code PRIMARY KEY (...)}) inside the same parens. + */ protected void appendColumnDefinitionList(List columns) { for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -33,6 +43,10 @@ protected void appendColumnDefinitionList(List columns) { } } + /** + * Appends a single column's definition. Default output is {@code }, e.g. {@code c0 INT}. Override to + * add constraints such as {@code NOT NULL}, {@code DEFAULT ...}, or {@code CHECK (...)}. + */ protected void appendColumnDefinition(C column) { sb.append(column.getName()); sb.append(" "); From 220835e5fb96383cf814ba9407e1891e282a528c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 25 Apr 2026 23:30:28 +0800 Subject: [PATCH 1221/1284] Fix checkstyle: add @param tags to AbstractTableGenerator Javadoc Co-Authored-By: Claude Opus 4.7 --- .../common/gen/AbstractTableGenerator.java | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java index c6ca55cbc..c8d1d9697 100644 --- a/src/sqlancer/common/gen/AbstractTableGenerator.java +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -6,12 +6,24 @@ public abstract class AbstractTableGenerator> extends AbstractGenerator { - /** Appends {@code CREATE TABLE }. */ + /** + * Appends {@code CREATE TABLE }. + * + * @param tableName + * the name of the table to create. + */ protected void appendCreateTable(String tableName) { appendCreateTable(tableName, false); } - /** Appends {@code CREATE TABLE [IF NOT EXISTS ]}. */ + /** + * Appends {@code CREATE TABLE [IF NOT EXISTS ]}. + * + * @param tableName + * the name of the table to create. + * @param ifNotExists + * whether to emit the {@code IF NOT EXISTS} clause. + */ protected void appendCreateTable(String tableName, boolean ifNotExists) { sb.append("CREATE TABLE "); if (ifNotExists) { @@ -23,6 +35,9 @@ protected void appendCreateTable(String tableName, boolean ifNotExists) { /** * Appends a parenthesized, comma-separated column definition list, e.g. {@code (c0 INT, c1 TEXT)}. Delegates each * column's rendering to {@link #appendColumnDefinition(AbstractTableColumn)}. + * + * @param columns + * the columns to render. */ protected void appendColumnDefinitions(List columns) { sb.append("("); @@ -33,6 +48,9 @@ protected void appendColumnDefinitions(List columns) { /** * Appends a comma-separated column definition list without enclosing parentheses, e.g. {@code c0 INT, c1 TEXT}. * Useful when subclasses also emit table-level constraints (e.g. {@code PRIMARY KEY (...)}) inside the same parens. + * + * @param columns + * the columns to render. */ protected void appendColumnDefinitionList(List columns) { for (int i = 0; i < columns.size(); i++) { @@ -46,6 +64,9 @@ protected void appendColumnDefinitionList(List columns) { /** * Appends a single column's definition. Default output is {@code }, e.g. {@code c0 INT}. Override to * add constraints such as {@code NOT NULL}, {@code DEFAULT ...}, or {@code CHECK (...)}. + * + * @param column + * the column whose definition to render. */ protected void appendColumnDefinition(C column) { sb.append(column.getName()); From 12048b67e9134820acc98037221fd1b41e1867c1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 26 Apr 2026 00:16:31 +0800 Subject: [PATCH 1222/1284] Refactor: extract common DELETE generation logic into AbstractDeleteGenerator Add appendDeleteFromTable, appendLimitClause, and appendReturningClause helpers in AbstractDeleteGenerator and migrate all 15 DELETE generators that extend it. Lift the WHERE-clause helper to AbstractGenerator since it is also used by UPDATE, partial-INDEX, and INSERT...ON CONFLICT generators, and migrate 17 of those generators to use it. Co-Authored-By: Claude Opus 4.7 --- .../gen/CockroachDBDeleteGenerator.java | 6 +-- .../gen/CockroachDBUpdateGenerator.java | 3 +- .../common/gen/AbstractDeleteGenerator.java | 50 +++++++++++++++++++ .../common/gen/AbstractGenerator.java | 13 +++++ .../databend/gen/DatabendDeleteGenerator.java | 6 +-- .../doris/gen/DorisDeleteGenerator.java | 6 +-- .../doris/gen/DorisUpdateGenerator.java | 3 +- .../duckdb/gen/DuckDBDeleteGenerator.java | 6 +-- src/sqlancer/h2/H2DeleteGenerator.java | 9 ++-- src/sqlancer/h2/H2UpdateGenerator.java | 3 +- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 3 +- .../mariadb/gen/MariaDBDeleteGenerator.java | 17 ++++--- .../gen/MaterializeDeleteGenerator.java | 7 +-- .../gen/MaterializeUpdateGenerator.java | 3 +- .../mysql/gen/MySQLDeleteGenerator.java | 3 +- .../mysql/gen/MySQLUpdateGenerator.java | 3 +- .../gen/OceanBaseDeleteGenerator.java | 3 +- .../gen/OceanBaseUpdateGenerator.java | 3 +- .../postgres/gen/PostgresDeleteGenerator.java | 13 ++--- .../postgres/gen/PostgresIndexGenerator.java | 3 +- .../postgres/gen/PostgresUpdateGenerator.java | 3 +- .../presto/gen/PrestoDeleteGenerator.java | 6 +-- .../presto/gen/PrestoIndexGenerator.java | 3 +- .../gen/dml/SQLite3DeleteGenerator.java | 6 +-- .../gen/dml/SQLite3UpdateGenerator.java | 3 +- .../tidb/gen/TiDBDeleteGenerator.java | 6 +-- .../tidb/gen/TiDBUpdateGenerator.java | 3 +- .../ycql/gen/YCQLDeleteGenerator.java | 6 +-- .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 3 +- .../ysql/gen/YSQLDeleteGenerator.java | 13 ++--- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 3 +- .../ysql/gen/YSQLUpdateGenerator.java | 3 +- 32 files changed, 116 insertions(+), 105 deletions(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index f0048d278..dd1e94aa9 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -24,12 +24,10 @@ public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { @Override public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); CockroachDBErrors.addExpressionErrors(errors); - sb.append(CockroachDBVisitor.asString(new CockroachDBExpressionGenerator(globalState) + appendWhereClause(CockroachDBVisitor.asString(new CockroachDBExpressionGenerator(globalState) .setColumns(table.getColumns()).generateExpression(CockroachDBDataType.BOOL.get()))); } else { errors.add("rejected: DELETE without WHERE clause (sql_safe_updates = true)"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index b367f5c59..06cc0f8ee 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -40,8 +40,7 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); + appendWhereClause(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); } errors.add("violates unique constraint"); errors.add("violates not-null constraint"); diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java index 8dfb0b0c6..f3b5a1955 100644 --- a/src/sqlancer/common/gen/AbstractDeleteGenerator.java +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -5,4 +5,54 @@ public abstract class AbstractDeleteGenerator extends AbstractGenerator { protected AbstractDeleteGenerator() { } + /** + * Appends {@code DELETE FROM }. + * + * @param tableName + * the name of the table to delete from. + */ + protected void appendDeleteFromTable(String tableName) { + appendDeleteFromTable(tableName, false); + } + + /** + * Appends {@code DELETE FROM [ONLY ]}. + * + * @param tableName + * the name of the table to delete from. + * @param only + * whether to emit the {@code ONLY} keyword (used by some databases to restrict deletion to the named + * table rather than its inheritance descendants). + */ + protected void appendDeleteFromTable(String tableName, boolean only) { + sb.append("DELETE FROM "); + if (only) { + sb.append("ONLY "); + } + sb.append(tableName); + } + + /** + * Appends {@code LIMIT } (with a leading space). + * + * @param value + * the LIMIT value, e.g. an integer literal or already-rendered expression. Converted via + * {@link StringBuilder#append(Object)}. + */ + protected void appendLimitClause(Object value) { + sb.append(" LIMIT "); + sb.append(value); + } + + /** + * Appends {@code RETURNING } (with a leading space). + * + * @param expression + * the rendered RETURNING expression. + */ + protected void appendReturningClause(String expression) { + sb.append(" RETURNING "); + sb.append(expression); + } + } diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java index 5d13fc746..dbdf100d5 100644 --- a/src/sqlancer/common/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -17,4 +17,17 @@ public SQLQueryAdapter getStatement() { public abstract void buildStatement(); + /** + * Appends {@code WHERE } (with a leading space). Subclasses are responsible for deciding whether to + * include the WHERE clause, typically based on a randomized boolean. Used by DELETE, UPDATE, partial-INDEX, and + * INSERT...ON CONFLICT generators. + * + * @param condition + * the rendered WHERE condition. + */ + protected void appendWhereClause(String condition) { + sb.append(" WHERE "); + sb.append(condition); + } + } diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index 5adabf175..22336fd72 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -22,11 +22,9 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); - sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + appendDeleteFromTable(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DatabendToStringVisitor.asString( + appendWhereClause(DatabendToStringVisitor.asString( new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN))); DatabendErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/doris/gen/DorisDeleteGenerator.java b/src/sqlancer/doris/gen/DorisDeleteGenerator.java index 66deaea8f..b155e0381 100644 --- a/src/sqlancer/doris/gen/DorisDeleteGenerator.java +++ b/src/sqlancer/doris/gen/DorisDeleteGenerator.java @@ -23,12 +23,10 @@ public static SQLQueryAdapter generate(DorisGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + appendWhereClause(DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) .setColumns(table.getColumns()).generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java index e0db2b7b4..93f835eff 100644 --- a/src/sqlancer/doris/gen/DorisUpdateGenerator.java +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -35,8 +35,7 @@ public void buildStatement() { sb.append(table.getName()); sb.append(" SET "); updateColumns(columns); - sb.append(" WHERE "); - sb.append(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + appendWhereClause(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addInsertErrors(errors); } diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index 42695a9f6..9f4f4ed6a 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -22,12 +22,10 @@ public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DuckDBToStringVisitor.asString( + appendWhereClause(DuckDBToStringVisitor.asString( new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } DuckDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java index dd0fdff34..291ee99c2 100644 --- a/src/sqlancer/h2/H2DeleteGenerator.java +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -20,17 +20,14 @@ public static SQLQueryAdapter getQuery(H2GlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(H2ToStringVisitor.asString( + appendWhereClause(H2ToStringVisitor.asString( new H2ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } if (Randomly.getBoolean()) { - sb.append(" LIMIT "); - sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); + appendLimitClause(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); } H2Errors.addExpressionErrors(errors); H2Errors.addDeleteErrors(errors); diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index fe63c7a4e..05e348038 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -34,8 +34,7 @@ public void buildStatement() { H2Errors.addInsertErrors(errors); H2Errors.addDeleteErrors(errors); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(H2ToStringVisitor.asString(gen.generateExpression())); + appendWhereClause(H2ToStringVisitor.asString(gen.generateExpression())); } H2Errors.addExpressionErrors(errors); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index 54380214f..2e6081df4 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -37,8 +37,7 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); - sb.append(HSQLDBToStringVisitor.asString( + appendWhereClause(HSQLDBToStringVisitor.asString( gen.generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)))); errors.add("data type of expression is not boolean"); HSQLDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index 2992f569d..da1651662 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -57,12 +57,13 @@ public void buildStatement() { sb.append(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); + String condition; if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); + condition = MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r)); } else { - sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); + condition = MariaDBVisitor.asString(expressionGenerator.getRandomExpression()); } + appendWhereClause(condition); } // ORDER BY + LIMIT @@ -75,18 +76,18 @@ public void buildStatement() { } if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" LIMIT "); - sb.append(Randomly.getNotCachedInteger(1, 10)); + appendLimitClause(Randomly.getNotCachedInteger(1, 10)); } // RETURNING clause (MariaDB >= 10.5) if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" RETURNING "); + String expression; if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); + expression = MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r)); } else { - sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); + expression = MariaDBVisitor.asString(expressionGenerator.getRandomExpression()); } + appendReturningClause(expression); } if (sb.toString().contains("RLIKE") || sb.toString().contains("REGEXP")) { diff --git a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java index 2aceddcc9..a5483ee1d 100644 --- a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java @@ -26,12 +26,9 @@ public void buildStatement() { errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - sb.append("DELETE FROM"); - sb.append(" "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, + appendWhereClause(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, table.getColumns(), MaterializeDataType.BOOLEAN))); } MaterializeCommon.addCommonExpressionErrors(errors); diff --git a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java index 7d338c027..abd288f99 100644 --- a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java @@ -52,10 +52,9 @@ public void buildStatement() { errors.add("but expression is of type"); MaterializeCommon.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); MaterializeExpression where = MaterializeExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), MaterializeDataType.BOOLEAN); - sb.append(MaterializeVisitor.asString(where)); + appendWhereClause(MaterializeVisitor.asString(where)); } } diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index 048ef335f..551264c4a 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -40,8 +40,7 @@ public void buildStatement() { sb.append(" FROM "); sb.append(randomTable.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(MySQLVisitor.asString(gen.generateExpression())); + appendWhereClause(MySQLVisitor.asString(gen.generateExpression())); MySQLErrors.addExpressionErrors(errors); } errors.addAll(Arrays.asList("doesn't have this option", diff --git a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java index 1aca84580..ad13a148a 100644 --- a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java @@ -34,9 +34,8 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); MySQLErrors.addExpressionErrors(errors); - sb.append(MySQLVisitor.asString(gen.generateExpression())); + appendWhereClause(MySQLVisitor.asString(gen.generateExpression())); } MySQLErrors.addInsertUpdateErrors(errors); errors.add("doesn't have this option"); diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index c46ace304..ec1aa7fc0 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -36,8 +36,7 @@ public void buildStatement() { sb.append(" FROM "); sb.append(randomTable.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + appendWhereClause(OceanBaseVisitor.asString(gen.generateExpression())); OceanBaseErrors.addExpressionErrors(errors); } errors.addAll(Arrays.asList("doesn't have this option", "Truncated incorrect DOUBLE value", diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index 44fdf39fe..51816e691 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -39,9 +39,8 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); OceanBaseErrors.addExpressionErrors(errors); - sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + appendWhereClause(OceanBaseVisitor.asString(gen.generateExpression())); errors.add("Data Too Long"); } errors.add("Duplicated primary key"); diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 452f5cdf4..3250b86c8 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -26,20 +26,13 @@ public void buildStatement() { errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - sb.append("DELETE FROM"); + appendDeleteFromTable(table.getName(), Randomly.getBoolean()); if (Randomly.getBoolean()) { - sb.append(" ONLY"); - } - sb.append(" "); - sb.append(table.getName()); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(PostgresVisitor.asString(PostgresExpressionGenerator.generateExpression(globalState, + appendWhereClause(PostgresVisitor.asString(PostgresExpressionGenerator.generateExpression(globalState, table.getColumns(), PostgresDataType.BOOLEAN))); } if (Randomly.getBoolean()) { - sb.append(" RETURNING "); - sb.append(PostgresVisitor + appendReturningClause(PostgresVisitor .asString(PostgresExpressionGenerator.generateExpression(globalState, table.getColumns()))); } PostgresCommon.addCommonExpressionErrors(errors); diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index bf70c32d4..c852684a9 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -107,10 +107,9 @@ public void buildStatement() { sb.append(")"); } if (Randomly.getBoolean()) { - sb.append(" WHERE "); PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(randomTable.getColumns()) .setGlobalState(globalState).generateExpression(PostgresDataType.BOOLEAN); - sb.append(PostgresVisitor.asString(expr)); + appendWhereClause(PostgresVisitor.asString(expr)); } errors.add("already contains data"); // CONCURRENT INDEX failed errors.add("You might need to add explicit type casts"); diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 1effad582..92257a95a 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -52,10 +52,9 @@ public void buildStatement() { errors.add("but expression is of type"); PostgresCommon.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); PostgresExpression where = PostgresExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), PostgresDataType.BOOLEAN); - sb.append(PostgresVisitor.asString(where)); + appendWhereClause(PostgresVisitor.asString(where)); } } diff --git a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java index 59b7b4174..4d5724992 100644 --- a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java +++ b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java @@ -24,12 +24,10 @@ public static SQLQueryAdapter generate(PrestoGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(PrestoToStringVisitor + appendWhereClause(PrestoToStringVisitor .asString(new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()))); } diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java index 5ec23c773..2d7b75cf8 100644 --- a/src/sqlancer/presto/gen/PrestoIndexGenerator.java +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -51,10 +51,9 @@ public void buildStatement() { } sb.append(")"); if (Randomly.getBoolean()) { - sb.append(" WHERE "); PrestoExpression expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); - sb.append(PrestoToStringVisitor.asString(expr)); + appendWhereClause(PrestoToStringVisitor.asString(expr)); } errors.add("already exists!"); } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 51b23eaa4..a19da484f 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -33,11 +33,9 @@ public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLi @Override public void buildStatement() { - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(SQLite3Visitor.asString( + appendWhereClause(SQLite3Visitor.asString( new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } SQLite3Errors.addExpectedExpressionErrors(errors); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 05f4b9a2e..9230f47ba 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -76,10 +76,9 @@ public void buildStatement() { } if (Randomly.getBoolean()) { - sb.append(" WHERE "); String whereClause = SQLite3Visitor.asString( new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression()); - sb.append(whereClause); + appendWhereClause(whereClause); } // ORDER BY and LIMIT are only supported by enabling a compile-time option diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index cbe5fc721..c83d1a6eb 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -41,8 +41,7 @@ public void buildStatement() { sb.append("FROM "); sb.append(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(TiDBVisitor.asString(gen.generateExpression())); + appendWhereClause(TiDBVisitor.asString(gen.generateExpression())); errors.add("Truncated incorrect"); errors.add("Data truncation"); errors.add("Truncated incorrect FLOAT value"); @@ -54,8 +53,7 @@ public void buildStatement() { .collect(Collectors.joining(", "))); } if (Randomly.getBoolean()) { - sb.append(" LIMIT "); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); + appendLimitClause(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); } errors.add("Bad Number"); errors.add("Truncated incorrect"); // https://github.com/pingcap/tidb/issues/24292 diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index af6430b48..dd79670d9 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -35,9 +35,8 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); - sb.append(TiDBVisitor.asString(gen.generateExpression())); + appendWhereClause(TiDBVisitor.asString(gen.generateExpression())); } TiDBErrors.addInsertErrors(errors); } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java index 6ebe0db5f..af99ae4a5 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java @@ -23,11 +23,9 @@ public static SQLQueryAdapter generate(YCQLGlobalState globalState) { @Override public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(YCQLToStringVisitor.asString( + appendWhereClause(YCQLToStringVisitor.asString( new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } YCQLErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java index 66c59a968..dd93b38d0 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -35,10 +35,9 @@ public void buildStatement() { sb.append(table.getName()); appendIndexColumnList(table.getRandomNonEmptyColumnSubset(), false); if (Randomly.getBoolean()) { - sb.append(" WHERE "); YCQLExpression expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(); - sb.append(YCQLToStringVisitor.asString(expr)); + appendWhereClause(YCQLToStringVisitor.asString(expr)); } errors.add("Query timed out after PT2S"); errors.add("Invalid SQL Statement"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java index 6e35c6862..9d3c7427d 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -27,20 +27,13 @@ public void buildStatement() { errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - sb.append("DELETE FROM"); + appendDeleteFromTable(table.getName(), Randomly.getBoolean()); if (Randomly.getBoolean()) { - sb.append(" ONLY"); - } - sb.append(" "); - sb.append(table.getName()); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(YSQLVisitor.asString( + appendWhereClause(YSQLVisitor.asString( YSQLExpressionGenerator.generateExpression(globalState, table.getColumns(), YSQLDataType.BOOLEAN))); } if (Randomly.getBoolean()) { - sb.append(" RETURNING "); - sb.append( + appendReturningClause( YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, table.getColumns()))); } YSQLErrors.addCommonExpressionErrors(errors); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index ac5b3242f..5453746b2 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -93,10 +93,9 @@ public void buildStatement() { sb.append(")"); } if (Randomly.getBoolean()) { - sb.append(" WHERE "); YSQLExpression expr = new YSQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()) .setGlobalState(globalState).generateExpression(YSQLDataType.BOOLEAN); - sb.append(YSQLVisitor.asString(expr)); + appendWhereClause(YSQLVisitor.asString(expr)); } errors.add("already contains data"); // CONCURRENT INDEX failed errors.add("You might need to add explicit type casts"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index 6c5fd4144..1a9e1de76 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -54,10 +54,9 @@ public void buildStatement() { errors.add("but expression is of type"); YSQLErrors.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); YSQLExpression where = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), YSQLDataType.BOOLEAN); - sb.append(YSQLVisitor.asString(where)); + appendWhereClause(YSQLVisitor.asString(where)); } } From 1d6722fe4c14db6cdb92b88c29a2486008a96079 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 10:13:07 +0800 Subject: [PATCH 1223/1284] Remove CnosDB support CnosDB returns EAGAIN ("Tskv: Index: index storage error: Resource temporarily unavailable (os error 11)") on DROP DATABASE under SQLancer's DDL load (cnosdb/cnosdb#2435). The CI job has been red on main for 14+ months, the only published image tags are rolling daily builds (no LTS), and upstream development appears stalled. Pin and retry attempts (#1341) did not help. Remove the CnosDB provider, tests, CI job, and documentation entries. Move CnosDB to the "Previously Supported DBMS" table in CONTRIBUTING.md. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/main.yml | 24 - CONTRIBUTING.md | 2 +- README.md | 2 +- src/check_names.py | 1 - src/sqlancer/Main.java | 2 - src/sqlancer/cnosdb/CnosDBBugs.java | 13 - .../cnosdb/CnosDBComparatorHelper.java | 145 ----- .../cnosdb/CnosDBCompoundDataType.java | 20 - src/sqlancer/cnosdb/CnosDBExpectedError.java | 87 --- src/sqlancer/cnosdb/CnosDBGlobalState.java | 28 - .../cnosdb/CnosDBLoggableFactory.java | 55 -- src/sqlancer/cnosdb/CnosDBOptions.java | 28 - src/sqlancer/cnosdb/CnosDBOracleFactory.java | 39 -- src/sqlancer/cnosdb/CnosDBProvider.java | 123 ----- src/sqlancer/cnosdb/CnosDBSchema.java | 243 -------- .../cnosdb/CnosDBToStringVisitor.java | 278 ---------- src/sqlancer/cnosdb/CnosDBVisitor.java | 102 ---- src/sqlancer/cnosdb/ast/CnosDBAggregate.java | 113 ---- src/sqlancer/cnosdb/ast/CnosDBAlias.java | 35 -- .../cnosdb/ast/CnosDBBetweenOperation.java | 34 -- .../ast/CnosDBBinaryArithmeticOperation.java | 69 --- .../ast/CnosDBBinaryComparisonOperation.java | 57 -- .../ast/CnosDBBinaryLogicalOperation.java | 33 -- .../cnosdb/ast/CnosDBCastOperation.java | 60 -- .../cnosdb/ast/CnosDBColumnValue.java | 27 - .../cnosdb/ast/CnosDBConcatOperation.java | 22 - src/sqlancer/cnosdb/ast/CnosDBConstant.java | 520 ------------------ src/sqlancer/cnosdb/ast/CnosDBExpression.java | 14 - src/sqlancer/cnosdb/ast/CnosDBFunction.java | 30 - .../ast/CnosDBFunctionWithUnknownResult.java | 104 ---- .../cnosdb/ast/CnosDBInOperation.java | 35 -- src/sqlancer/cnosdb/ast/CnosDBJoin.java | 46 -- .../cnosdb/ast/CnosDBLikeOperation.java | 22 - .../cnosdb/ast/CnosDBOrderByTerm.java | 37 -- .../cnosdb/ast/CnosDBPostfixOperation.java | 97 ---- .../cnosdb/ast/CnosDBPostfixText.java | 29 - .../cnosdb/ast/CnosDBPrefixOperation.java | 73 --- src/sqlancer/cnosdb/ast/CnosDBSelect.java | 102 ---- src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java | 28 - src/sqlancer/cnosdb/client/CnosDBClient.java | 110 ---- .../cnosdb/client/CnosDBConnection.java | 27 - .../cnosdb/client/CnosDBException.java | 9 - .../cnosdb/client/CnosDBResultSet.java | 52 -- src/sqlancer/cnosdb/gen/CnosDBCommon.java | 31 -- .../cnosdb/gen/CnosDBExpressionGenerator.java | 461 ---------------- .../cnosdb/gen/CnosDBInsertGenerator.java | 59 -- .../cnosdb/gen/CnosDBTableGenerator.java | 77 --- .../cnosdb/oracle/CnosDBNoRECBase.java | 23 - .../cnosdb/oracle/CnosDBNoRECOracle.java | 171 ------ .../oracle/tlp/CnosDBTLPAggregateOracle.java | 176 ------ .../cnosdb/oracle/tlp/CnosDBTLPBase.java | 112 ---- .../oracle/tlp/CnosDBTLPHavingOracle.java | 65 --- .../oracle/tlp/CnosDBTLPWhereOracle.java | 46 -- .../cnosdb/query/CnosDBOtherQuery.java | 32 -- .../cnosdb/query/CnosDBQueryAdapter.java | 42 -- .../cnosdb/query/CnosDBQueryProvider.java | 6 - .../cnosdb/query/CnosDBSelectQuery.java | 39 -- test/sqlancer/dbms/TestCnosDBNoREC.java | 22 - test/sqlancer/dbms/TestCnosDBTLP.java | 22 - test/sqlancer/dbms/TestConfig.java | 1 - 60 files changed, 2 insertions(+), 4360 deletions(-) delete mode 100644 src/sqlancer/cnosdb/CnosDBBugs.java delete mode 100644 src/sqlancer/cnosdb/CnosDBComparatorHelper.java delete mode 100644 src/sqlancer/cnosdb/CnosDBCompoundDataType.java delete mode 100644 src/sqlancer/cnosdb/CnosDBExpectedError.java delete mode 100644 src/sqlancer/cnosdb/CnosDBGlobalState.java delete mode 100644 src/sqlancer/cnosdb/CnosDBLoggableFactory.java delete mode 100644 src/sqlancer/cnosdb/CnosDBOptions.java delete mode 100644 src/sqlancer/cnosdb/CnosDBOracleFactory.java delete mode 100644 src/sqlancer/cnosdb/CnosDBProvider.java delete mode 100644 src/sqlancer/cnosdb/CnosDBSchema.java delete mode 100644 src/sqlancer/cnosdb/CnosDBToStringVisitor.java delete mode 100644 src/sqlancer/cnosdb/CnosDBVisitor.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBAggregate.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBAlias.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBCastOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBColumnValue.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBConstant.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBExpression.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunction.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBInOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBJoin.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixText.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBSelect.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBClient.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBConnection.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBException.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBResultSet.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBCommon.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java delete mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java delete mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBOtherQuery.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryProvider.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBSelectQuery.java delete mode 100644 test/sqlancer/dbms/TestCnosDBNoREC.java delete mode 100644 test/sqlancer/dbms/TestCnosDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e85c5c219..f0e1b429e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,30 +90,6 @@ jobs: - name: Run Tests run: CITUS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCitus test - cnosdb: - name: DBMS Tests (CnosDB, creation only) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - - name: Build SQLancer - run: mvn -B package -DskipTests=true - - name: Set up CnosDB - run: | - docker pull cnosdb/cnosdb:community-latest - docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done - - name: Run Tests - run: | - CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBNoREC test - sleep 20 - CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBTLP test - clickhouse: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea5baea1c..76b8833ea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,6 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | | Databend | Working | Typed | | | QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | -| CnosDB | Working | Typed | The implementation of CnosDB currently uses Restful API. | | Materialize | Working | Typed | | | Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | | Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | @@ -82,6 +81,7 @@ Some DBMS were once supported but subsequently removed. | Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | | MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | | StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. +| CnosDB | | This implementation was removed because the CnosDB image is unstable under SQLancer's DDL load (see [cnosdb/cnosdb#2435](https://github.com/cnosdb/cnosdb/issues/2435)) and the project appears no longer maintained. | ### Unfixed Bugs diff --git a/README.md b/README.md index 134f47666..f41e32d3c 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Usage: SQLancer [options] [command] [command options] **Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. -**Supported DBMSs.** SQLancer requires DBMS-specific code for each DBMS that it supports. As of January 2025, it provides support for Citus, ClickHouse, CnosDB, CockroachDB, Databend, (Apache) DataFusion, (Apache) Doris, DuckDB, H2, HSQLDB, MariaDB, Materialize, MySQL, OceanBase, PostgreSQL, Presto, QuestDB, SQLite3, TiDB, and YugabyteDB. The extent to which the individual DBMSs are supported [differs](https://github.com/sqlancer/sqlancer/blob/documentation-approaches/CONTRIBUTING.md). +**Supported DBMSs.** SQLancer requires DBMS-specific code for each DBMS that it supports. As of January 2025, it provides support for Citus, ClickHouse, CockroachDB, Databend, (Apache) DataFusion, (Apache) Doris, DuckDB, H2, HSQLDB, MariaDB, Materialize, MySQL, OceanBase, PostgreSQL, Presto, QuestDB, SQLite3, TiDB, and YugabyteDB. The extent to which the individual DBMSs are supported [differs](https://github.com/sqlancer/sqlancer/blob/documentation-approaches/CONTRIBUTING.md). # Approaches and Papers diff --git a/src/check_names.py b/src/check_names.py index f2ab346c6..453580f88 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -35,7 +35,6 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files: dict[str:List[str]] = dict() name_to_files["Citus"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "citus")) name_to_files["ClickHouse"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "clickhouse")) - name_to_files["CnosDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cnosdb")) name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) name_to_files["DataFusion"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "datafusion")) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index faf35e3c9..f273f5b95 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -26,7 +26,6 @@ import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.cnosdb.CnosDBProvider; import sqlancer.cockroachdb.CockroachDBProvider; import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; @@ -750,7 +749,6 @@ private static void checkForIssue799(List> providers) "No DBMS implementations (i.e., instantiations of the DatabaseProvider class) were found. You likely ran into an issue described in https://github.com/sqlancer/sqlancer/issues/799. As a workaround, I now statically load all supported providers as of June 7, 2023."); providers.add(new CitusProvider()); providers.add(new ClickHouseProvider()); - providers.add(new CnosDBProvider()); providers.add(new CockroachDBProvider()); providers.add(new DatabendProvider()); providers.add(new DorisProvider()); diff --git a/src/sqlancer/cnosdb/CnosDBBugs.java b/src/sqlancer/cnosdb/CnosDBBugs.java deleted file mode 100644 index 4e6eb96e9..000000000 --- a/src/sqlancer/cnosdb/CnosDBBugs.java +++ /dev/null @@ -1,13 +0,0 @@ -package sqlancer.cnosdb; - -public final class CnosDBBugs { - - // https://github.com/cnosdb/cnosdb/issues/786 - public static final boolean BUG786 = true; - - // https://github.com/apache/arrow-rs/issues/3547 - public static final boolean BUG3547 = true; - - private CnosDBBugs() { - } -} diff --git a/src/sqlancer/cnosdb/CnosDBComparatorHelper.java b/src/sqlancer/cnosdb/CnosDBComparatorHelper.java deleted file mode 100644 index 46b6ba615..000000000 --- a/src/sqlancer/cnosdb/CnosDBComparatorHelper.java +++ /dev/null @@ -1,145 +0,0 @@ -package sqlancer.cnosdb; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.function.UnaryOperator; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.cnosdb.query.CnosDBSelectQuery; -import sqlancer.common.query.ExpectedErrors; - -public final class CnosDBComparatorHelper { - - private CnosDBComparatorHelper() { - } - - public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, - CnosDBGlobalState state) throws Exception { - if (state.getOptions().logEachSelect()) { - // TODO: refactor me - state.getLogger().writeCurrent(queryString); - try { - state.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, errors); - List result = new ArrayList<>(); - CnosDBResultSet resultSet; - try { - q.executeAndGet(state); - resultSet = q.getResultSet(); - if (resultSet == null) { - throw new AssertionError(q); - } - while (resultSet.next()) { - result.add(resultSet.getString(1)); - } - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw e; - } - if (e instanceof NumberFormatException) { - throw new IgnoreMeException(); - } - if (e.getMessage() == null) { - throw new AssertionError(queryString, e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(queryString, e); - } - - return result; - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - String originalQueryString, List combinedString, CnosDBGlobalState state) { - if (resultSet.size() != secondResultSet.size()) { - String queryFormatString = "-- %s;\n-- cardinality: %d"; - String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); - String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), - secondResultSet.size()); - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", - resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); - throw new AssertionError(assertionMessage); - } - - Set firstHashSet = new HashSet<>(resultSet); - Set secondHashSet = new HashSet<>(secondResultSet); - - if (!firstHashSet.equals(secondHashSet)) { - Set firstResultSetMisses = new HashSet<>(firstHashSet); - firstResultSetMisses.removeAll(secondHashSet); - Set secondResultSetMisses = new HashSet<>(secondHashSet); - secondResultSetMisses.removeAll(firstHashSet); - String queryFormatString = "-- %s;\n-- misses: %s"; - String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); - String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), - secondResultSetMisses); - // update the SELECT queries to be logged at the bottom of the error log file - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", - firstQueryString, secondQueryString); - throw new AssertionError(assertionMessage); - } - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - String originalQueryString, List combinedString, CnosDBGlobalState state, - UnaryOperator canonicalizationRule) { - // Overloaded version of assumeResultSetsAreEqual that takes a canonicalization function which is applied to - // both result sets before their comparison. - List canonicalizedResultSet = resultSet.stream().map(canonicalizationRule).collect(Collectors.toList()); - List canonicalizedSecondResultSet = secondResultSet.stream().map(canonicalizationRule) - .collect(Collectors.toList()); - assumeResultSetsAreEqual(canonicalizedResultSet, canonicalizedSecondResultSet, originalQueryString, - combinedString, state); - } - - public static List getCombinedResultSet(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, - ExpectedErrors errors) throws Exception { - List secondResultSet; - if (asUnion) { - String unionString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL " - + thirdQueryString; - combinedString.add(unionString); - secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); - } else { - secondResultSet = new ArrayList<>(); - secondResultSet.addAll(getResultSetFirstColumnAsString(firstQueryString, errors, state)); - secondResultSet.addAll(getResultSetFirstColumnAsString(secondQueryString, errors, state)); - secondResultSet.addAll(getResultSetFirstColumnAsString(thirdQueryString, errors, state)); - combinedString.add(firstQueryString); - combinedString.add(secondQueryString); - combinedString.add(thirdQueryString); - } - return secondResultSet; - } - - public static List getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, - ExpectedErrors errors) throws Exception { - String unionString; - if (asUnion) { - unionString = firstQueryString + " UNION " + secondQueryString + " UNION " + thirdQueryString; - } else { - unionString = "SELECT DISTINCT * FROM (" + firstQueryString + " UNION ALL " + secondQueryString - + " UNION ALL " + thirdQueryString + ")"; - } - List secondResultSet; - combinedString.add(unionString); - secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); - return secondResultSet; - } -} diff --git a/src/sqlancer/cnosdb/CnosDBCompoundDataType.java b/src/sqlancer/cnosdb/CnosDBCompoundDataType.java deleted file mode 100644 index 034f0fc90..000000000 --- a/src/sqlancer/cnosdb/CnosDBCompoundDataType.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public final class CnosDBCompoundDataType { - - private final CnosDBDataType dataType; - - private CnosDBCompoundDataType(CnosDBDataType dataType) { - this.dataType = dataType; - } - - public static CnosDBCompoundDataType create(CnosDBDataType type) { - return new CnosDBCompoundDataType(type); - } - - public CnosDBDataType getDataType() { - return dataType; - } -} diff --git a/src/sqlancer/cnosdb/CnosDBExpectedError.java b/src/sqlancer/cnosdb/CnosDBExpectedError.java deleted file mode 100644 index 61dba101b..000000000 --- a/src/sqlancer/cnosdb/CnosDBExpectedError.java +++ /dev/null @@ -1,87 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.common.query.ExpectedErrors; - -public final class CnosDBExpectedError { - - private CnosDBExpectedError() { - } - - public static List getExpectedErrors() { - ArrayList errors = new ArrayList<>(); - - errors.add("have the same name. Consider aliasing"); - errors.add( - "error: Optimizer rule 'projection_push_down' failed due to unexpected error: Schema error: Schema contains duplicate qualified field name"); - errors.add("Projection references non-aggregate values:"); - errors.add("External err: Schema error: No field named"); - errors.add( - "Optimizer rule 'common_sub_expression_eliminate' failed due to unexpected error: Schema error: No field named"); - errors.add("Binary"); - errors.add("Invalid pattern in LIKE expression"); - errors.add("If the projection contains the time column, it must contain the field column."); - errors.add("Schema error: No field named"); - errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:"); - errors.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error"); - errors.add("Schema error: No field named "); - errors.add("err: External err: Schema error: No field named"); - errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error"); - errors.add("Csv error: CSV Writer does not support List"); - errors.add("This feature is not implemented: cross join."); - errors.add("Execution error: field position must be greater than zero"); - errors.add("First argument of `DATE_PART` must be non-null scalar Utf8"); - errors.add("Cannot create filter with non-boolean predicate 'NULL' returning Null"); - errors.add("requested character too large for encoding."); - errors.add("Can not find compatible types to compare Boolean with [Utf8]."); - errors.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT"); - errors.add("HAVING clause references non-aggregate values:"); - errors.add("Cannot create filter with non-boolean predicate"); - errors.add("negative substring length not allowed"); - errors.add("The function Sum does not support inputs of type Boolean."); - errors.add("The function Avg does not support inputs of type Boolean."); - errors.add("Percentile value must be between 0.0 and 1.0 inclusive"); - errors.add("Date part '' not supported"); - errors.add("Min/Max accumulator not implemented for type Boolean."); - errors.add("meta need get_series_id_by_filter"); - errors.add("Arrow: Cast error:"); - errors.add("Arrow error: Cast error:"); - errors.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:"); - errors.add("Arrow error: Divide by zero error"); - errors.add("desired percentile argument must be float literal"); - errors.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)"); - errors.add("Execution error: Date part"); - errors.add("Physical plan does not support logical expression MIN(Boolean"); - errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64"); - errors.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64."); - errors.add("Data type UInt64 not supported for binary operation '#' on dyn arrays."); - errors.add("Arrow: Divide by zero error"); - errors.add("The function ApproxPercentileCont does not support inputs of type Null."); - errors.add("can't be evaluated because there isn't a common type to coerce the types to"); - errors.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug"); - errors.add("The function ApproxMedian does not support inputs of type Null."); - errors.add("null character not permitted."); - errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Null."); - errors.add("This feature is not implemented"); - errors.add("The function Avg does not support inputs of type Null."); - errors.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]"); - errors.add( - "Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed."); - errors.add("Coercion from"); - - errors.add("Error parsing timestamp"); - errors.add("lpad requested length"); - errors.add("rpad requested length"); - errors.add("No function matches the given name and argument types"); - return errors; - } - - public static ExpectedErrors expectedErrors() { - ExpectedErrors res = new ExpectedErrors(); - res.addAll(getExpectedErrors()); - return res; - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBGlobalState.java b/src/sqlancer/cnosdb/CnosDBGlobalState.java deleted file mode 100644 index 9f34e03a5..000000000 --- a/src/sqlancer/cnosdb/CnosDBGlobalState.java +++ /dev/null @@ -1,28 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.ExecutionTimer; -import sqlancer.GlobalState; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.query.Query; - -public class CnosDBGlobalState extends GlobalState { - - @Override - protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { - boolean logExecutionTime = getOptions().logExecutionTime(); - if (success && getOptions().printSucceedingStatements()) { - System.out.println(q.getQueryString()); - } - if (logExecutionTime) { - getLogger().writeCurrent(" -- " + timer.end().asString()); - } - if (q.couldAffectSchema()) { - updateSchema(); - } - } - - @Override - public CnosDBSchema readSchema() throws Exception { - return CnosDBSchema.fromConnection(getConnection()); - } -} diff --git a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java deleted file mode 100644 index 407621c8b..000000000 --- a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java +++ /dev/null @@ -1,55 +0,0 @@ -package sqlancer.cnosdb; - -import java.io.PrintWriter; -import java.io.StringWriter; - -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.cnosdb.query.CnosDBQueryAdapter; -import sqlancer.common.log.Loggable; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.log.LoggedString; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; - -public class CnosDBLoggableFactory extends LoggableFactory { - - @Override - protected Loggable createLoggable(String input, String suffix) { - String completeString = input; - if (!input.endsWith(";")) { - completeString += ";"; - } - if (suffix != null && !suffix.isEmpty()) { - completeString += suffix; - } - return new LoggedString(completeString); - } - - @Override - public CnosDBQueryAdapter getQueryForStateToReproduce(String queryString) { - return new CnosDBOtherQuery(queryString, CnosDBExpectedError.expectedErrors()); - } - - @Override - public CnosDBQueryAdapter commentOutQuery(Query query) { - String queryString = query.getLogString(); - String newQueryString = "-- " + queryString; - ExpectedErrors errors = new ExpectedErrors(); - return new CnosDBOtherQuery(newQueryString, errors); - } - - @Override - protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { - String sb = "-- Time: " + time + "\n" + "-- Database: " + databaseName + "\n" + "-- Database version: " - + databaseVersion + "\n" + "-- seed value: " + seedValue + "\n"; - return new LoggedString(sb); - } - - @Override - public Loggable convertStacktraceToLoggable(Throwable throwable) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - throwable.printStackTrace(pw); - return new LoggedString("--" + sw.toString().replace("\n", "\n--")); - } -} diff --git a/src/sqlancer/cnosdb/CnosDBOptions.java b/src/sqlancer/cnosdb/CnosDBOptions.java deleted file mode 100644 index f101c2d38..000000000 --- a/src/sqlancer/cnosdb/CnosDBOptions.java +++ /dev/null @@ -1,28 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; - -@Parameters(separators = "=", commandDescription = "CnosDB (default port: " + CnosDBOptions.DEFAULT_PORT - + ", default host: " + CnosDBOptions.DEFAULT_HOST + ")") -public class CnosDBOptions implements DBMSSpecificOptions { - - public static final String DEFAULT_HOST = "localhost"; - public static final int DEFAULT_PORT = 31001; - - @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for CnosDB") - public List oracle = List.of(CnosDBOracleFactory.QUERY_PARTITIONING); - - @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the CnosDB", arity = 1) - public String connectionURL = String.format("http://%s:%d", CnosDBOptions.DEFAULT_HOST, CnosDBOptions.DEFAULT_PORT); - - @Override - public List getTestOracleFactory() { - return oracle; - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBOracleFactory.java b/src/sqlancer/cnosdb/CnosDBOracleFactory.java deleted file mode 100644 index 7cb9c4fc6..000000000 --- a/src/sqlancer/cnosdb/CnosDBOracleFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.OracleFactory; -import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPAggregateOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPHavingOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPWhereOracle; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; - -public enum CnosDBOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBNoRECOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - List> oracles = new ArrayList<>(); - oracles.add(new CnosDBTLPWhereOracle(globalState)); - oracles.add(new CnosDBTLPHavingOracle(globalState)); - oracles.add(new CnosDBTLPAggregateOracle(globalState)); - return new CompositeTestOracle<>(oracles, globalState); - } - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBProvider.java b/src/sqlancer/cnosdb/CnosDBProvider.java deleted file mode 100644 index 8b69c53b3..000000000 --- a/src/sqlancer/cnosdb/CnosDBProvider.java +++ /dev/null @@ -1,123 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.Objects; - -import com.google.auto.service.AutoService; - -import sqlancer.AbstractAction; -import sqlancer.DatabaseProvider; -import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; -import sqlancer.Randomly; -import sqlancer.StatementExecutor; -import sqlancer.cnosdb.client.CnosDBClient; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.cnosdb.gen.CnosDBInsertGenerator; -import sqlancer.cnosdb.gen.CnosDBTableGenerator; -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.cnosdb.query.CnosDBQueryProvider; -import sqlancer.common.log.LoggableFactory; - -@AutoService(DatabaseProvider.class) -public class CnosDBProvider extends ProviderAdapter { - - protected String username; - protected String password; - protected String host; - protected int port; - protected String databaseName; - - public CnosDBProvider() { - super(CnosDBGlobalState.class, CnosDBOptions.class); - } - - protected CnosDBProvider(Class globalClass, Class optionClass) { - super(globalClass, optionClass); - } - - protected static int mapActions(CnosDBGlobalState globalState, Action a) { - Randomly r = globalState.getRandomly(); - int nrPerformed; - if (Objects.requireNonNull(a) == Action.INSERT) { - nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - } else { - throw new AssertionError(a); - } - return nrPerformed; - - } - - @Override - protected void checkViewsAreValid(CnosDBGlobalState globalState) { - } - - @Override - public void generateDatabase(CnosDBGlobalState globalState) throws Exception { - createTables(globalState, Randomly.fromOptions(4, 5, 6)); - prepareTables(globalState); - - } - - @Override - public CnosDBConnection createDatabase(CnosDBGlobalState globalState) throws Exception { - - username = globalState.getOptions().getUserName(); - password = globalState.getOptions().getPassword(); - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); - databaseName = globalState.getDatabaseName(); - CnosDBClient client = new CnosDBClient(host, port, username, password, databaseName); - CnosDBConnection connection = new CnosDBConnection(client); - client.execute("DROP DATABASE IF EXISTS " + databaseName); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - client.execute("CREATE DATABASE " + databaseName); - globalState.getState().logStatement("CREATE DATABASE " + databaseName); - - return connection; - } - - protected void createTables(CnosDBGlobalState globalState, int numTables) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < numTables) { - String tableName = String.format("m%d", globalState.getSchema().getDatabaseTables().size()); - CnosDBOtherQuery createTable = CnosDBTableGenerator.generate(tableName); - globalState.executeStatement(createTable); - } - } - - protected void prepareTables(CnosDBGlobalState globalState) throws Exception { - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - CnosDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public String getDBMSName() { - return "CnosDB".toLowerCase(); - } - - @Override - public LoggableFactory getLoggableFactory() { - return new CnosDBLoggableFactory(); - } - - public enum Action implements AbstractAction { - INSERT(CnosDBInsertGenerator::insert); - - private final CnosDBQueryProvider sqlQueryProvider; - - Action(CnosDBQueryProvider sqlQueryProvider) { - this.sqlQueryProvider = sqlQueryProvider; - } - - @Override - public CnosDBOtherQuery getQuery(CnosDBGlobalState state) throws Exception { - return new CnosDBOtherQuery(sqlQueryProvider.getQuery(state).getQueryString(), - CnosDBExpectedError.expectedErrors()); - } - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBSchema.java b/src/sqlancer/cnosdb/CnosDBSchema.java deleted file mode 100644 index 022969ce5..000000000 --- a/src/sqlancer/cnosdb/CnosDBSchema.java +++ /dev/null @@ -1,243 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import sqlancer.Randomly; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.common.schema.AbstractRowValue; -import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.common.schema.AbstractTables; -import sqlancer.common.schema.TableIndex; - -public class CnosDBSchema extends AbstractSchema { - - private final String databaseName; - - public CnosDBSchema(List databaseTables, String databaseName) { - super(databaseTables); - this.databaseName = databaseName; - } - - public static CnosDBDataType getColumnType(String typeString) { - switch (typeString.toLowerCase()) { - case "bigint": - return CnosDBDataType.INT; - case "boolean": - return CnosDBDataType.BOOLEAN; - case "string": - return CnosDBDataType.STRING; - case "double": - return CnosDBDataType.DOUBLE; - case "bigint unsigned": - case "unsigned": - return CnosDBDataType.UINT; - case "timestamp(nanosecond)": - return CnosDBDataType.TIMESTAMP; - default: - throw new AssertionError(typeString); - } - } - - public static CnosDBSchema fromConnection(CnosDBConnection con) throws Exception { - CnosDBResultSet tablesRes = con.getClient().executeQuery("SHOW TABLES"); - - List tables = new ArrayList<>(); - while (tablesRes.next()) { - String tableName = tablesRes.getString(1); - List columns = getTableColumns(con, tableName); - tables.add(new CnosDBTable(tableName, columns)); - } - - return new CnosDBSchema(tables, con.getClient().getDatabase()); - } - - protected static List getTableColumns(CnosDBConnection con, String tableName) throws Exception { - CnosDBResultSet columnsRes = con.getClient().executeQuery("DESCRIBE TABLE " + tableName); - List columns = new ArrayList<>(); - CnosDBTable table = new CnosDBTable(tableName, columns); - while (columnsRes.next()) { - String columnName = columnsRes.getString(1); - String columnType = columnsRes.getString(3).toLowerCase(); - CnosDBDataType dataType = CnosDBSchema.getColumnType(columnsRes.getString(2)); - CnosDBColumn column; - if (columnType.contentEquals("time")) { - column = new CnosDBTimeColumn(); - } else if (columnType.contentEquals("tag")) { - column = new CnosDBTagColumn(columnName); - } else { - column = new CnosDBFieldColumn(columnName, dataType); - } - column.setTable(table); - columns.add(column); - } - - return columns; - } - - public CnosDBTables getRandomTableNonEmptyTables() { - return new CnosDBTables(Randomly.nonEmptySubset(getDatabaseTables())); - } - - public String getDatabaseName() { - return databaseName; - } - - public enum CnosDBDataType { - INT, BOOLEAN, STRING, DOUBLE, UINT, TIMESTAMP; - - public static CnosDBDataType getRandomType() { - return Randomly.fromOptions(values()); - } - - public static CnosDBDataType getRandomTypeWithoutTimeStamp() { - List dataTypes = new ArrayList<>(Arrays.asList(values())); - dataTypes.remove(TIMESTAMP); - return Randomly.fromList(dataTypes); - } - } - - public static class CnosDBColumn extends AbstractTableColumn { - - public CnosDBColumn(String name, CnosDBDataType columnType) { - super(name, null, columnType); - } - - public static CnosDBColumn createDummy(String name) { - return new CnosDBColumn(name, CnosDBDataType.INT); - } - - } - - public static class CnosDBTagColumn extends CnosDBColumn { - public CnosDBTagColumn(String name) { - super(name, CnosDBDataType.STRING); - } - } - - public static class CnosDBTimeColumn extends CnosDBColumn { - public CnosDBTimeColumn() { - super("TIME", CnosDBDataType.TIMESTAMP); - } - } - - public static class CnosDBFieldColumn extends CnosDBColumn { - public CnosDBFieldColumn(String name, CnosDBDataType columnType) { - super(name, columnType); - assert columnType != CnosDBDataType.TIMESTAMP; - } - } - - public static class CnosDBTables extends AbstractTables { - - public CnosDBTables(List tables) { - super(tables); - } - - public CnosDBRowValue getRandomRowValue(CnosDBConnection con) { - return null; - } - - public List getRandomColumnsWithOnlyOneField() { - ArrayList res = new ArrayList<>(); - this.getTables().forEach(table -> res.addAll(table.getRandomColumnsWithOnlyOneField())); - return res; - } - - } - - public static class CnosDBRowValue extends AbstractRowValue { - - protected CnosDBRowValue(CnosDBTables tables, Map values) { - super(tables, values); - } - - } - - public static class CnosDBTable extends AbstractTable { - - public CnosDBTable(String tableName, List columns) { - super(tableName, columns, null, false); - } - - @Override - public List getColumns() { - List res = super.getColumns(); - boolean hasTime = false; - for (CnosDBColumn column : res) { - if (column instanceof CnosDBTimeColumn) { - hasTime = true; - break; - } - } - assert hasTime; - - return res; - } - - public List getRandomColumnsWithOnlyOneField() { - ArrayList res = new ArrayList<>(); - boolean hasField = false; - for (CnosDBColumn column : getColumns()) { - if (column instanceof CnosDBTagColumn && Randomly.getBoolean()) { - res.add(column); - } else if (column instanceof CnosDBFieldColumn && !hasField) { - res.add(column); - hasField = true; - } - } - return res; - } - - // SELECT COUNT(*) FROM table; - @Override - public long getNrRows(CnosDBGlobalState globalState) { - long res; - try { - CnosDBResultSet tableCountRes = globalState.getConnection().getClient() - .executeQuery("SELECT COUNT(time) FROM " + this.name); - tableCountRes.next(); - res = tableCountRes.getLong(1); - } catch (Exception e) { - res = 0; - } - return res; - } - - @Override - public List getRandomNonEmptyColumnSubset() { - List selectedColumns = new ArrayList<>(); - ArrayList remainingColumns = new ArrayList<>(this.getColumns()); - - remainingColumns.removeIf(column -> column instanceof CnosDBTimeColumn); - CnosDBTimeColumn timeColumn = new CnosDBTimeColumn(); - timeColumn.setTable(this); - selectedColumns.add(timeColumn); - - remainingColumns.stream().filter(column -> column instanceof CnosDBTagColumn).findFirst().ifPresent(tag -> { - selectedColumns.add(tag); - remainingColumns.remove(tag); - }); - - remainingColumns.stream().filter(column -> column instanceof CnosDBFieldColumn).findFirst() - .ifPresent(field -> { - selectedColumns.add(field); - remainingColumns.remove(field); - }); - - int nr = Math.min(Randomly.smallNumber() + 1, remainingColumns.size()); - for (int i = 0; i < nr; i++) { - selectedColumns - .add(remainingColumns.remove((int) Randomly.getNotCachedInteger(0, remainingColumns.size()))); - } - return selectedColumns; - } - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java deleted file mode 100644 index 388e2ccd8..000000000 --- a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java +++ /dev/null @@ -1,278 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.Randomly; -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBBetweenOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBFunction; -import sqlancer.cnosdb.ast.CnosDBInOperation; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBLikeOperation; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixText; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.ast.CnosDBSimilarTo; -import sqlancer.common.visitor.BinaryOperation; -import sqlancer.common.visitor.ToStringVisitor; - -public final class CnosDBToStringVisitor extends ToStringVisitor implements CnosDBVisitor { - - @Override - public void visitSpecific(CnosDBExpression expr) { - CnosDBVisitor.super.visit(expr); - } - - @Override - public void visit(CnosDBConstant constant) { - sb.append(constant.getTextRepresentation()); - } - - @Override - public String get() { - return sb.toString(); - } - - @Override - public void visit(CnosDBPostfixOperation op) { - sb.append("("); - visit(op.getExpression()); - sb.append(")"); - sb.append(" "); - sb.append(op.getOperatorTextRepresentation()); - } - - @Override - public void visit(CnosDBColumnValue c) { - sb.append(c.getColumn().getFullQualifiedName()); - } - - @Override - public void visit(CnosDBPrefixOperation op) { - sb.append(op.getTextRepresentation()); - sb.append(" ("); - visit(op.getExpression()); - sb.append(")"); - } - - @Override - public void visit(CnosDBFromTable from) { - sb.append(from.getTable().getName()); - } - - @Override - public void visit(CnosDBSubquery subquery) { - sb.append("("); - visit(subquery.getSelect()); - sb.append(") AS "); - sb.append(subquery.getName()); - } - - @Override - public void visit(CnosDBSelect s) { - sb.append("SELECT "); - switch (s.getSelectOption()) { - case DISTINCT: - sb.append("DISTINCT "); - if (s.getDistinctOnClause() != null) { - sb.append("ON ("); - visit(s.getDistinctOnClause()); - sb.append(") "); - } - break; - case ALL: - sb.append(Randomly.fromOptions("ALL ", "")); - break; - default: - throw new AssertionError(); - } - if (s.getFetchColumns() == null) { - sb.append("*"); - } else { - visit(s.getFetchColumns()); - } - sb.append(" FROM "); - visit(s.getFromList()); - - for (CnosDBJoin j : s.getJoinClauses()) { - sb.append(" "); - switch (j.getType()) { - case INNER: - if (Randomly.getBoolean()) { - sb.append("INNER "); - } - sb.append("JOIN"); - break; - case LEFT: - sb.append("LEFT OUTER JOIN"); - break; - case RIGHT: - sb.append("RIGHT OUTER JOIN"); - break; - case FULL: - sb.append("FULL OUTER JOIN"); - break; - // case CROSS: - // sb.append("CROSS JOIN"); - // break; - default: - throw new AssertionError(j.getType()); - } - sb.append(" "); - visit(j.getTableReference()); - // if (j.getType() != CnosDBJoinType.CROSS) { - sb.append(" ON "); - visit(j.getOnClause()); - // } - } - - if (s.getWhereClause() != null) { - sb.append(" WHERE "); - visit(s.getWhereClause()); - } - if (!s.getGroupByExpressions().isEmpty()) { - sb.append(" GROUP BY "); - visit(s.getGroupByExpressions()); - } - if (s.getHavingClause() != null) { - sb.append(" HAVING "); - visit(s.getHavingClause()); - - } - if (!s.getOrderByClauses().isEmpty()) { - sb.append(" ORDER BY "); - visit(s.getOrderByClauses()); - } - if (s.getLimitClause() != null) { - sb.append(" LIMIT "); - visit(s.getLimitClause()); - } - - if (s.getOffsetClause() != null) { - sb.append(" OFFSET "); - visit(s.getOffsetClause()); - } - } - - @Override - public void visit(CnosDBOrderByTerm op) { - visit(op.getExpr()); - sb.append(" "); - sb.append(op.getOrder()); - } - - @Override - public void visit(CnosDBFunction f) { - sb.append(f.getFunctionName()); - sb.append("("); - int i = 0; - for (CnosDBExpression arg : f.getArguments()) { - if (i++ != 0) { - sb.append(", "); - } - visit(arg); - } - sb.append(")"); - } - - @Override - public void visit(CnosDBCastOperation cast) { - sb.append("CAST( "); - visit(cast.getExpression()); - sb.append(" AS "); - appendType(cast); - sb.append(")"); - } - - private void appendType(CnosDBCastOperation cast) { - CnosDBCompoundDataType compoundType = cast.getCompoundType(); - switch (compoundType.getDataType()) { - case BOOLEAN: - sb.append("BOOLEAN"); - break; - case INT: - sb.append("BIGINT"); - break; - case STRING: - sb.append(Randomly.fromOptions("STRING")); - break; - case DOUBLE: - sb.append("DOUBLE"); - break; - case UINT: - sb.append("BIGINT UNSIGNED"); - break; - case TIMESTAMP: - sb.append("TIMESTAMP"); - break; - - default: - throw new AssertionError(cast.getType()); - } - } - - @Override - public void visit(CnosDBBetweenOperation op) { - sb.append("("); - visit(op.getExpr()); - sb.append(") BETWEEN ("); - visit(op.getLeft()); - sb.append(") AND ("); - visit(op.getRight()); - sb.append(")"); - } - - @Override - public void visit(CnosDBInOperation op) { - sb.append("("); - visit(op.getExpr()); - sb.append(")"); - if (!op.isTrue()) { - sb.append(" NOT"); - } - sb.append(" IN ("); - visit(op.getListElements()); - sb.append(")"); - } - - @Override - public void visit(CnosDBPostfixText op) { - visit(op.getExpr()); - sb.append(op.getText()); - } - - @Override - public void visit(CnosDBAggregate op) { - sb.append(op.getFunction()); - sb.append("("); - visit(op.getArgs()); - sb.append(")"); - } - - @Override - public void visit(CnosDBSimilarTo op) { - sb.append("("); - visit(op.getString()); - sb.append(" SIMILAR TO "); - visit(op.getSimilarTo()); - sb.append(")"); - } - - @Override - public void visit(CnosDBBinaryLogicalOperation op) { - super.visit((BinaryOperation) op); - } - - @Override - public void visit(CnosDBLikeOperation op) { - super.visit((BinaryOperation) op); - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBVisitor.java b/src/sqlancer/cnosdb/CnosDBVisitor.java deleted file mode 100644 index 7c1af7224..000000000 --- a/src/sqlancer/cnosdb/CnosDBVisitor.java +++ /dev/null @@ -1,102 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBBetweenOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBFunction; -import sqlancer.cnosdb.ast.CnosDBInOperation; -import sqlancer.cnosdb.ast.CnosDBLikeOperation; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixText; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.ast.CnosDBSimilarTo; - -public interface CnosDBVisitor { - - static String asString(CnosDBExpression expr) { - CnosDBToStringVisitor visitor = new CnosDBToStringVisitor(); - visitor.visit(expr); - return visitor.get(); - } - - void visit(CnosDBConstant constant); - - void visit(CnosDBPostfixOperation op); - - void visit(CnosDBColumnValue c); - - void visit(CnosDBPrefixOperation op); - - void visit(CnosDBSelect op); - - void visit(CnosDBOrderByTerm op); - - void visit(CnosDBFunction f); - - void visit(CnosDBCastOperation cast); - - void visit(CnosDBBetweenOperation op); - - void visit(CnosDBInOperation op); - - void visit(CnosDBPostfixText op); - - void visit(CnosDBAggregate op); - - void visit(CnosDBFromTable from); - - void visit(CnosDBSubquery subquery); - - void visit(CnosDBBinaryLogicalOperation op); - - void visit(CnosDBLikeOperation op); - - void visit(CnosDBSimilarTo op); - - default void visit(CnosDBExpression expression) { - if (expression instanceof CnosDBConstant) { - visit((CnosDBConstant) expression); - } else if (expression instanceof CnosDBPostfixOperation) { - visit((CnosDBPostfixOperation) expression); - } else if (expression instanceof CnosDBColumnValue) { - visit((CnosDBColumnValue) expression); - } else if (expression instanceof CnosDBPrefixOperation) { - visit((CnosDBPrefixOperation) expression); - } else if (expression instanceof CnosDBSelect) { - visit((CnosDBSelect) expression); - } else if (expression instanceof CnosDBOrderByTerm) { - visit((CnosDBOrderByTerm) expression); - } else if (expression instanceof CnosDBFunction) { - visit((CnosDBFunction) expression); - } else if (expression instanceof CnosDBCastOperation) { - visit((CnosDBCastOperation) expression); - } else if (expression instanceof CnosDBBetweenOperation) { - visit((CnosDBBetweenOperation) expression); - } else if (expression instanceof CnosDBInOperation) { - visit((CnosDBInOperation) expression); - } else if (expression instanceof CnosDBAggregate) { - visit((CnosDBAggregate) expression); - } else if (expression instanceof CnosDBPostfixText) { - visit((CnosDBPostfixText) expression); - } else if (expression instanceof CnosDBSimilarTo) { - visit((CnosDBSimilarTo) expression); - } else if (expression instanceof CnosDBFromTable) { - visit((CnosDBFromTable) expression); - } else if (expression instanceof CnosDBSubquery) { - visit((CnosDBSubquery) expression); - } else if (expression instanceof CnosDBLikeOperation) { - visit((CnosDBLikeOperation) expression); - } else { - throw new AssertionError(expression); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAggregate.java b/src/sqlancer/cnosdb/ast/CnosDBAggregate.java deleted file mode 100644 index df30717b4..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBAggregate.java +++ /dev/null @@ -1,113 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBBugs; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; -import sqlancer.common.ast.FunctionNode; - -public class CnosDBAggregate extends FunctionNode - implements CnosDBExpression { - - public CnosDBAggregate(List args, CnosDBAggregateFunction func) { - super(func, args); - } - - public enum CnosDBAggregateFunction { - AVG(CnosDBDataType.DOUBLE), - MAX(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, - CnosDBDataType.UINT), - MIN(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, - CnosDBDataType.UINT), - COUNT(CnosDBDataType.INT) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.getRandomType() }; - } - }, - SUM(CnosDBDataType.INT, CnosDBDataType.DOUBLE, CnosDBDataType.UINT), APPROX_MEDIAN(CnosDBDataType.DOUBLE), - - VAR(CnosDBDataType.DOUBLE), VAR_SAMP(CnosDBDataType.DOUBLE), VAR_POP(CnosDBDataType.DOUBLE), - STDDEV(CnosDBDataType.DOUBLE), STDDEV_SAMP(CnosDBDataType.DOUBLE), STDDEV_POP(CnosDBDataType.DOUBLE), - COVAR(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - COVAR_SAMP(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.INT }; - } - }, - CORR(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - COVAR_POP(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - - APPROX_PERCENTILE_CONT(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - APPROX_PERCENTILE_CONT_WITH_WEIGHT(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - APPROX_DISTINCT(CnosDBDataType.UINT), GROUPING(CnosDBDataType.INT), ARRAY_AGG(CnosDBDataType.STRING); - - private final CnosDBDataType[] supportedReturnTypes; - - CnosDBAggregateFunction(CnosDBDataType... supportedReturnTypes) { - this.supportedReturnTypes = supportedReturnTypes.clone(); - } - - public static List getAggregates(CnosDBDataType type) { - List res = Stream.of(values()).filter(p -> p.supportsReturnType(type)) - .collect(Collectors.toList()); - if (CnosDBBugs.BUG786) { - res.removeAll(List.of(VAR, VAR_POP, VAR_SAMP, STDDEV, STDDEV_POP, STDDEV_SAMP, CORR, COVAR, COVAR_POP, - COVAR_SAMP, APPROX_PERCENTILE_CONT_WITH_WEIGHT, APPROX_DISTINCT, APPROX_PERCENTILE_CONT, - APPROX_PERCENTILE_CONT_WITH_WEIGHT, GROUPING, ARRAY_AGG)); - } - - return res; - } - - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { returnType }; - } - - public boolean supportsReturnType(CnosDBDataType returnType) { - return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType) - || supportedReturnTypes.length == 0; - } - - public CnosDBDataType getRandomReturnType() { - if (supportedReturnTypes.length == 0) { - return Randomly.fromOptions(CnosDBDataType.getRandomType()); - } else { - return Randomly.fromOptions(supportedReturnTypes); - } - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAlias.java b/src/sqlancer/cnosdb/ast/CnosDBAlias.java deleted file mode 100644 index 86bba199f..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBAlias.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.common.visitor.UnaryOperation; - -public class CnosDBAlias implements UnaryOperation, CnosDBExpression { - - private final CnosDBExpression expr; - private final String alias; - - public CnosDBAlias(CnosDBExpression expr, String alias) { - this.expr = expr; - this.alias = alias; - } - - @Override - public CnosDBExpression getExpression() { - return expr; - } - - @Override - public String getOperatorRepresentation() { - return " as " + alias; - } - - @Override - public OperatorKind getOperatorKind() { - return OperatorKind.POSTFIX; - } - - @Override - public boolean omitBracketsWhenPrinting() { - return true; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java deleted file mode 100644 index d0addced1..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java +++ /dev/null @@ -1,34 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public final class CnosDBBetweenOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final CnosDBExpression left; - private final CnosDBExpression right; - - public CnosDBBetweenOperation(CnosDBExpression expr, CnosDBExpression left, CnosDBExpression right) { - this.expr = expr; - this.left = left; - this.right = right; - } - - public CnosDBExpression getExpr() { - return expr; - } - - public CnosDBExpression getLeft() { - return left; - } - - public CnosDBExpression getRight() { - return right; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java deleted file mode 100644 index acf3e93d5..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java +++ /dev/null @@ -1,69 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; -import sqlancer.common.ast.BinaryOperatorNode; - -public class CnosDBBinaryArithmeticOperation extends BinaryOperatorNode - implements CnosDBExpression { - - public CnosDBBinaryArithmeticOperation(CnosDBExpression left, CnosDBExpression right, CnosDBBinaryOperator op) { - super(left, right, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.INT; - } - - public enum CnosDBBinaryOperator implements BinaryOperatorNode.Operator { - - ADDITION("+") { - }, - SUBTRACTION("-") { - }, - MULTIPLICATION("*") { - }, - DIVISION("/") { - - }, - MODULO("%") { - }, - EXPONENTIATION("^") { - }; - - private final String textRepresentation; - - CnosDBBinaryOperator(String textRepresentation) { - this.textRepresentation = textRepresentation; - } - - public static CnosDBBinaryOperator getRandom(CnosDBDataType dataType) { - List ops = new ArrayList<>(Arrays.asList(values())); - switch (dataType) { - case DOUBLE: - case UINT: - case STRING: - ops.remove(EXPONENTIATION); - ops.remove(MODULO); - break; - default: - break; - } - - return Randomly.fromList(ops); - } - - @Override - public String getTextRepresentation() { - return textRepresentation; - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java deleted file mode 100644 index af38849c9..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java +++ /dev/null @@ -1,57 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator; -import sqlancer.common.ast.BinaryOperatorNode; - -public class CnosDBBinaryComparisonOperation - extends BinaryOperatorNode implements CnosDBExpression { - - public CnosDBBinaryComparisonOperation(CnosDBExpression left, CnosDBExpression right, - CnosDBBinaryComparisonOperator op) { - super(left, right, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - public enum CnosDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { - EQUALS("=") { - }, - IS_DISTINCT("IS DISTINCT FROM") { - }, - IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { - }, - NOT_EQUALS("!=") { - }, - LESS("<") { - }, - LESS_EQUALS("<=") { - }, - GREATER(">") { - }, - GREATER_EQUALS(">=") { - - }; - - private final String textRepresentation; - - CnosDBBinaryComparisonOperator(String textRepresentation) { - this.textRepresentation = textRepresentation; - } - - public static CnosDBBinaryComparisonOperator getRandom() { - return Randomly.fromOptions(CnosDBBinaryComparisonOperator.values()); - } - - @Override - public String getTextRepresentation() { - return textRepresentation; - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java deleted file mode 100644 index bad8a3b75..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java +++ /dev/null @@ -1,33 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; -import sqlancer.common.ast.BinaryOperatorNode; - -public class CnosDBBinaryLogicalOperation extends BinaryOperatorNode - implements CnosDBExpression { - - public CnosDBBinaryLogicalOperation(CnosDBExpression left, CnosDBExpression right, BinaryLogicalOperator op) { - super(left, right, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - public enum BinaryLogicalOperator implements BinaryOperatorNode.Operator { - AND, OR; - - public static BinaryLogicalOperator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return toString(); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java b/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java deleted file mode 100644 index 41db62d81..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java +++ /dev/null @@ -1,60 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.cnosdb.CnosDBCompoundDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBCastOperation implements CnosDBExpression { - - private final CnosDBExpression expression; - private final CnosDBCompoundDataType type; - - public CnosDBCastOperation(CnosDBExpression expression, CnosDBCompoundDataType type) { - if (expression == null) { - throw new AssertionError(); - } - this.expression = expression; - this.type = type; - } - - public static List canCastTo(CnosDBDataType dataType) { - List options = new ArrayList<>(Arrays.asList(CnosDBDataType.values())); - - switch (dataType) { - case UINT: - case BOOLEAN: - case DOUBLE: - options.remove(CnosDBDataType.TIMESTAMP); - break; - case TIMESTAMP: - options.remove(CnosDBDataType.BOOLEAN); - options.remove(CnosDBDataType.UINT); - options.remove(CnosDBDataType.DOUBLE); - break; - default: - break; - } - return options; - } - - @Override - public CnosDBDataType getExpressionType() { - return type.getDataType(); - } - - public CnosDBExpression getExpression() { - return expression; - } - - public CnosDBDataType getType() { - return type.getDataType(); - } - - public CnosDBCompoundDataType getCompoundType() { - return type; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java b/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java deleted file mode 100644 index f90b6120f..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java +++ /dev/null @@ -1,27 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBColumnValue implements CnosDBExpression { - - private final CnosDBColumn c; - - public CnosDBColumnValue(CnosDBColumn c) { - this.c = c; - } - - public static CnosDBColumnValue create(CnosDBColumn c) { - return new CnosDBColumnValue(c); - } - - @Override - public CnosDBDataType getExpressionType() { - return c.getType(); - } - - public CnosDBColumn getColumn() { - return c; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java b/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java deleted file mode 100644 index 6821f83b8..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryNode; - -public class CnosDBConcatOperation extends BinaryNode implements CnosDBExpression { - - public CnosDBConcatOperation(CnosDBExpression left, CnosDBExpression right) { - super(left, right); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.STRING; - } - - @Override - public String getOperatorRepresentation() { - return "||"; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConstant.java b/src/sqlancer/cnosdb/ast/CnosDBConstant.java deleted file mode 100644 index 42ecd3908..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBConstant.java +++ /dev/null @@ -1,520 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.math.BigDecimal; -import java.text.SimpleDateFormat; -import java.util.Date; - -import sqlancer.IgnoreMeException; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public abstract class CnosDBConstant implements CnosDBExpression { - - public static CnosDBConstant createNullConstant() { - return new CnosDBNullConstant(); - } - - public static CnosDBConstant createIntConstant(long val) { - return new IntConstant(val, false); - } - - public static CnosDBConstant createBooleanConstant(boolean val) { - return new BooleanConstant(val); - } - - public static CnosDBConstant createFalse() { - return createBooleanConstant(false); - } - - public static CnosDBConstant createTrue() { - return createBooleanConstant(true); - } - - public static CnosDBConstant createStringConstant(String string) { - return new StringConstant(string); - } - - public static CnosDBConstant createDoubleConstant(double val) { - return new DoubleConstant(val); - } - - public static CnosDBConstant createUintConstant(long val) { - return new IntConstant(val, true); - } - - public static CnosDBConstant createTimeStampConstant(long val) { - return new TimeStampConstant(val); - } - - public abstract String getTextRepresentation(); - - public String asString() { - throw new UnsupportedOperationException(this.toString()); - } - - public boolean isString() { - return false; - } - - public boolean isNull() { - return false; - } - - public boolean asBoolean() { - throw new UnsupportedOperationException(this.toString()); - } - - public long asInt() { - throw new UnsupportedOperationException(this.toString()); - } - - public double asDouble() { - throw new UnsupportedOperationException(this.toString()); - } - - public boolean isBoolean() { - return false; - } - - public abstract CnosDBConstant isEquals(CnosDBConstant rightVal); - - public boolean isInt() { - return false; - } - - protected abstract CnosDBConstant isLessThan(CnosDBConstant rightVal); - - @Override - public String toString() { - return getTextRepresentation(); - } - - public abstract CnosDBConstant cast(CnosDBDataType type); - - public static class BooleanConstant extends CnosDBConstant { - - private final boolean value; - - public BooleanConstant(boolean value) { - this.value = value; - } - - @Override - public String getTextRepresentation() { - return value ? "TRUE" : "FALSE"; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - @Override - public boolean asBoolean() { - return value; - } - - @Override - public boolean isBoolean() { - return true; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return CnosDBConstant.createBooleanConstant(value == rightVal.asBoolean()); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(value == rightVal.cast(CnosDBDataType.BOOLEAN).asBoolean()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isString()) { - return isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); - } else { - assert rightVal.isBoolean(); - return CnosDBConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - switch (type) { - case BOOLEAN: - return this; - case INT: - return CnosDBConstant.createIntConstant(value ? 1 : 0); - case UINT: - return CnosDBConstant.createUintConstant(value ? 1 : 0); - case STRING: - return CnosDBConstant.createStringConstant(value ? "true" : "false"); - default: - return null; - } - } - - } - - public static class CnosDBNullConstant extends CnosDBConstant { - - @Override - public String getTextRepresentation() { - return "NULL"; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - - @Override - public boolean isNull() { - return true; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - return CnosDBConstant.createNullConstant(); - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - return CnosDBConstant.createNullConstant(); - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - return CnosDBConstant.createNullConstant(); - } - } - - public static class StringConstant extends CnosDBConstant { - - private final String value; - - public StringConstant(String value) { - this.value = value; - } - - @Override - public String getTextRepresentation() { - return String.format("'%s'", value.replace("'", "''")); - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isInt()) { - return cast(CnosDBDataType.INT).isEquals(rightVal.cast(CnosDBDataType.INT)); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal.cast(CnosDBDataType.BOOLEAN)); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isInt()) { - return cast(CnosDBDataType.INT).isLessThan(rightVal.cast(CnosDBDataType.INT)); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - if (type == CnosDBDataType.STRING) { - return this; - } - String s = value.trim(); - switch (type) { - case BOOLEAN: - try { - return CnosDBConstant.createBooleanConstant(Long.parseLong(s) != 0); - } catch (NumberFormatException ignored) { - } - switch (s.toUpperCase()) { - case "T": - case "TR": - case "TRU": - case "TRUE": - case "1": - case "YES": - case "YE": - case "Y": - case "ON": - return CnosDBConstant.createTrue(); - case "F": - case "FA": - case "FAL": - case "FALS": - case "FALSE": - case "N": - case "NO": - case "OF": - case "OFF": - default: - return CnosDBConstant.createFalse(); - } - case INT: - try { - return CnosDBConstant.createIntConstant(Long.parseLong(s)); - } catch (NumberFormatException e) { - return CnosDBConstant.createIntConstant(-1); - } - case UINT: - try { - return CnosDBConstant.createUintConstant(Long.parseUnsignedLong(s)); - } catch (NumberFormatException e) { - return CnosDBConstant.createUintConstant(0); - } - case DOUBLE: - try { - return CnosDBConstant.createDoubleConstant(Double.parseDouble(s)); - } catch (NumberFormatException e) { - return CnosDBConstant.createDoubleConstant(0.0); - } - - default: - return null; - } - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.STRING; - } - - @Override - public boolean isString() { - return true; - } - - @Override - public String asString() { - return value; - } - - } - - public static class IntConstant extends CnosDBConstant { - - private final long val; - private final boolean unsigned; - - public IntConstant(long val, boolean unsigned) { - this.val = val; - this.unsigned = unsigned; - } - - @Override - public String getTextRepresentation() { - if (unsigned) { - return Long.toUnsignedString(val); - } else { - return String.valueOf(val); - } - } - - @Override - public CnosDBDataType getExpressionType() { - if (unsigned) { - return CnosDBDataType.UINT; - } - return CnosDBDataType.INT; - } - - @Override - public long asInt() { - return val; - } - - @Override - public double asDouble() { - return val; - } - - @Override - public boolean isInt() { - return true; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); - } else if (rightVal.isInt()) { - return CnosDBConstant.createBooleanConstant(val == rightVal.asInt()); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.INT).asInt()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isInt()) { - return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); - } else if (rightVal.isBoolean()) { - throw new AssertionError(rightVal); - } else if (rightVal.getExpressionType() == CnosDBDataType.UINT) { - return CnosDBConstant.createBooleanConstant(Long.compareUnsigned(val, rightVal.asInt()) < 0); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.INT).asInt()); - } else { - throw new IgnoreMeException(); - } - - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - switch (type) { - case BOOLEAN: - return CnosDBConstant.createBooleanConstant(val != 0); - case INT: - return CnosDBConstant.createIntConstant(val); - case STRING: - return CnosDBConstant.createStringConstant(String.valueOf(val)); - case UINT: - return CnosDBConstant.createUintConstant(val); - case DOUBLE: - return CnosDBConstant.createDoubleConstant(val); - default: - return null; - } - } - } - - public static class TimeStampConstant extends CnosDBConstant { - final long val; - - TimeStampConstant(long time) { - val = time; - } - - @Override - public String getTextRepresentation() { - return "CAST (" + val + " AS TIMESTAMP)"; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return createNullConstant(); - } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { - return createBooleanConstant(val == rightVal.asInt()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { - return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - switch (type) { - case INT: - return createIntConstant(val); - case STRING: - final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); - return CnosDBConstant.createStringConstant(dateFormat.format(new Date(val))); - default: - return null; - } - } - - @Override - public long asInt() { - return val; - } - - } - - public static class DoubleConstant extends CnosDBConstant { - - private final double val; - - public DoubleConstant(double val) { - this.val = val; - } - - @Override - public String getTextRepresentation() { - if (Double.isFinite(val)) { - BigDecimal bigDecimal = new BigDecimal(val); - return bigDecimal.toPlainString(); - } else { - return String.valueOf(0.0); - } - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.DOUBLE; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal); - } else { - return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); - } - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); - } else { - return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - return null; - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBExpression.java b/src/sqlancer/cnosdb/ast/CnosDBExpression.java deleted file mode 100644 index 63997a0f5..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBExpression.java +++ /dev/null @@ -1,14 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public interface CnosDBExpression { - - default CnosDBDataType getExpressionType() { - return null; - } - - default CnosDBConstant getExpectedValue() { - throw new AssertionError("Not impl"); - } -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunction.java b/src/sqlancer/cnosdb/ast/CnosDBFunction.java deleted file mode 100644 index 7a35d703e..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBFunction.java +++ /dev/null @@ -1,30 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBFunction implements CnosDBExpression { - - private final String func; - private final CnosDBExpression[] args; - private final CnosDBDataType returnType; - - public CnosDBFunction(CnosDBFunctionWithUnknownResult f, CnosDBDataType returnType, CnosDBExpression... args) { - this.func = f.getName(); - this.returnType = returnType; - this.args = args.clone(); - } - - public String getFunctionName() { - return func; - } - - public CnosDBExpression[] getArguments() { - return args.clone(); - } - - @Override - public CnosDBDataType getExpressionType() { - return returnType; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java b/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java deleted file mode 100644 index 485f2309d..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java +++ /dev/null @@ -1,104 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import sqlancer.cnosdb.CnosDBBugs; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; - -public enum CnosDBFunctionWithUnknownResult { - - // String functions - ASCII("ascii", CnosDBDataType.INT, CnosDBDataType.STRING), - BTRIM("btrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - CHAR_LENGTH("char_length", CnosDBDataType.INT, CnosDBDataType.STRING), - CHARACTER_LENGTH("character_length", CnosDBDataType.INT, CnosDBDataType.STRING), - CONCAT("concat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - CONCAT_WS("concat_ws", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - CHR("chr", CnosDBDataType.STRING, CnosDBDataType.INT), - BIT_LENGTH("bit_length", CnosDBDataType.INT, CnosDBDataType.STRING), - INITCAP("initcap", CnosDBDataType.STRING, CnosDBDataType.STRING), - - LEFT("left", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - LENGTH("length", CnosDBDataType.UINT, CnosDBDataType.STRING), - LOWER("lower", CnosDBDataType.STRING, CnosDBDataType.STRING), - UPPER("upper", CnosDBDataType.STRING, CnosDBDataType.STRING), - LPAD3("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), - LPAD2("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - RPAD3("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), - RPAD2("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - LTRIM("ltrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - OCTET_LENGTH("octet_length", CnosDBDataType.INT, CnosDBDataType.STRING), - // REPEAT("repeat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - REPLACE("replace", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - REVERSE("reverse", CnosDBDataType.STRING, CnosDBDataType.STRING), - RIGHT("right", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - RTRIM("rtrim", CnosDBDataType.STRING, CnosDBDataType.STRING), - SPLIT_PART("split_part", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - STARTS_WITH("starts_with", CnosDBDataType.BOOLEAN, CnosDBDataType.STRING, CnosDBDataType.STRING), - STRPOS("strpos", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.STRING), - SUBSTR("substr", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.INT), - TRANSLATE("translate", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - MD5("md5", CnosDBDataType.STRING, CnosDBDataType.STRING), - // mathematical functions - ABS("abs", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - CEIL("ceil", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - EXP("exp", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), LN("ln", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - LOG2("log2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - LOG10("log10", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - POWER("power", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ROUND("round", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - TRUNC("trunc", CnosDBDataType.DOUBLE, CnosDBDataType.INT), - FLOOR("floor", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - SIGNUM("signum", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ACOS("acos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ASIN("asin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ATAN2("atan2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - COS("cos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), SIN("sin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - SQRT("sqrt", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - TAN("tan", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - DATE_PART("date_part", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP), - TO_TIMESTAMP("to_timestamp", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), - TO_TIMESTAMP_MILLIS("to_timestamp_millis", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), - TO_TIMESTAMP_MICROS("to_timestamp_micros", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), - TO_TIMESTAMP_SECONDS("to_timestamp_seconds", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT); - - private final String functionName; - private final CnosDBDataType returnType; - private final CnosDBDataType[] argTypes; - - CnosDBFunctionWithUnknownResult(String functionName, CnosDBDataType returnType, CnosDBDataType... indexType) { - this.functionName = functionName; - this.returnType = returnType; - this.argTypes = indexType.clone(); - - } - - public static List getSupportedFunctions(CnosDBDataType type) { - List res = Stream.of(values()) - .filter(function -> function.isCompatibleWithReturnType(type)).collect(Collectors.toList()); - if (CnosDBBugs.BUG3547) { - res.removeAll(List.of(TO_TIMESTAMP, TO_TIMESTAMP_MICROS, TO_TIMESTAMP_MILLIS, TO_TIMESTAMP_SECONDS)); - } - return res; - } - - public boolean isCompatibleWithReturnType(CnosDBDataType t) { - return t == returnType; - } - - public CnosDBExpression[] getArguments(CnosDBDataType ignore, CnosDBExpressionGenerator gen, int depth) { - CnosDBExpression[] args = new CnosDBExpression[argTypes.length]; - for (int i = 0; i < args.length; i++) { - args[i] = gen.generateExpression(depth, argTypes[i]); - } - return args; - } - - public String getName() { - return functionName; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBInOperation.java b/src/sqlancer/cnosdb/ast/CnosDBInOperation.java deleted file mode 100644 index c0ffd34ed..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBInOperation.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.List; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBInOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final List listElements; - private final boolean isTrue; - - public CnosDBInOperation(CnosDBExpression expr, List listElements, boolean isTrue) { - this.expr = expr; - this.listElements = listElements; - this.isTrue = isTrue; - } - - public CnosDBExpression getExpr() { - return expr; - } - - public List getListElements() { - return listElements; - } - - public boolean isTrue() { - return isTrue; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBJoin.java b/src/sqlancer/cnosdb/ast/CnosDBJoin.java deleted file mode 100644 index eea88466f..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBJoin.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBJoin implements CnosDBExpression { - - private final CnosDBExpression tableReference; - private final CnosDBExpression onClause; - private final CnosDBJoinType type; - - public CnosDBJoin(CnosDBExpression tableReference, CnosDBExpression onClause, CnosDBJoinType type) { - this.tableReference = tableReference; - this.onClause = onClause; - this.type = type; - } - - public CnosDBExpression getTableReference() { - return tableReference; - } - - public CnosDBExpression getOnClause() { - return onClause; - } - - public CnosDBJoinType getType() { - return type; - } - - @Override - public CnosDBDataType getExpressionType() { - throw new AssertionError(); - } - - public enum CnosDBJoinType { - INNER, LEFT, RIGHT, FULL; - // now not support - // CROSS; - - public static CnosDBJoinType getRandom() { - return Randomly.fromOptions(values()); - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java b/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java deleted file mode 100644 index 616cd39ee..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryNode; - -public class CnosDBLikeOperation extends BinaryNode implements CnosDBExpression { - - public CnosDBLikeOperation(CnosDBExpression left, CnosDBExpression right) { - super(left, right); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - @Override - public String getOperatorRepresentation() { - return "LIKE"; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java b/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java deleted file mode 100644 index de5812d76..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java +++ /dev/null @@ -1,37 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBOrderByTerm implements CnosDBExpression { - - private final CnosDBOrder order; - private final CnosDBExpression expr; - - public CnosDBOrderByTerm(CnosDBExpression expr, CnosDBOrder order) { - this.expr = expr; - this.order = order; - } - - public CnosDBOrder getOrder() { - return order; - } - - public CnosDBExpression getExpr() { - return expr; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - - public enum CnosDBOrder { - ASC, DESC; - - public static CnosDBOrder getRandomOrder() { - return Randomly.fromOptions(CnosDBOrder.values()); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java deleted file mode 100644 index f37621f44..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java +++ /dev/null @@ -1,97 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryOperatorNode.Operator; - -public class CnosDBPostfixOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final String operatorTextRepresentation; - - public CnosDBPostfixOperation(CnosDBExpression expr, PostfixOperator op) { - this.expr = expr; - this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); - } - - public static CnosDBExpression create(CnosDBExpression expr, PostfixOperator op) { - return new CnosDBPostfixOperation(expr, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - public String getOperatorTextRepresentation() { - return operatorTextRepresentation; - } - - public CnosDBExpression getExpression() { - return expr; - } - - public enum PostfixOperator implements Operator { - IS_NULL("IS NULL"/* , "ISNULL" */) { - @Override - public CnosDBDataType[] getInputDataTypes() { - return CnosDBDataType.values(); - } - - }, - IS_UNKNOWN("IS UNKNOWN") { - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - }, - - IS_NOT_NULL("IS NOT NULL"/* "NOTNULL" */) { - - @Override - public CnosDBDataType[] getInputDataTypes() { - return CnosDBDataType.values(); - } - - }, - IS_NOT_UNKNOWN("IS NOT UNKNOWN") { - - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - }, - IS_TRUE("IS TRUE") { - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - - }, - IS_FALSE("IS FALSE") { - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - - }; - - private final String[] textRepresentations; - - PostfixOperator(String... textRepresentations) { - this.textRepresentations = textRepresentations.clone(); - } - - public static PostfixOperator getRandom() { - return Randomly.fromOptions(values()); - } - - public abstract CnosDBDataType[] getInputDataTypes(); - - @Override - public String getTextRepresentation() { - return toString(); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java deleted file mode 100644 index 241fab89a..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java +++ /dev/null @@ -1,29 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBPostfixText implements CnosDBExpression { - - private final CnosDBExpression expr; - private final String text; - private final CnosDBDataType type; - - public CnosDBPostfixText(CnosDBExpression expr, String text, CnosDBDataType type) { - this.expr = expr; - this.text = text; - this.type = type; - } - - public CnosDBExpression getExpr() { - return expr; - } - - public String getText() { - return text; - } - - @Override - public CnosDBDataType getExpressionType() { - return type; - } -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java deleted file mode 100644 index db37f0089..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java +++ /dev/null @@ -1,73 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryOperatorNode.Operator; - -public class CnosDBPrefixOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final PrefixOperator op; - - public CnosDBPrefixOperation(CnosDBExpression expr, PrefixOperator op) { - this.expr = expr; - this.op = op; - } - - @Override - public CnosDBDataType getExpressionType() { - return op.getExpressionType(); - } - - public CnosDBDataType[] getInputDataTypes() { - return op.dataTypes; - } - - public String getTextRepresentation() { - return op.textRepresentation; - } - - public CnosDBExpression getExpression() { - return expr; - } - - public enum PrefixOperator implements Operator { - NOT("NOT", CnosDBDataType.BOOLEAN) { - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - }, - UNARY_PLUS("+", CnosDBDataType.INT) { - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.INT; - } - - }, - UNARY_MINUS("-", CnosDBDataType.INT) { - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.INT; - } - - }; - - private final String textRepresentation; - private final CnosDBDataType[] dataTypes; - - PrefixOperator(String textRepresentation, CnosDBDataType... dataTypes) { - this.textRepresentation = textRepresentation; - this.dataTypes = dataTypes.clone(); - } - - public abstract CnosDBDataType getExpressionType(); - - @Override - public String getTextRepresentation() { - return toString(); - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSelect.java b/src/sqlancer/cnosdb/ast/CnosDBSelect.java deleted file mode 100644 index 0db657f19..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBSelect.java +++ /dev/null @@ -1,102 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.Collections; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.common.ast.SelectBase; - -public class CnosDBSelect extends SelectBase implements CnosDBExpression { - - private SelectType selectOption = SelectType.ALL; - private List joinClauses = Collections.emptyList(); - private CnosDBExpression distinctOnClause; - - public void setSelectType(SelectType fromOptions) { - this.setSelectOption(fromOptions); - } - - public SelectType getSelectOption() { - return selectOption; - } - - public void setSelectOption(SelectType fromOptions) { - this.selectOption = fromOptions; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - - public List getJoinClauses() { - return joinClauses; - } - - public void setJoinClauses(List joinStatements) { - this.joinClauses = joinStatements; - - } - - public CnosDBExpression getDistinctOnClause() { - return distinctOnClause; - } - - public void setDistinctOnClause(CnosDBExpression distinctOnClause) { - if (selectOption != SelectType.DISTINCT) { - throw new IllegalArgumentException(); - } - this.distinctOnClause = distinctOnClause; - } - - public enum SelectType { - DISTINCT, ALL; - - public static SelectType getRandom() { - return Randomly.fromOptions(values()); - } - } - - public static class CnosDBFromTable implements CnosDBExpression { - private final CnosDBTable t; - - public CnosDBFromTable(CnosDBTable t) { - this.t = t; - } - - public CnosDBTable getTable() { - return t; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - } - - public static class CnosDBSubquery implements CnosDBExpression { - private final CnosDBSelect s; - private final String name; - - public CnosDBSubquery(CnosDBSelect s, String name) { - this.s = s; - this.name = name; - } - - public CnosDBSelect getSelect() { - return s; - } - - public String getName() { - return name; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java b/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java deleted file mode 100644 index 9e3467ada..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java +++ /dev/null @@ -1,28 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBSimilarTo implements CnosDBExpression { - - private final CnosDBExpression string; - private final CnosDBExpression similarTo; - - public CnosDBSimilarTo(CnosDBExpression string, CnosDBExpression similarTo) { - this.string = string; - this.similarTo = similarTo; - } - - public CnosDBExpression getString() { - return string; - } - - public CnosDBExpression getSimilarTo() { - return similarTo; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - -} diff --git a/src/sqlancer/cnosdb/client/CnosDBClient.java b/src/sqlancer/cnosdb/client/CnosDBClient.java deleted file mode 100644 index ccc9dcc16..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBClient.java +++ /dev/null @@ -1,110 +0,0 @@ -package sqlancer.cnosdb.client; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; - -import org.apache.commons.codec.binary.Base64; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; - -import com.arangodb.internal.util.IOUtils; - -public class CnosDBClient { - private final String userName; - private final String password; - private final String host; - private final int port; - - private final String database; - private final CloseableHttpClient client; - - public CnosDBClient(String host, int port, String userName, String password, String database) { - this.host = host; - this.port = port; - this.userName = userName; - this.password = password; - this.database = database; - this.client = HttpClientBuilder.create().build(); - } - - private String url() { - return "http://" + host + ":" + port + "/api/v1/"; - } - - public String ping() throws Exception { - HttpGet httpGet = new HttpGet(this.url() + "ping"); - httpGet.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); - CloseableHttpResponse resp = client.execute(httpGet); - - String content = IOUtils.toString(resp.getEntity().getContent()); - resp.close(); - return content; - } - - public CnosDBResultSet executeQuery(String query) throws Exception { - HttpUriRequest request = createRequest(query); - CloseableHttpResponse resp = client.execute(request); - String text = IOUtils.toString(resp.getEntity().getContent()); - if (resp.getStatusLine().getStatusCode() != 200) { - resp.close(); - throw new CnosDBException(database + ":" + query + ";\n" + text); - } - resp.close(); - InputStream stream = new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); - - return new CnosDBResultSet(new InputStreamReader(stream)); - } - - public boolean execute(String query) throws Exception { - HttpUriRequest request = createRequest(query); - CloseableHttpResponse resp = client.execute(request); - if (resp.getStatusLine().getStatusCode() != 200) { - String res = IOUtils.toString(resp.getEntity().getContent()); - resp.close(); - throw new CnosDBException(query + res); - } - resp.close(); - return true; - } - - public void close() throws IOException { - client.close(); - } - - public String getDatabase() { - return this.database; - } - - private String getAuth() { - String auth = userName + ":" + password; - byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.ISO_8859_1)); - return "Basic " + new String(encodedAuth); - - } - - private HttpUriRequest createRequest(String query) throws URISyntaxException, UnsupportedEncodingException { - - URIBuilder builder = new URIBuilder(this.url() + "sql"); - builder.setParameter("db", database); - builder.setParameter("pretty", "true"); - HttpPost httpPost = new HttpPost(builder.build()); - - httpPost.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); - StringEntity stringEntity = new StringEntity(query); - httpPost.setEntity(stringEntity); - return httpPost; - } - -} diff --git a/src/sqlancer/cnosdb/client/CnosDBConnection.java b/src/sqlancer/cnosdb/client/CnosDBConnection.java deleted file mode 100644 index 9277f203b..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBConnection.java +++ /dev/null @@ -1,27 +0,0 @@ -package sqlancer.cnosdb.client; - -import java.io.IOException; - -import sqlancer.SQLancerDBConnection; - -public class CnosDBConnection implements SQLancerDBConnection { - private final CnosDBClient client; - - public CnosDBConnection(CnosDBClient client) { - this.client = client; - } - - @Override - public String getDatabaseVersion() throws Exception { - return client.ping(); - } - - public CnosDBClient getClient() { - return client; - } - - @Override - public void close() throws IOException { - client.close(); - } -} diff --git a/src/sqlancer/cnosdb/client/CnosDBException.java b/src/sqlancer/cnosdb/client/CnosDBException.java deleted file mode 100644 index a1055e90b..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBException.java +++ /dev/null @@ -1,9 +0,0 @@ -package sqlancer.cnosdb.client; - -public class CnosDBException extends RuntimeException { - private static final long serialVersionUID = 1L; - - CnosDBException(String message) { - super(message); - } -} diff --git a/src/sqlancer/cnosdb/client/CnosDBResultSet.java b/src/sqlancer/cnosdb/client/CnosDBResultSet.java deleted file mode 100644 index 877b6ba5d..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBResultSet.java +++ /dev/null @@ -1,52 +0,0 @@ -package sqlancer.cnosdb.client; - -import java.io.Reader; -import java.sql.SQLException; -import java.util.Iterator; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; - -import sqlancer.IgnoreMeException; - -public class CnosDBResultSet { - private final Iterator records; - private CSVRecord next; - - public CnosDBResultSet(Reader in) throws Exception { - Iterable records = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).build() - .parse(in); - this.records = records.iterator(); - } - - public void close() { - } - - public boolean next() throws SQLException { - if (records.hasNext()) { - next = records.next(); - return true; - } - return false; - } - - public int getInt(int i) throws SQLException { - return Integer.parseInt(next.get(i - 1)); - } - - public String getString(int i) throws SQLException { - return next.get(i - 1); - } - - public long getLong(int i) throws SQLException { - if (next.get(i - 1).isEmpty()) { - throw new IgnoreMeException(); - } - return Long.parseLong(next.get(i - 1)); - } - - // public boolean getBool(int i) throws Exception { - // return Boolean.parseBoolean(getString(i)); - // } - -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBCommon.java b/src/sqlancer/cnosdb/gen/CnosDBCommon.java deleted file mode 100644 index 6c7b0bba7..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBCommon.java +++ /dev/null @@ -1,31 +0,0 @@ -package sqlancer.cnosdb.gen; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public final class CnosDBCommon { - - private CnosDBCommon() { - } - - public static void appendDataType(CnosDBDataType type, StringBuilder sb) throws AssertionError { - switch (type) { - case BOOLEAN: - sb.append("BOOLEAN"); - break; - case INT: - sb.append("BIGINT"); - break; - case STRING: - sb.append("STRING"); - break; - case DOUBLE: - sb.append("DOUBLE"); - break; - case UINT: - sb.append("BIGINT UNSIGNED"); - break; - default: - throw new AssertionError(type); - } - } -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java deleted file mode 100644 index 121f78254..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java +++ /dev/null @@ -1,461 +0,0 @@ -package sqlancer.cnosdb.gen; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBCompoundDataType; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; -import sqlancer.cnosdb.ast.CnosDBBetweenOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; -import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConcatOperation; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBFunction; -import sqlancer.cnosdb.ast.CnosDBFunctionWithUnknownResult; -import sqlancer.cnosdb.ast.CnosDBInOperation; -import sqlancer.cnosdb.ast.CnosDBLikeOperation; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm.CnosDBOrder; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; -import sqlancer.cnosdb.ast.CnosDBSimilarTo; -import sqlancer.common.gen.ExpressionGenerator; - -public class CnosDBExpressionGenerator implements ExpressionGenerator { - - private final int maxDepth; - - private final Randomly r; - - private List columns; - - private boolean allowAggregateFunctions; - - public CnosDBExpressionGenerator(CnosDBGlobalState globalState) { - this.r = globalState.getRandomly(); - this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); - } - - public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, CnosDBDataType type) { - return new CnosDBExpressionGenerator(globalState).generateExpression(0, type); - } - - private static CnosDBCompoundDataType getCompoundDataType(CnosDBDataType type) { - return CnosDBCompoundDataType.create(type); - } - - public static CnosDBExpression generateConstant(Randomly r, CnosDBDataType type) { - if (Randomly.getBooleanWithSmallProbability()) { - return CnosDBConstant.createNullConstant(); - } - switch (type) { - case INT: - return CnosDBConstant.createIntConstant(r.getInteger()); - case UINT: - return CnosDBConstant.createUintConstant(r.getPositiveInteger()); - case TIMESTAMP: - return CnosDBConstant.createTimeStampConstant(r.getPositiveIntegerNotNull()); - case BOOLEAN: - return CnosDBConstant.createBooleanConstant(Randomly.getBoolean()); - case STRING: - return CnosDBConstant.createStringConstant(r.getString()); - case DOUBLE: - return CnosDBConstant.createDoubleConstant(r.getDouble()); - default: - throw new AssertionError(type); - } - } - - public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns, - CnosDBDataType type) { - return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); - } - - public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns) { - return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0); - } - - public CnosDBExpressionGenerator setColumns(List columns) { - this.columns = columns; - return this; - } - - public CnosDBExpression generateExpression(int depth) { - return generateExpression(depth, CnosDBDataType.getRandomType()); - } - - public List generateOrderBy() { - List orderBys = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber(); i++) { - orderBys.add(new CnosDBOrderByTerm(CnosDBColumnValue.create(Randomly.fromList(columns)), - CnosDBOrder.getRandomOrder())); - } - return orderBys; - } - - private CnosDBExpression generateFunctionWithUnknownResult(int depth, CnosDBDataType type) { - List supportedFunctions = CnosDBFunctionWithUnknownResult - .getSupportedFunctions(type); - if (supportedFunctions.isEmpty()) { - throw new IgnoreMeException(); - } - CnosDBFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); - return new CnosDBFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); - } - - private CnosDBExpression generateBooleanExpression(int depth) { - List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); - BooleanExpression option = Randomly.fromList(validOptions); - switch (option) { - case POSTFIX_OPERATOR: - PostfixOperator random = PostfixOperator.getRandom(); - return CnosDBPostfixOperation - .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); - case IN_OPERATION: - return inOperation(depth + 1); - case NOT: - return new CnosDBPrefixOperation(generateExpression(depth + 1, CnosDBDataType.BOOLEAN), PrefixOperator.NOT); - case BINARY_LOGICAL_OPERATOR: - CnosDBExpression first = generateExpression(depth + 1, CnosDBDataType.BOOLEAN); - int nr = Randomly.smallNumber() + 1; - for (int i = 0; i < nr; i++) { - first = new CnosDBBinaryLogicalOperation(first, generateExpression(depth + 1, CnosDBDataType.BOOLEAN), - BinaryLogicalOperator.getRandom()); - } - return first; - case BINARY_COMPARISON: - CnosDBDataType dataType = getMeaningfulType(); - return generateComparison(depth, dataType); - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.BOOLEAN); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.BOOLEAN); - case LIKE: - return new CnosDBLikeOperation(generateExpression(depth + 1, CnosDBDataType.STRING), - generateExpression(depth + 1, CnosDBDataType.STRING)); - case BETWEEN: - CnosDBDataType type = getMeaningfulType(); - return new CnosDBBetweenOperation(generateExpression(depth + 1, type), generateExpression(depth + 1, type), - generateExpression(depth + 1, type)); - case SIMILAR_TO: - return new CnosDBSimilarTo(generateExpression(depth + 1, CnosDBDataType.STRING), - generateExpression(depth + 1, CnosDBDataType.STRING)); - default: - throw new AssertionError(); - } - } - - private CnosDBDataType getMeaningfulType() { - // make it more likely that the expression does not only consist of constant - // expressions - if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { - return CnosDBDataType.getRandomType(); - } else { - return Randomly.fromList(columns).getType(); - } - } - - private CnosDBExpression generateFunction(int depth, CnosDBDataType type) { - return generateFunctionWithUnknownResult(depth, type); - } - - private CnosDBExpression generateComparison(int depth, CnosDBDataType dataType) { - CnosDBExpression leftExpr = generateExpression(depth + 1, dataType); - CnosDBExpression rightExpr = generateExpression(depth + 1, dataType); - return getComparison(leftExpr, rightExpr); - } - - private CnosDBExpression getComparison(CnosDBExpression leftExpr, CnosDBExpression rightExpr) { - return new CnosDBBinaryComparisonOperation(leftExpr, rightExpr, - CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator.getRandom()); - } - - private CnosDBExpression inOperation(int depth) { - CnosDBDataType type = CnosDBDataType.getRandomType(); - CnosDBExpression leftExpr = generateExpression(depth + 1, type); - List rightExpr = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - rightExpr.add(generateConstant(new Randomly(), type)); - } - return new CnosDBInOperation(leftExpr, rightExpr, Randomly.getBoolean()); - } - - public CnosDBExpression generateExpression(int depth, CnosDBDataType originalType) { - return generateExpressionInternal(depth, originalType); - } - - private CnosDBExpression generateExpressionInternal(int depth, CnosDBDataType dataType) throws AssertionError { - if (allowAggregateFunctions && Randomly.getBoolean()) { - return getAggregate(dataType); - } - - if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { - // generic expression - if (Randomly.getBoolean() || depth > maxDepth) { - if (Randomly.getBooleanWithRatherLowProbability()) { - return generateConstant(r, dataType); - } else { - if (filterColumns(dataType).isEmpty()) { - return generateConstant(r, dataType); - } else { - return createColumnOfType(dataType); - } - } - } else { - if (Randomly.getBoolean()) { - return generateCastExpression(depth + 1, dataType); - } else { - return generateFunctionWithUnknownResult(depth, dataType); - } - } - } else { - switch (dataType) { - case BOOLEAN: - return generateBooleanExpression(depth); - case INT: - return generateIntExpression(depth); - case UINT: - return generateUIntExpression(depth); - case STRING: - return generateStringExpression(depth); - case DOUBLE: - return generateFloatExpression(depth); - case TIMESTAMP: - return generateTimeStampExpression(depth); - default: - throw new AssertionError(dataType); - } - } - } - - private CnosDBExpression generateStringExpression(int depth) { - StringExpression option; - List validOptions = new ArrayList<>(Arrays.asList(StringExpression.values())); - option = Randomly.fromList(validOptions); - - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.STRING); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.STRING); - case CONCAT: - return generateConcat(depth); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateConcat(int depth) { - CnosDBExpression left = generateExpression(depth + 1, CnosDBDataType.STRING); - CnosDBExpression right = generateExpression(depth + 1); - return new CnosDBConcatOperation(left, right); - } - - private CnosDBExpression generateIntExpression(int depth) { - IntExpression option; - option = Randomly.fromOptions(IntExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.INT); - case UNARY_OPERATION: - CnosDBExpression intExpression = generateExpression(depth + 1, CnosDBDataType.INT); - return new CnosDBPrefixOperation(intExpression, - Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.INT); - case BINARY_ARITHMETIC_EXPRESSION: - return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.INT), - generateExpression(depth + 1, CnosDBDataType.INT), - CnosDBBinaryOperator.getRandom(CnosDBDataType.INT)); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateUIntExpression(int depth) { - UIntExpression option = Randomly.fromOptions(UIntExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.UINT); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.UINT); - case BINARY_ARITHMETIC_EXPRESSION: - return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.UINT), - generateExpression(depth + 1, CnosDBDataType.UINT), - CnosDBBinaryOperator.getRandom(CnosDBDataType.UINT)); - default: - throw new AssertionError(); - } - - } - - private CnosDBExpression generateFloatExpression(int depth) { - FloatExpression option; - option = Randomly.fromOptions(FloatExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.DOUBLE); - case UNARY_OPERATION: - CnosDBExpression floatExpression = generateExpression(depth + 1, CnosDBDataType.DOUBLE); - return new CnosDBPrefixOperation(floatExpression, - Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.DOUBLE); - case BINARY_ARITHMETIC_EXPRESSION: - return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.DOUBLE), - generateExpression(depth + 1, CnosDBDataType.DOUBLE), - CnosDBBinaryOperator.getRandom(CnosDBDataType.DOUBLE)); - case CONSTANT: - return generateConstant(r, CnosDBDataType.DOUBLE); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateTimeStampExpression(int depth) { - if (Randomly.getBoolean()) { - return generateConstant(r, CnosDBDataType.TIMESTAMP); - } - TimestampExpression option; - option = Randomly.fromOptions(TimestampExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.TIMESTAMP); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.TIMESTAMP); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateCastExpression(int depth, CnosDBDataType castToType) { - CnosDBDataType castFromType = Randomly.fromList(CnosDBCastOperation.canCastTo(castToType)); - return new CnosDBCastOperation(generateExpression(depth + 1, castFromType), getCompoundDataType(castToType)); - } - - private CnosDBExpression createColumnOfType(CnosDBDataType type) { - List columns = filterColumns(type); - if (columns.isEmpty()) { - throw new IgnoreMeException(); - } - CnosDBColumn fromList = Randomly.fromList(columns); - return CnosDBColumnValue.create(fromList); - } - - final List filterColumns(CnosDBDataType type) { - if (columns == null) { - return Collections.emptyList(); - } else { - return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); - } - } - - public List generateExpressions(int nr) { - List expressions = new ArrayList<>(); - for (int i = 0; i < nr; i++) { - expressions.add(generateExpression(0)); - } - return expressions; - } - - public CnosDBExpression generateExpression(CnosDBDataType dataType) { - return generateExpression(0, dataType); - } - - public CnosDBExpression generateHavingClause() { - this.allowAggregateFunctions = true; - CnosDBExpression expression = generateExpression(CnosDBDataType.BOOLEAN); - this.allowAggregateFunctions = false; - return expression; - } - - public CnosDBExpression generateAggregate() { - return getAggregate(CnosDBDataType.getRandomType()); - } - - private CnosDBExpression getAggregate(CnosDBDataType dataType) { - if (dataType == CnosDBDataType.BOOLEAN) { - List aggregates = CnosDBAggregateFunction.getAggregates(CnosDBDataType.INT); - CnosDBAggregateFunction agg = Randomly.fromList(aggregates); - return new CnosDBCastOperation(generateArgsForAggregate(dataType, agg), - CnosDBCompoundDataType.create(CnosDBDataType.BOOLEAN)); - } else { - List aggregates = CnosDBAggregateFunction.getAggregates(dataType); - CnosDBAggregateFunction agg = Randomly.fromList(aggregates); - return generateArgsForAggregate(dataType, agg); - } - } - - public CnosDBAggregate generateArgsForAggregate(CnosDBDataType dataType, CnosDBAggregateFunction agg) { - CnosDBDataType[] types = agg.getArgsTypes(dataType); - List args = new ArrayList<>(); - for (CnosDBDataType argType : types) { - args.add(createColumnOfType(argType)); - // args.add(generateExpression(argType)); - } - return new CnosDBAggregate(args, agg); - } - - public CnosDBExpressionGenerator allowAggregates(boolean value) { - allowAggregateFunctions = value; - return this; - } - - @Override - public CnosDBExpression generatePredicate() { - return generateExpression(CnosDBDataType.BOOLEAN); - } - - @Override - public CnosDBExpression negatePredicate(CnosDBExpression predicate) { - return new CnosDBPrefixOperation(predicate, PrefixOperator.NOT); - } - - @Override - public CnosDBExpression isNull(CnosDBExpression expr) { - return new CnosDBPostfixOperation(expr, PostfixOperator.IS_NULL); - } - - private enum BooleanExpression { - POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, CAST, LIKE, BETWEEN, IN_OPERATION, - SIMILAR_TO, - } - - private enum StringExpression { - CAST, FUNCTION, CONCAT - } - - private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION - } - - private enum UIntExpression { - FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION - } - - private enum FloatExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, CONSTANT - } - - private enum TimestampExpression { - FUNCTION, CAST - } - -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java deleted file mode 100644 index 0d575d3c7..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java +++ /dev/null @@ -1,59 +0,0 @@ -package sqlancer.cnosdb.gen; - -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.schema.AbstractTableColumn; - -public final class CnosDBInsertGenerator { - - private CnosDBInsertGenerator() { - } - - public static CnosDBOtherQuery insert(CnosDBGlobalState globalState) { - CnosDBTable table = globalState.getSchema().getRandomTable(); - ExpectedErrors errors = new ExpectedErrors(); - errors.add("Column time cannot be null."); - StringBuilder sb = new StringBuilder(); - sb.append("INSERT "); - sb.append(table.getName()); - List columns = table.getRandomNonEmptyColumnSubset(); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES"); - - int n = Randomly.smallNumber() + 1; - for (int i = 0; i < n; i++) { - if (i != 0) { - sb.append(", "); - } - insertRow(globalState, sb, columns); - } - - // error - return new CnosDBOtherQuery(sb.toString(), errors); - } - - private static void insertRow(CnosDBGlobalState globalState, StringBuilder sb, List columns) { - sb.append("("); - for (int i = 0; i < columns.size(); i++) { - if (i > 0) { - sb.append(", "); - } - CnosDBExpression generateConstant = CnosDBExpressionGenerator.generateConstant(globalState.getRandomly(), - columns.get(i).getType()); - sb.append(CnosDBVisitor.asString(generateConstant)); - } - sb.append(")"); - } - -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java deleted file mode 100644 index c046ad3e9..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java +++ /dev/null @@ -1,77 +0,0 @@ -package sqlancer.cnosdb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBFieldColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTagColumn; -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.common.query.ExpectedErrors; - -public class CnosDBTableGenerator { - - protected final ExpectedErrors errors = new ExpectedErrors(); - private final String tableName; - private final StringBuilder sb = new StringBuilder(); - private final List columnsToBeAdd = new ArrayList<>(); - private CnosDBTable table; - - public CnosDBTableGenerator(String tableName) { - this.tableName = tableName; - } - - public static CnosDBOtherQuery generate(String tableName) { - return new CnosDBTableGenerator(tableName).generate(); - } - - protected CnosDBOtherQuery generate() { - table = new CnosDBTable(tableName, columnsToBeAdd); - - sb.append("CREATE TABLE"); - if (Randomly.getBoolean()) { - sb.append(" IF NOT EXISTS"); - } - sb.append(" "); - sb.append(tableName); - - sb.append("("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - String name = String.format("f%d", i); - createField(name); - sb.append(", "); - } - - sb.append("TAGS("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - if (i != 0) { - sb.append(", "); - } - String name = String.format("t%d", i); - createTag(name); - } - sb.append("))"); - return new CnosDBOtherQuery(sb.toString(), new ExpectedErrors()); - } - - private void createField(String name) throws AssertionError { - sb.append(name); - sb.append(" "); - CnosDBDataType type = CnosDBDataType.getRandomTypeWithoutTimeStamp(); - CnosDBCommon.appendDataType(type, sb); - CnosDBFieldColumn c = new CnosDBFieldColumn(name, type); - c.setTable(table); - sb.append(" "); - columnsToBeAdd.add(c); - } - - private void createTag(String name) { - sb.append(name); - CnosDBColumn column = new CnosDBTagColumn(name); - column.setTable(table); - columnsToBeAdd.add(column); - } -} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java deleted file mode 100644 index 472aa8f66..000000000 --- a/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java +++ /dev/null @@ -1,23 +0,0 @@ -package sqlancer.cnosdb.oracle; - -import sqlancer.Main; -import sqlancer.MainOptions; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.oracle.TestOracle; - -public abstract class CnosDBNoRECBase implements TestOracle { - protected final CnosDBGlobalState state; - protected final Main.StateLogger logger; - protected final MainOptions options; - protected final CnosDBConnection con; - protected String optimizedQueryString; - protected String unoptimizedQueryString; - - public CnosDBNoRECBase(CnosDBGlobalState state) { - this.state = state; - this.con = state.getConnection(); - this.logger = state.getLogger(); - this.options = state.getOptions(); - } -} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java deleted file mode 100644 index 0c817c655..000000000 --- a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java +++ /dev/null @@ -1,171 +0,0 @@ -package sqlancer.cnosdb.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBCompoundDataType; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBJoin.CnosDBJoinType; -import sqlancer.cnosdb.ast.CnosDBPostfixText; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.ast.CnosDBSelect.SelectType; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPBase; -import sqlancer.cnosdb.query.CnosDBSelectQuery; -import sqlancer.common.oracle.TestOracle; - -public class CnosDBNoRECOracle extends CnosDBNoRECBase implements TestOracle { - - private final CnosDBSchema s; - - public CnosDBNoRECOracle(CnosDBGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - } - - public static List getJoinStatements(CnosDBGlobalState globalState, List columns, - List tables) { - List joinStatements = new ArrayList<>(); - CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); - CnosDBTable table = Randomly.fromList(tables); - tables.remove(table); - CnosDBJoinType options = CnosDBJoinType.getRandom(); - CnosDBJoin j = new CnosDBJoin(new CnosDBFromTable(table), joinClause, options); - joinStatements.add(j); - } - // JOIN subqueries - for (int i = 0; i < Randomly.smallNumber(); i++) { - CnosDBTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); - CnosDBSubquery subquery = CnosDBTLPBase.createSubquery(globalState, String.format("sub%d", i), - subqueryTables); - CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); - CnosDBJoinType options = CnosDBJoinType.getRandom(); - CnosDBJoin j = new CnosDBJoin(subquery, joinClause, options); - joinStatements.add(j); - } - return joinStatements; - } - - @Override - public void check() throws Exception { - CnosDBTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - CnosDBExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); - - List joinStatements = getJoinStatements(state, columns, tables); - List fromTables = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, List.of(CnosDBColumn.createDummy("f0")), - randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } - } - - private CnosDBExpression getRandomWhereCondition(List columns) { - return new CnosDBExpressionGenerator(state).setColumns(columns).generateExpression(CnosDBDataType.BOOLEAN); - } - - private int getUnoptimizedQueryCount(List fromTables, CnosDBExpression randomWhereCondition, - List joinStatements) throws Exception { - CnosDBSelect select = new CnosDBSelect(); - CnosDBCastOperation isTrue = new CnosDBCastOperation(randomWhereCondition, - CnosDBCompoundDataType.create(CnosDBDataType.INT)); - CnosDBPostfixText asText = new CnosDBPostfixText(isTrue, " as count", CnosDBDataType.INT); - select.setFetchColumns(List.of(asText)); - select.setFromList(fromTables); - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + CnosDBVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - CnosDBSelectQuery q = new CnosDBSelectQuery(unoptimizedQueryString, CnosDBExpectedError.expectedErrors()); - CnosDBResultSet rs; - try { - q.executeAndGet(state); - rs = q.getResultSet(); - } catch (Exception e) { - if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(List randomTables, List columns, - CnosDBExpression randomWhereCondition, List joinStatements) { - CnosDBSelect select = new CnosDBSelect(); - CnosDBColumnValue allColumns = new CnosDBColumnValue(Randomly.fromList(columns)); - select.setFetchColumns(List.of(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new CnosDBExpressionGenerator(state).setColumns(columns).generateOrderBy()); - } - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - optimizedQueryString = CnosDBVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - CnosDBSelectQuery query = new CnosDBSelectQuery(optimizedQueryString, CnosDBExpectedError.expectedErrors()); - CnosDBResultSet rs; - try { - query.executeAndGet(state); - rs = query.getResultSet(); - while (rs.next()) { - firstCount++; - } - } catch (Exception e) { - if (query.getExpectedErrors().errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - - throw new IgnoreMeException(); - } - return firstCount; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java deleted file mode 100644 index b51624a94..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java +++ /dev/null @@ -1,176 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import sqlancer.ComparatorHelper; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; -import sqlancer.cnosdb.ast.CnosDBAlias; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.cnosdb.query.CnosDBSelectQuery; -import sqlancer.common.oracle.TestOracle; - -public class CnosDBTLPAggregateOracle extends CnosDBTLPBase implements TestOracle { - - private String firstResult; - private String secondResult; - private String originalQuery; - private String metamorphicQuery; - - public CnosDBTLPAggregateOracle(CnosDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - aggregateCheck(); - } - - protected void aggregateCheck() { - CnosDBAggregateFunction aggregateFunction = Randomly.fromOptions(CnosDBAggregateFunction.MAX, - CnosDBAggregateFunction.MIN, CnosDBAggregateFunction.SUM); - - CnosDBAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), - aggregateFunction); - List fetchColumns = new ArrayList<>(); - fetchColumns.add(aggregate); - while (Randomly.getBooleanWithRatherLowProbability()) { - fetchColumns.add(gen.generateAggregate()); - } - select.setFetchColumns(fetchColumns); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - originalQuery = CnosDBVisitor.asString(select); - firstResult = getAggregateResult(originalQuery); - metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); - secondResult = getAggregateResult(metamorphicQuery); - - String queryFormatString = "-- %s;\n-- result: %s"; - String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); - String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - if (firstResult == null && secondResult != null || firstResult != null && secondResult == null - || firstResult != null && !firstResult.contentEquals(secondResult) - && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { - if (secondResult != null && secondResult.contains("Inf")) { - throw new IgnoreMeException(); // FIXME: average computation - } - String assertionMessage = String.format("%s: the results mismatch!\n%s\n%s", this.s.getDatabaseName(), - firstQueryString, secondQueryString); - throw new AssertionError(assertionMessage); - } - } - - private String createMetamorphicUnionQuery(CnosDBSelect select, CnosDBAggregate aggregate, - List from) { - String metamorphicQuery; - CnosDBExpression whereClause = gen.generateExpression(CnosDBDataType.BOOLEAN); - CnosDBExpression negatedClause = new CnosDBPrefixOperation(whereClause, PrefixOperator.NOT); - CnosDBExpression notNullClause = new CnosDBPostfixOperation(whereClause, PostfixOperator.IS_NULL); - List mappedAggregate = mapped(aggregate); - CnosDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); - CnosDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); - CnosDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; - metamorphicQuery += CnosDBVisitor.asString(leftSelect) + " UNION ALL " + CnosDBVisitor.asString(middleSelect) - + " UNION ALL " + CnosDBVisitor.asString(rightSelect); - metamorphicQuery += ") as asdf"; - return metamorphicQuery; - } - - private String getAggregateResult(String queryString) { - // log TLP Aggregate SELECT queries on the current log file - if (state.getOptions().logEachSelect()) { - // TODO: refactor me - state.getLogger().writeCurrent(queryString); - try { - state.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - String resultString = null; - - CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, CnosDBExpectedError.expectedErrors()); - try { - q.executeAndGet(state); - CnosDBResultSet result = q.getResultSet(); - - if (result == null || !result.next()) { - throw new IgnoreMeException(); - } - - resultString = result.getString(1); - - } catch (Exception e) { - if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - } - - return resultString; - } - - private List mapped(CnosDBAggregate aggregate) { - switch (aggregate.getFunction()) { - case SUM: - case MAX: - case MIN: - return aliasArgs(List.of(aggregate)); - // now not support - // case COUNT: - // case AVG: - default: - throw new AssertionError(aggregate.getFunction()); - } - } - - private List aliasArgs(List originalAggregateArgs) { - List args = new ArrayList<>(); - int i = 0; - for (CnosDBExpression expr : originalAggregateArgs) { - args.add(new CnosDBAlias(expr, "agg" + i++)); - } - return args; - } - - private String getOuterAggregateFunction(CnosDBAggregate aggregate) { - if (Objects.requireNonNull(aggregate.getFunction()) == CnosDBAggregateFunction.COUNT) { - return CnosDBAggregateFunction.SUM + "(agg0)"; - } - return aggregate.getFunction() + "(agg0)"; - } - - private CnosDBSelect getSelect(List aggregates, List from, - CnosDBExpression whereClause, List joinList) { - CnosDBSelect leftSelect = new CnosDBSelect(); - leftSelect.setFetchColumns(aggregates); - leftSelect.setFromList(from); - leftSelect.setWhereClause(whereClause); - leftSelect.setJoinClauses(joinList); - if (Randomly.getBooleanWithSmallProbability()) { - leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - } - return leftSelect; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java deleted file mode 100644 index bd7ba3b55..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java +++ /dev/null @@ -1,112 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; -import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; - -public class CnosDBTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - protected CnosDBSchema s; - protected CnosDBTables targetTables; - protected CnosDBExpressionGenerator gen; - protected CnosDBSelect select; - - public CnosDBTLPBase(CnosDBGlobalState state) { - super(state); - } - - public static CnosDBSubquery createSubquery(CnosDBGlobalState globalState, String name, CnosDBTables tables) { - List columns = new ArrayList<>(); - CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(tables.getColumns()); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - columns.add(gen.generateExpression(0)); - } - CnosDBSelect select = new CnosDBSelect(); - select.setFromList(tables.getTables().stream().map(CnosDBFromTable::new).collect(Collectors.toList())); - select.setFetchColumns(columns); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(0, CnosDBDataType.BOOLEAN)); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - if (Randomly.getBoolean()) { - select.setLimitClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - if (Randomly.getBoolean()) { - select.setOffsetClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - } - } - return new CnosDBSubquery(select, name); - } - - @Override - public void check() throws Exception { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - List tables = targetTables.getTables(); - List joins = getJoinStatements(targetTables.getColumns(), tables); - generateSelectBase(tables, joins); - } - - protected List getJoinStatements(List columns, List tables) { - return CnosDBNoRECOracle.getJoinStatements(state, columns, tables); - } - - protected void generateSelectBase(List tables, List joins) { - List tableList = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); - gen = new CnosDBExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new CnosDBSelect(); - select.setFetchColumns(generateFetchColumns()); - select.setFromList(tableList); - select.setWhereClause(null); - select.setJoinClauses(joins); - } - - List generateFetchColumns() { - if (Randomly.getBooleanWithRatherLowProbability()) { - return List.of(new CnosDBColumnValue(CnosDBColumn.createDummy("*"))); - } - List fetchColumns = new ArrayList<>(); - List targetColumns = targetTables.getRandomColumnsWithOnlyOneField(); - - ArrayList columns = new ArrayList<>(); - targetColumns.forEach(column -> column.getTable().getColumns().stream() - .filter(field -> field instanceof CnosDBSchema.CnosDBFieldColumn).findFirst().ifPresent(columns::add)); - targetColumns.addAll(columns); - - targetColumns = targetColumns.stream().distinct().collect(Collectors.toList()); - - for (CnosDBColumn c : targetColumns) { - fetchColumns.add(new CnosDBColumnValue(c)); - } - return fetchColumns; - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java deleted file mode 100644 index 283d59a23..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java +++ /dev/null @@ -1,65 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBComparatorHelper; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBExpression; - -public class CnosDBTLPHavingOracle extends CnosDBTLPBase { - - public CnosDBTLPHavingOracle(CnosDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - havingCheck(); - } - - protected void havingCheck() throws Exception { - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(CnosDBDataType.BOOLEAN)); - } - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - select.setHavingClause(null); - String originalQueryString = CnosDBVisitor.asString(select); - List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, - CnosDBExpectedError.expectedErrors(), state); - - boolean orderBy = Randomly.getBoolean(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBy()); - } - select.setHavingClause(predicate); - String firstQueryString = CnosDBVisitor.asString(select); - select.setHavingClause(negatedPredicate); - String secondQueryString = CnosDBVisitor.asString(select); - select.setHavingClause(isNullPredicate); - String thirdQueryString = CnosDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, CnosDBExpectedError.expectedErrors()); - CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - - @Override - protected CnosDBExpression generatePredicate() { - return gen.generateHavingClause(); - } - - @Override - List generateFetchColumns() { - List expressions = gen.allowAggregates(true).generateExpressions(Randomly.smallNumber() + 1); - gen.allowAggregates(false); - return expressions; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java deleted file mode 100644 index 8e118435d..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBComparatorHelper; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBVisitor; - -public class CnosDBTLPWhereOracle extends CnosDBTLPBase { - - public CnosDBTLPWhereOracle(CnosDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - whereCheck(); - } - - protected void whereCheck() throws Exception { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - String originalQueryString = CnosDBVisitor.asString(select); - List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, - CnosDBExpectedError.expectedErrors(), state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = CnosDBVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = CnosDBVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = CnosDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, CnosDBExpectedError.expectedErrors()); - CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } -} diff --git a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java deleted file mode 100644 index f0a37056c..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java +++ /dev/null @@ -1,32 +0,0 @@ -package sqlancer.cnosdb.query; - -import sqlancer.GlobalState; -import sqlancer.IgnoreMeException; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.query.ExpectedErrors; - -public class CnosDBOtherQuery extends CnosDBQueryAdapter { - private static final long serialVersionUID = 1L; - - public CnosDBOtherQuery(String query, ExpectedErrors errors) { - super(query, errors); - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - globalState.getConnection().getClient().execute(query); - } catch (Exception e) { - if (this.errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - } - return true; - } -} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java deleted file mode 100644 index 115f96ffc..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.cnosdb.query; - -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; - -public abstract class CnosDBQueryAdapter extends Query { - private static final long serialVersionUID = 1L; - - String query; - ExpectedErrors errors; - - public CnosDBQueryAdapter(String query, ExpectedErrors errors) { - this.query = query; - this.errors = errors; - } - - @Override - public String getLogString() { - return query; - } - - @Override - public String getQueryString() { - return query; - } - - @Override - public String getUnterminatedQueryString() { - return null; - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public ExpectedErrors getExpectedErrors() { - return errors; - } -} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java b/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java deleted file mode 100644 index dee38abf4..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.cnosdb.query; - -@FunctionalInterface -public interface CnosDBQueryProvider { - CnosDBOtherQuery getQuery(S globalState) throws Exception; -} diff --git a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java deleted file mode 100644 index 1c9228182..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.cnosdb.query; - -import sqlancer.GlobalState; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLancerResultSet; - -public class CnosDBSelectQuery extends CnosDBQueryAdapter { - private static final long serialVersionUID = 1L; - CnosDBResultSet resultSet; - - public CnosDBSelectQuery(String query, ExpectedErrors errors) { - super(query, errors); - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - globalState.getConnection().getClient().execute(query); - return false; - } - - @Override - public > SQLancerResultSet executeAndGet(G globalState, - String... fills) throws Exception { - resultSet = globalState.getConnection().getClient().executeQuery(query); - return null; - } - - public CnosDBResultSet getResultSet() { - return resultSet; - } -} diff --git a/test/sqlancer/dbms/TestCnosDBNoREC.java b/test/sqlancer/dbms/TestCnosDBNoREC.java deleted file mode 100644 index 1a89a972a..000000000 --- a/test/sqlancer/dbms/TestCnosDBNoREC.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestCnosDBNoREC { - - @Test - public void testCnosDBNoREC() { - assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); - // Run with 0 queries as current implementation is resulting in database crashes - assertEquals(0, - Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", - "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", - "--oracle", "NOREC" })); - } - -} diff --git a/test/sqlancer/dbms/TestCnosDBTLP.java b/test/sqlancer/dbms/TestCnosDBTLP.java deleted file mode 100644 index 4b12aa409..000000000 --- a/test/sqlancer/dbms/TestCnosDBTLP.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestCnosDBTLP { - - @Test - public void testCnosDBTLP() { - assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); - // Run with 0 queries as current implementation is resulting in database crashes - assertEquals(0, - Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", - "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", - "--oracle", "QUERY_PARTITIONING" })); - } - -} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index f6be45648..f2372f266 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -5,7 +5,6 @@ public class TestConfig { public static final String SECONDS = "300"; public static final String CLICKHOUSE_ENV = "CLICKHOUSE_AVAILABLE"; - public static final String CNOSDB_ENV = "CNOSDB_AVAILABLE"; public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; From 7ba8bba56c5397d6b6d345add31ac9be420b4a41 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 23:13:37 +0800 Subject: [PATCH 1224/1284] Remove Databend bug19738 workaround after upstream fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue (databendlabs/databend#19738) was fixed in databendlabs/databend#19740 — re-enable AVG in aggregate testing and stop suppressing the related error messages. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/DatabendBugs.java | 1 - src/sqlancer/databend/DatabendErrors.java | 4 ---- .../tlp/DatabendQueryPartitioningAggregateTester.java | 11 ++++------- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e85c5c219..437e40350 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.896-nightly + image: datafuselabs/databend:v1.2.900-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index a058ccc32..dd11512d8 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -19,7 +19,6 @@ public final class DatabendBugs { public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 - public static boolean bug19738 = true; // https://github.com/databendlabs/databend/issues/19738 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 746a4e848..fdd8a3a69 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -47,10 +47,6 @@ public static List getExpressionErrors() { if (DatabendBugs.bug15568) { errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); } - if (DatabendBugs.bug19738) { - errors.add("UnwindError"); - errors.add("unable to cast `NULL`"); - } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 6d52caea3..ee3656413 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -10,7 +10,6 @@ import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -45,12 +44,10 @@ public DatabendQueryPartitioningAggregateTester(DatabendGlobalState state) { @Override public void check() throws SQLException { super.check(); - List aggregateFunctions = new ArrayList<>(List.of(DatabendAggregateFunction.MAX, - DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT - /* , DatabendAggregateFunction.STDDEV_POP */)); - if (!DatabendBugs.bug19738) { - aggregateFunctions.add(DatabendAggregateFunction.AVG); - } + List aggregateFunctions = new ArrayList<>( + List.of(DatabendAggregateFunction.MAX, DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, + DatabendAggregateFunction.COUNT, DatabendAggregateFunction.AVG + /* , DatabendAggregateFunction.STDDEV_POP */)); DatabendAggregateFunction aggregateFunction = Randomly.fromList(aggregateFunctions); DatabendFunctionOperation aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); From cbd284789221a6ee5680ae3c37753fec9cb815e0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 23:15:16 +0800 Subject: [PATCH 1225/1284] Suppress Databend bug19773 (eager-aggregation Decimal mismatch family) Filed as databendlabs/databend#19773. The narrow AVG-only fix in databendlabs/databend#19740 left several related shapes still broken on v1.2.900-nightly, all in the eager-aggregation rewrite path: 1. SUM(decimal_literal) over a cross join inside UNION ALL with an outer aggregate fails with `failed to downcast column Decimal128(...) into ... CoreDecimal`. 2. Plain SUM(decimal_literal) over an N-table cross join fails with `assertion left == right ... Decimal precision: 38 vs 18` once the SUM result is wide enough to be promoted to Decimal128. The original report (#19738) claimed SUM did not trigger the bug; it does, just at higher cardinalities than AVG does. 3. Outer SUM over UNION ALL of inner COUNTs fails with `unable to cast `NULL` to type `UInt64` ... CAST(_eager_final_count (#N) AS UInt64)`. Suppress all three narrowly via distinctive substrings rather than reinstating the broad `UnwindError` / `unable to cast `NULL`` matches the previous commit removed. Co-Authored-By: Claude Opus 4.7 --- src/sqlancer/databend/DatabendBugs.java | 1 + src/sqlancer/databend/DatabendErrors.java | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index dd11512d8..ae2f74a33 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -19,6 +19,7 @@ public final class DatabendBugs { public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 + public static boolean bug19773 = true; // https://github.com/databendlabs/databend/issues/19773 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index fdd8a3a69..3e056d003 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -47,6 +47,11 @@ public static List getExpressionErrors() { if (DatabendBugs.bug15568) { errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); } + if (DatabendBugs.bug19773) { + errors.add("failed to downcast column Decimal128"); + errors.add("Decimal(DecimalSize { precision: 38"); + errors.add("_eager_final_count"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE From da231dd6c28716ec5af893372be7ab6c8f80e7b1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 23:55:28 +0800 Subject: [PATCH 1226/1284] Citus: skip INHERITS generation until citusdata/citus#8553 is fixed Citus's distributed planner returns wrong results when an inheritance parent is cross-joined with a distributed table inside a LEFT JOIN ... ON FALSE and a WHERE filters on the parent column, which the TLP-WHERE oracle keeps tripping over (~1 in 8 Citus CI runs). Make PostgresTableGenerator.generateInherits() protected so CitusTableGenerator can override it as a no-op while the new CitusBugs.bug8553 flag is set. Co-Authored-By: Claude Opus 4.7 --- src/sqlancer/citus/CitusBugs.java | 3 +++ src/sqlancer/citus/gen/CitusTableGenerator.java | 9 +++++++++ src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/citus/CitusBugs.java b/src/sqlancer/citus/CitusBugs.java index a6f4910e6..1fda3be6a 100644 --- a/src/sqlancer/citus/CitusBugs.java +++ b/src/sqlancer/citus/CitusBugs.java @@ -33,6 +33,9 @@ public final class CitusBugs { // https://github.com/citusdata/citus/issues/6298 public static boolean bug6298 = true; + // https://github.com/citusdata/citus/issues/8553 + public static boolean bug8553 = true; + private CitusBugs() { } diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java index 86e5d40cf..5d6b8b249 100644 --- a/src/sqlancer/citus/gen/CitusTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -1,5 +1,6 @@ package sqlancer.citus.gen; +import sqlancer.citus.CitusBugs; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; @@ -20,4 +21,12 @@ public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchem return new CitusTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); } + @Override + protected void generateInherits() { + if (CitusBugs.bug8553) { + return; + } + super.generateInherits(); + } + } diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 29ccfcf2c..9a7bfb032 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -206,7 +206,7 @@ private void generateUsing() { sb.append(globalState.getRandomTableAccessMethod()); } - private void generateInherits() { + protected void generateInherits() { if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) From d2b7151a96c3a482a221969ded411d7ef1dd4cfa Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 30 May 2026 16:39:08 +0100 Subject: [PATCH 1227/1284] Update MySQL CI to MySQL 9.7.0 and JDBC driver to mysql-connector-j 9.7.0 --- .github/workflows/main.yml | 2 +- pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c511fc0a..4543900b7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -444,7 +444,7 @@ jobs: runs-on: ubuntu-latest services: mysql: - image: mysql:8.4 + image: mysql:9.7.0 env: MYSQL_ROOT_PASSWORD: root ports: diff --git a/pom.xml b/pom.xml index 7c9a1106b..c4bc71f82 100644 --- a/pom.xml +++ b/pom.xml @@ -302,9 +302,9 @@ 3.49.1.0 - mysql - mysql-connector-java - 8.0.30 + com.mysql + mysql-connector-j + 9.7.0 org.mariadb.jdbc From 3efe71a4f193fed29a7815ac9d020e1c83f7f6b8 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 30 May 2026 16:46:17 +0100 Subject: [PATCH 1228/1284] Add 'incorrect FLOAT value' as expected error following update to MySQL 9.7.0 --- src/sqlancer/mysql/MySQLErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 989c8fed6..6843a27c3 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -49,6 +49,7 @@ public static List getInsertUpdateErrors() { errors.add("doesn't have a default value"); errors.add("Data truncation"); errors.add("Incorrect integer value"); + errors.add("Incorrect FLOAT value"); errors.add("Duplicate entry"); errors.add("Data truncated for column"); errors.add("Data truncated for functional index"); From 725d31de0e2702ecf7d09a6d4e3417d57493f849 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 30 May 2026 16:49:36 +0100 Subject: [PATCH 1229/1284] Add 'incorrect DOUBLE value' as expected error following update to MySQL 9.7.0 --- src/sqlancer/mysql/MySQLErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 6843a27c3..cb9ad4f01 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -50,6 +50,7 @@ public static List getInsertUpdateErrors() { errors.add("Data truncation"); errors.add("Incorrect integer value"); errors.add("Incorrect FLOAT value"); + errors.add("Incorrect DOUBLE value"); errors.add("Duplicate entry"); errors.add("Data truncated for column"); errors.add("Data truncated for functional index"); From 9dbefd9269463471c5fd3d6199fc182ce993ccfd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 1 Jun 2026 17:32:24 +0100 Subject: [PATCH 1230/1284] Implement workaround for MySQL CREATE INDEX on integer column bug --- src/sqlancer/mysql/MySQLBugs.java | 4 +++ src/sqlancer/mysql/ast/MySQLConstant.java | 9 +++++++ .../mysql/gen/MySQLInsertGenerator.java | 27 +++++++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 8cb8a3391..1f9b7ce2f 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -37,6 +37,10 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=114534 public static boolean bug114534 = true; + // https://bugs.mysql.com/bug.php?id=120711 + // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. + public static boolean bug120711 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index 2e4922f8e..5fb0698b9 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -68,6 +68,11 @@ public MySQLDoubleConstant(double val) { } } + @Override + public double getDouble() { + return val; + } + @Override public String getTextRepresentation() { return String.valueOf(val); @@ -381,6 +386,10 @@ public long getInt() { throw new UnsupportedOperationException(); } + public double getDouble() { + throw new UnsupportedOperationException(); + } + public boolean isSigned() { return false; } diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 86083fd2d..696231a24 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -8,10 +8,14 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLConstant; public class MySQLInsertGenerator { @@ -84,8 +88,27 @@ private SQLQueryAdapter generateInto() { if (c != 0) { sb.append(", "); } - sb.append(MySQLVisitor.asString(gen.generateConstant())); - + MySQLExpression constExpr; + // Bug workaround: for integer columns, reject numeric values that round to 1. Regenerate until valid. + if (MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { + while (true) { + constExpr = gen.generateConstant(); + boolean reject = false; + if (constExpr instanceof MySQLConstant.MySQLIntConstant) { + long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); + reject = value == 1; + } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { + double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); + reject = value >= 0.5 && value < 1.5; + } + if (!reject) { + break; + } + } + } else { + constExpr = gen.generateConstant(); + } + sb.append(MySQLVisitor.asString(constExpr)); } sb.append(")"); } From 2112337c335b2b01e8d6d50a978e853053e70771 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 1 Jun 2026 11:05:16 +0100 Subject: [PATCH 1231/1284] Implement workaround for MySQL DECIMAL UNIQUE bug --- src/sqlancer/mysql/MySQLBugs.java | 4 ++++ src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 1f9b7ce2f..9f0b7589c 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -37,6 +37,10 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=114534 public static boolean bug114534 = true; + // https://bugs.mysql.com/bug.php?id=120710 + // Inserting a NULL and a value which rounds to 0 into a DECIMAL column causes result set mismatch. + public static boolean bug120710 = true; + // https://bugs.mysql.com/bug.php?id=120711 // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. public static boolean bug120711 = true; diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 696231a24..dc2b51bf7 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -105,6 +105,22 @@ private SQLQueryAdapter generateInto() { break; } } + // Bug workaround: for decimal columns, reject values that round to 0. Regenerate until valid. + } else if (MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { + while (true) { + constExpr = gen.generateConstant(); + boolean reject = false; + if (constExpr instanceof MySQLConstant.MySQLIntConstant) { + long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); + reject = value == 0; + } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { + double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); + reject = value >= -0.5 && value < 0.5; + } + if (!reject) { + break; + } + } } else { constExpr = gen.generateConstant(); } From f857438c035761c63836971503847ed8212c16b1 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 8 Jun 2026 10:22:30 +0800 Subject: [PATCH 1232/1284] Make the workaround for MySQL insertion bugs more robust against string generation (which may implicitly cast to undesirable integer/double) --- src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index dc2b51bf7..b57ab5158 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -100,6 +100,8 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= 0.5 && value < 1.5; + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 1 + reject = true; } if (!reject) { break; @@ -116,6 +118,8 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= -0.5 && value < 0.5; + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 0 + reject = true; } if (!reject) { break; From 04b7549e04bdc693367086fda6196ddf5645b20c Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 8 Jun 2026 15:10:18 +0800 Subject: [PATCH 1233/1284] Fix EXPLAIN format following change of default since previous version of MySQL --- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 98641ab26..8af6923e0 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -253,7 +253,7 @@ public List generateFetchColumns(boolean shouldCreateDummy) { @Override public String generateExplainQuery(MySQLSelect select) { - return "EXPLAIN " + select.asString(); + return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed from TRADITIONAL to TREE, hence TRADITIONAL must now be specified } public MySQLAggregate generateAggregate() { From e004455565157ce630499346c20cbe9a2351b100 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 9 Jun 2026 14:40:27 +0800 Subject: [PATCH 1234/1284] Refactor workaround logic for MySQL insertion bugs for easier extension --- .../mysql/gen/MySQLInsertGenerator.java | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index b57ab5158..d15d9def1 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -89,11 +89,13 @@ private SQLQueryAdapter generateInto() { sb.append(", "); } MySQLExpression constExpr; - // Bug workaround: for integer columns, reject numeric values that round to 1. Regenerate until valid. - if (MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { - while (true) { - constExpr = gen.generateConstant(); - boolean reject = false; + // loop to regenerate until expression is valid (for bug workarounds) + while (true) { + constExpr = gen.generateConstant(); + boolean reject = false; + + // Bug workaround: for integer columns, reject values that round to 1 + if (!reject && MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { if (constExpr instanceof MySQLConstant.MySQLIntConstant) { long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); reject = value == 1; @@ -103,15 +105,10 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 1 reject = true; } - if (!reject) { - break; - } } - // Bug workaround: for decimal columns, reject values that round to 0. Regenerate until valid. - } else if (MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { - while (true) { - constExpr = gen.generateConstant(); - boolean reject = false; + + // Bug workaround: for decimal columns, reject values that round to 0 + if (!reject && MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { if (constExpr instanceof MySQLConstant.MySQLIntConstant) { long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); reject = value == 0; @@ -121,12 +118,11 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 0 reject = true; } - if (!reject) { - break; - } } - } else { - constExpr = gen.generateConstant(); + + if (!reject) { + break; + } } sb.append(MySQLVisitor.asString(constExpr)); } From 383d79c8de0f72fc49ce59a5276ea1a023827561 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 9 Jun 2026 15:21:24 +0800 Subject: [PATCH 1235/1284] Implement workaround for MySQL CREATE INDEX between NULL inserts CERT bug --- src/sqlancer/mysql/MySQLBugs.java | 4 ++++ src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 9f0b7589c..8032d78ab 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -45,6 +45,10 @@ public final class MySQLBugs { // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. public static boolean bug120711 = true; + // https://bugs.mysql.com/bug.php?id=120712 + // Creating an index in between two NULL inserts causes inconsistent CERT result. + public static boolean bug120712 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index d15d9def1..a0f44ac9e 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -10,6 +10,7 @@ import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -120,6 +121,13 @@ private SQLQueryAdapter generateInto() { } } + // Bug workaround: if using CERT oracle, reject NULL values + if (!reject && MySQLBugs.bug120712 && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.CERT)) { + if (constExpr instanceof MySQLConstant.MySQLNullConstant) { + reject = true; + } + } + if (!reject) { break; } From 2eced5c4c33b321105d8d9c17ff2069228f1399e Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 12:48:38 +0800 Subject: [PATCH 1236/1284] Implement workaround for oracles reporting inconsistency with ZEROFILL in MySQL despite the behaviour being expected --- src/sqlancer/mysql/MySQLBugs.java | 6 +----- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 2 +- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 5 +++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 8032d78ab..5ad2cfcbd 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -3,12 +3,8 @@ // do not make the fields final to avoid warnings public final class MySQLBugs { - // https://bugs.mysql.com/bug.php?id=99127 0.9 > t0.c0 malfunctions when c0 is - // an INT UNSIGNED - public static boolean bug99127 = true; - // https://bugs.mysql.com/99182 BETWEEN malfunctions for DECIMAL and TEXT - public static boolean bug99181 = true; + public static boolean bug99182 = true; // https://bugs.mysql.com/bug.php?id=99183 public static boolean bug99183 = true; diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 8af6923e0..513867e39 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -112,7 +112,7 @@ public MySQLExpression generateExpression(int depth) { case EXISTS: return getExists(); case BETWEEN_OPERATOR: - if (MySQLBugs.bug99181) { + if (MySQLBugs.bug99182) { // TODO: there are a number of bugs that are triggered by the BETWEEN operator throw new IgnoreMeException(); } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index bc0533295..17e5e9f13 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -13,6 +13,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; @@ -356,10 +357,10 @@ private void appendType(MySQLDataType randomType) { throw new AssertionError(); } if (randomType.isNumeric()) { - if (Randomly.getBoolean() && randomType != MySQLDataType.INT && !MySQLBugs.bug99127) { + if (Randomly.getBoolean() && randomType != MySQLDataType.INT) { sb.append(" UNSIGNED"); } - if (!globalState.usesPQS() && Randomly.getBoolean()) { + if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS || o == MySQLOracleFactory.DQP)) { sb.append(" ZEROFILL"); } } From 4fad414a842a346b788add59608c0a523ef1a737 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 18:24:33 +0800 Subject: [PATCH 1237/1284] Add loop counter for MySQLInsertGenerator regeneration attempts --- src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index a0f44ac9e..a35549405 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -24,6 +24,7 @@ public class MySQLInsertGenerator { private final StringBuilder sb = new StringBuilder(); private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; + private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug workarounds) public MySQLInsertGenerator(MySQLGlobalState globalState, MySQLTable table) { this.globalState = globalState; @@ -91,7 +92,12 @@ private SQLQueryAdapter generateInto() { } MySQLExpression constExpr; // loop to regenerate until expression is valid (for bug workarounds) + int regenerationAttempts = 0; while (true) { + regenerationAttempts++; + if (regenerationAttempts > MAX_REGENERATION_ATTEMPTS) { + throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + " attempts while generating constant for column " + columns.get(c).getName()); + } constExpr = gen.generateConstant(); boolean reject = false; From 5004bc5730e72a9f380de73c4437d3d7dcf76439 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 18:36:32 +0800 Subject: [PATCH 1238/1284] Format to pass CI tests --- src/sqlancer/mysql/MySQLBugs.java | 3 ++- .../mysql/gen/MySQLExpressionGenerator.java | 4 ++- .../mysql/gen/MySQLInsertGenerator.java | 25 +++++++++++-------- .../mysql/gen/MySQLTableGenerator.java | 4 ++- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 5ad2cfcbd..e4fae3cd7 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -38,7 +38,8 @@ public final class MySQLBugs { public static boolean bug120710 = true; // https://bugs.mysql.com/bug.php?id=120711 - // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. + // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set + // mismatch. public static boolean bug120711 = true; // https://bugs.mysql.com/bug.php?id=120712 diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 513867e39..d8ce5dd37 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -253,7 +253,9 @@ public List generateFetchColumns(boolean shouldCreateDummy) { @Override public String generateExplainQuery(MySQLSelect select) { - return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed from TRADITIONAL to TREE, hence TRADITIONAL must now be specified + return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed + // from TRADITIONAL to TREE, hence TRADITIONAL must + // now be specified } public MySQLAggregate generateAggregate() { diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index a35549405..0e464dead 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -7,16 +7,16 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLBugs; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; -import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExpression; public class MySQLInsertGenerator { @@ -24,7 +24,8 @@ public class MySQLInsertGenerator { private final StringBuilder sb = new StringBuilder(); private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; - private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug workarounds) + private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug + // workarounds) public MySQLInsertGenerator(MySQLGlobalState globalState, MySQLTable table) { this.globalState = globalState; @@ -96,7 +97,8 @@ private SQLQueryAdapter generateInto() { while (true) { regenerationAttempts++; if (regenerationAttempts > MAX_REGENERATION_ATTEMPTS) { - throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + " attempts while generating constant for column " + columns.get(c).getName()); + throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + + " attempts while generating constant for column " + columns.get(c).getName()); } constExpr = gen.generateConstant(); boolean reject = false; @@ -109,7 +111,8 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= 0.5 && value < 1.5; - } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 1 + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may + // be implicitly cast to 1 reject = true; } } @@ -122,16 +125,18 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= -0.5 && value < 0.5; - } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 0 + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may + // be implicitly cast to 0 reject = true; } } // Bug workaround: if using CERT oracle, reject NULL values - if (!reject && MySQLBugs.bug120712 && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.CERT)) { - if (constExpr instanceof MySQLConstant.MySQLNullConstant) { - reject = true; - } + if (!reject && MySQLBugs.bug120712 + && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.CERT) + && constExpr instanceof MySQLConstant.MySQLNullConstant) { + reject = true; } if (!reject) { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 17e5e9f13..054a66cb6 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -360,7 +360,9 @@ private void appendType(MySQLDataType randomType) { if (Randomly.getBoolean() && randomType != MySQLDataType.INT) { sb.append(" UNSIGNED"); } - if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS || o == MySQLOracleFactory.DQP)) { + if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS + || o == MySQLOracleFactory.DQP)) { sb.append(" ZEROFILL"); } } From df5f32ecc1f2d52b675d5534ea68880d6439167c Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 20:01:57 +0800 Subject: [PATCH 1239/1284] Modify MySQLDQEOracle from TEXT to VARCHAR to prevent MEMORY-engine incompatibility --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index 8ddb6f315..bb55e585d 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -467,7 +467,7 @@ private List getErrors() throws SQLException { public void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { String tableName = table.getName(); - String addColumnRowID = String.format("ALTER TABLE %s ADD %s TEXT", tableName, COLUMN_ROWID); + String addColumnRowID = String.format("ALTER TABLE %s ADD %s VARCHAR(36)", tableName, COLUMN_ROWID); new SQLQueryAdapter(addColumnRowID).execute(state, false); state.getState().getLocalState().log(addColumnRowID); From f952b256ef9041b2c2b314fa809576625e2f9baa Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 18 Jun 2026 13:49:57 +0800 Subject: [PATCH 1240/1284] Fix MySQLDQEOracle false positives from non-deterministic ORDER BY LIMIT and known SELECT/DML error discrepancies Appends rowId as an ORDER BY tiebreaker to eliminate non-determinism when user columns contain duplicate values (e.g. NULLs) under LIMIT. Introduces isKnownSelectDMLDiscrepancy to suppress false positives from error codes that MySQL legitimately raises in UPDATE/DELETE but not SELECT (or vice versa) due to differing execution paths: WHERE-clause type coercion (1292, 1366), functional index maintenance (1030, 3751), and range optimizer memory limits (3170). --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index bb55e585d..cefcf4589 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -71,6 +71,10 @@ public String generateSelectStatement(AbstractTables tables, String tableN for (MySQLColumn column : Randomly.nonEmptySubset(mySQLTables.getColumns())) { orderColumns.add(column.getFullQualifiedName()); } + // rowId tiebreaker ensures ORDER BY LIMIT is deterministic when user columns have duplicate values + for (MySQLTable table : mySQLTables.getTables()) { + orderColumns.add(table.getName() + "." + COLUMN_ROWID); + } if (Randomly.getBooleanWithRatherLowProbability()) { generateLimit = true; @@ -185,7 +189,13 @@ public void check() throws SQLException { public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult updateResult) { if (updateResult.hasEmptyErrors()) { if (!selectResult.hasEmptyErrors()) { - return "SELECT has errors, but UPDATE does not."; + // Tolerate SELECT-only discrepancy errors (e.g. 1292 raised in SELECT but not UPDATE + // due to different short-circuit evaluation paths). + boolean selectHasNonDiscrepancyErrors = selectResult.getQueryErrors().stream() + .anyMatch(e -> !isKnownSelectDMLDiscrepancy(e)); + if (selectHasNonDiscrepancyErrors) { + return "SELECT has errors, but UPDATE does not."; + } } if (!selectResult.hasSameAccessedRows(updateResult)) { return "SELECT accessed different rows from UPDATE."; @@ -201,9 +211,14 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult } // update errors should all appear in the select errors + // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in UPDATE but not SELECT + // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); + if (isKnownSelectDMLDiscrepancy(updateError)) { + continue; + } if (!isFound(selectErrors, updateError)) { return "SELECT has different errors from UPDATE."; } @@ -247,7 +262,13 @@ private static boolean isFound(List selectErrors, SQLQueryError t public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { if (deleteResult.hasEmptyErrors()) { if (!selectResult.hasEmptyErrors()) { - return "SELECT has errors, but DELETE does not."; + // Tolerate SELECT-only discrepancy errors (e.g. 1292 raised in SELECT but not DELETE + // due to different short-circuit evaluation paths). + boolean selectHasNonDiscrepancyErrors = selectResult.getQueryErrors().stream() + .anyMatch(e -> !isKnownSelectDMLDiscrepancy(e)); + if (selectHasNonDiscrepancyErrors) { + return "SELECT has errors, but DELETE does not."; + } } if (!selectResult.hasSameAccessedRows(deleteResult)) { return "SELECT accessed different rows from DELETE."; @@ -263,9 +284,14 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult } // delete errors should all appear in the select errors + // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in DELETE but not SELECT + // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); + if (isKnownSelectDMLDiscrepancy(deleteError)) { + continue; + } if (!isFound(selectErrors, deleteError)) { return "SELECT has different errors from DELETE."; } @@ -349,6 +375,33 @@ private boolean hasDeleteSpecificErrors(SQLQueryResult deleteResult) { } + /* + * Errors that MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. These are + * acceptable discrepancies and should be skipped when checking that DML errors appear in SELECT errors. They are + * not treated as stop errors (hasStopErrors) so row comparison still proceeds normally. + * + * 1292: Truncated incorrect DOUBLE value — WHERE clause type coercion; MySQL may short-circuit in SELECT but + * evaluate fully in UPDATE/DELETE, raising this at ERROR level vs WARNING in SELECT. 1366: Incorrect + * integer/decimal/float value for column — same WHERE clause coercion discrepancy. 1030: Got error from storage + * engine — raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes so cannot + * produce this error. 3170: range_optimizer_max_mem_size exceeded — MySQL applies this memory budget differently + * for SELECT vs DML; the fallback full-scan still evaluates the WHERE predicate correctly. 3751: Data truncated for + * functional index — raised when a functional index expression truncates a value during DML; structurally + * impossible in SELECT. + */ + private static boolean isKnownSelectDMLDiscrepancy(SQLQueryError error) { + switch (error.getCode()) { + case 1030: + case 1292: + case 1366: + case 3170: + case 3751: + return true; + default: + return false; + } + } + private boolean hasStopErrors(SQLQueryResult queryResult) { return queryResult.getQueryErrors().stream() .anyMatch(error -> error.getLevel() == SQLQueryError.ErrorLevel.ERROR); From 2d50787b9f93539d35ccd41cf64aa35f2acdfe58 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 10:10:05 +0800 Subject: [PATCH 1241/1284] Move MySQLDQEOracle DML discrepancies into enum for maintainability --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index cefcf4589..429ef3c89 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -211,8 +211,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult } // update errors should all appear in the select errors - // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in UPDATE but not SELECT - // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. + // known SELECT/DML discrepancy errors are skipped: see KnownSelectDMLDiscrepancy for the full list. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); @@ -284,8 +283,7 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult } // delete errors should all appear in the select errors - // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in DELETE but not SELECT - // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. + // known SELECT/DML discrepancy errors are skipped: see KnownSelectDMLDiscrepancy for the full list. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); @@ -375,31 +373,36 @@ private boolean hasDeleteSpecificErrors(SQLQueryResult deleteResult) { } - /* - * Errors that MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. These are - * acceptable discrepancies and should be skipped when checking that DML errors appear in SELECT errors. They are - * not treated as stop errors (hasStopErrors) so row comparison still proceeds normally. - * - * 1292: Truncated incorrect DOUBLE value — WHERE clause type coercion; MySQL may short-circuit in SELECT but - * evaluate fully in UPDATE/DELETE, raising this at ERROR level vs WARNING in SELECT. 1366: Incorrect - * integer/decimal/float value for column — same WHERE clause coercion discrepancy. 1030: Got error from storage - * engine — raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes so cannot - * produce this error. 3170: range_optimizer_max_mem_size exceeded — MySQL applies this memory budget differently - * for SELECT vs DML; the fallback full-scan still evaluates the WHERE predicate correctly. 3751: Data truncated for - * functional index — raised when a functional index expression truncates a value during DML; structurally - * impossible in SELECT. - */ + // Errors MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. Acceptable + // discrepancies that should be skipped; not treated as stop errors so row comparison proceeds normally. + private enum KnownSelectDMLDiscrepancy { + // WHERE clause type coercion: MySQL may short-circuit in SELECT but evaluate fully in UPDATE/DELETE, + // raising this at ERROR level vs WARNING in SELECT. + TRUNCATED_DOUBLE_VALUE(1292), + // Same WHERE clause coercion discrepancy as TRUNCATED_DOUBLE_VALUE. + INCORRECT_COLUMN_VALUE(1366), + // Raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes. + STORAGE_ENGINE_ERROR(1030), + // MySQL applies this memory budget differently for SELECT vs DML; the fallback full-scan still + // evaluates the WHERE predicate correctly. + RANGE_OPTIMIZER_MEM_EXCEEDED(3170), + // Raised when a functional index expression truncates a value during DML; structurally impossible in SELECT. + FUNCTIONAL_INDEX_DATA_TRUNCATED(3751); + + private final int code; + + KnownSelectDMLDiscrepancy(int code) { + this.code = code; + } + } + private static boolean isKnownSelectDMLDiscrepancy(SQLQueryError error) { - switch (error.getCode()) { - case 1030: - case 1292: - case 1366: - case 3170: - case 3751: - return true; - default: - return false; + for (KnownSelectDMLDiscrepancy discrepancy : KnownSelectDMLDiscrepancy.values()) { + if (discrepancy.code == error.getCode()) { + return true; + } } + return false; } private boolean hasStopErrors(SQLQueryResult queryResult) { From 9efa585c35fe25594a0bf1f4157974e775925849 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 10:47:47 +0800 Subject: [PATCH 1242/1284] Ignore expected prefix-key-on-partitioned-table error in MySQLIndexGenerator MySQL rejects CREATE INDEX with a prefix key part (e.g. c0(3)) on a column that participates in PARTITION BY KEY(). MySQLIndexGenerator generated these without checking partition membership and did not include this error in ExpectedErrors, causing checkException to escalate it to a fatal AssertionError. Adds the error substring to the expected errors list. --- src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index 550893db5..028886831 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -120,6 +120,8 @@ public SQLQueryAdapter create() { errors.add("Data truncated for functional index"); errors.add("used in key specification without a key length"); errors.add("Row size too large"); // seems to happen together with MIN_ROWS in the table declaration + errors.add("in the PARTITION BY KEY() clause is not supported"); // prefix key parts disallowed on + // KEY-partitioned columns return new SQLQueryAdapter(string, errors, true); } From 8fd48d3d3d092485a0c730887a083f925cc4c2ca Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 13:05:14 +0800 Subject: [PATCH 1243/1284] Fix YugabyteDB CI by adding ysql_yb_enable_listen_notify flag so that LISTEN and NOTIFY statements are enabled --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c511fc0a..91d19197f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -646,7 +646,7 @@ jobs: - name: Set up Yugabyte run: | docker pull yugabytedb/yugabyte:latest - docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false + docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false --tserver_flags="ysql_yb_enable_listen_notify=true" --master_flags="ysql_yb_enable_listen_notify=true" until pg_isready -h localhost -p 5433; do sleep 1; done - name: Run Tests run: | From 376b9d4c1f6b3118a4926af35e35d163ce73beed Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 13:34:22 +0800 Subject: [PATCH 1244/1284] Add YCQL readiness check for YugabyteDB so that TestYCQL does not run prematurely --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91d19197f..02507d58f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -648,6 +648,7 @@ jobs: docker pull yugabytedb/yugabyte:latest docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false --tserver_flags="ysql_yb_enable_listen_notify=true" --master_flags="ysql_yb_enable_listen_notify=true" until pg_isready -h localhost -p 5433; do sleep 1; done + until nc -z localhost 9042; do sleep 1; done - name: Run Tests run: | YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLNoREC test From 2821cbd02890185a210739c42028356b4aa41222 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 11 Jul 2026 20:01:22 +0800 Subject: [PATCH 1245/1284] Remove NoRECBase and DocumentRemovalOracleBase, as these were dead code --- .../oracle/DocumentRemovalOracleBase.java | 29 ------------------- src/sqlancer/common/oracle/NoRECBase.java | 26 ----------------- 2 files changed, 55 deletions(-) delete mode 100644 src/sqlancer/common/oracle/DocumentRemovalOracleBase.java delete mode 100644 src/sqlancer/common/oracle/NoRECBase.java diff --git a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java b/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java deleted file mode 100644 index b6c0ee509..000000000 --- a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java +++ /dev/null @@ -1,29 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.GlobalState; -import sqlancer.common.gen.ExpressionGenerator; - -public abstract class DocumentRemovalOracleBase> implements TestOracle { - - protected E predicate; - - protected final S state; - - protected DocumentRemovalOracleBase(S state) { - this.state = state; - } - - protected void initializeDocumentRemovalOracle() { - ExpressionGenerator gen = getGen(); - if (gen == null) { - throw new IllegalStateException(); - } - predicate = gen.generatePredicate(); - if (predicate == null) { - throw new IllegalStateException(); - } - } - - protected abstract ExpressionGenerator getGen(); - -} diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java deleted file mode 100644 index 2ac0dbb43..000000000 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ /dev/null @@ -1,26 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; -import sqlancer.common.query.ExpectedErrors; - -public abstract class NoRECBase> implements TestOracle { - - protected final S state; - protected final ExpectedErrors errors = new ExpectedErrors(); - protected final StateLogger logger; - protected final MainOptions options; - protected final SQLConnection con; - protected String optimizedQueryString; - protected String unoptimizedQueryString; - - protected NoRECBase(S state) { - this.state = state; - this.con = state.getConnection(); - this.logger = state.getLogger(); - this.options = state.getOptions(); - } - -} From f984df4cece25768c588774e992f36681bd48655 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 13 Jul 2026 18:30:37 +0800 Subject: [PATCH 1246/1284] Fix TLPWhereReproducer bug that prevented reducer from performing any reduction --- src/sqlancer/MainOptions.java | 4 ++-- src/sqlancer/common/oracle/TLPWhereOracle.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index a5142fcf0..25b769312 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -126,10 +126,10 @@ public class MainOptions { @Parameter(names = "--serialize-reproduce-state", description = "Serialize the state to reproduce") private boolean serializeReproduceState = false; // NOPMD - @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") + @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer. Implemented for TLP WHERE and NoREC only") private boolean useReducer = false; // NOPMD - @Parameter(names = "--reduce-ast", description = "EXPERIMENTAL perform AST reduction after statement reduction") + @Parameter(names = "--reduce-ast", description = "EXPERIMENTAL Perform AST reduction after statement reduction") private boolean reduceAST = false; // NOPMD @Parameter(names = "--statement-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the statement reducer will do") diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 14834a62f..31ecc4481 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -106,15 +106,15 @@ public void check() throws SQLException { select.setWhereClause(predicates.isNullPredicate); String thirdQueryString = select.asString(); + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, + firstResultSet, orderBy); + List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); - - reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - firstResultSet, orderBy); } @Override From 73157498bda4fd958e8ac829f39cb4560b2e1284 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 13 Jul 2026 19:06:42 +0800 Subject: [PATCH 1247/1284] Fix bug where reducer logs were being overwritten, causing logs to end up empty --- src/sqlancer/Main.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index f273f5b95..33cf5fbe3 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -502,11 +502,13 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } - try { - logger.getReduceFileWriter().close(); - logger.reduceFileWriter = null; - } catch (IOException e) { - throw new AssertionError(e); + if (logger.reduceFileWriter != null) { + try { + logger.reduceFileWriter.close(); + logger.reduceFileWriter = null; + } catch (IOException e) { + throw new AssertionError(e); + } } throw new AssertionError("Found a potential bug, please check reducer log for detail."); From 2b3b28b051e75dceb919daf12638b0a0dc817ffe Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 14 Jul 2026 13:46:43 +0800 Subject: [PATCH 1248/1284] Fix bug where reducer always reduced to single statement --- src/sqlancer/common/oracle/NoRECOracle.java | 16 ++++++++-- .../common/oracle/TLPWhereOracle.java | 29 ++++++++++++------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index caf3dff87..0f043a07f 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -1,7 +1,6 @@ package sqlancer.common.oracle; import java.sql.SQLException; -import java.util.Objects; import java.util.function.Function; import sqlancer.IgnoreMeException; @@ -42,7 +41,20 @@ private static class NoRECReproducer> implements @Override public boolean bugStillTriggers(G globalState) { - return !Objects.equals(optimizedQuery.apply(globalState), unoptimizedQuery.apply(globalState)); + int optimizedCount; + int unoptimizedCount; + try { + optimizedCount = optimizedQuery.apply(globalState); + unoptimizedCount = unoptimizedQuery.apply(globalState); + } catch (RuntimeException | AssertionError e) { + // the queries could not be executed on the reduced database (e.g., a statement they + // depend on was removed), which is not the count mismatch that is being reduced + return false; + } + if (optimizedCount == -1 || unoptimizedCount == -1) { + return false; + } + return optimizedCount != unoptimizedCount; } } diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 31ecc4481..60b8d0c29 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -34,30 +34,37 @@ private class TLPWhereReproducer implements Reproducer { final String secondQueryString; final String thirdQueryString; final String originalQueryString; - final List resultSet; final boolean orderBy; TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, - String originalQueryString, List resultSet, boolean orderBy) { + String originalQueryString, boolean orderBy) { this.firstQueryString = firstQueryString; this.secondQueryString = secondQueryString; this.thirdQueryString = thirdQueryString; this.originalQueryString = originalQueryString; - this.resultSet = resultSet; this.orderBy = orderBy; } @Override public boolean bugStillTriggers(G globalState) { + List firstResultSet; + List combinedString = new ArrayList<>(); + List secondResultSet; try { - List combinedString1 = new ArrayList<>(); - List secondResultSet1 = ComparatorHelper.getCombinedResultSet(firstQueryString, - secondQueryString, thirdQueryString, combinedString1, !orderBy, globalState, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet1, originalQueryString, - combinedString1, globalState); - } catch (AssertionError triggeredError) { + firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + globalState); + secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, globalState, errors); + } catch (SQLException | RuntimeException | AssertionError e) { + // the queries could not be executed on the reduced database (e.g., a statement they + // depend on was removed), which is not the result set mismatch that is being reduced + return false; + } + try { + ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, + combinedString, globalState); + } catch (AssertionError resultSetMismatch) { return true; - } catch (SQLException ignored) { } return false; } @@ -107,7 +114,7 @@ public void check() throws SQLException { String thirdQueryString = select.asString(); reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - firstResultSet, orderBy); + orderBy); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, From 7d11ca0bb1b76446c18751cd46058b2c9c801ce8 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 14 Jul 2026 13:56:38 +0800 Subject: [PATCH 1249/1284] Extend reducer to allow reducing DBMS errors (i.e. errors other than oracle logic bugs) --- src/sqlancer/Main.java | 14 +++- src/sqlancer/Reproducer.java | 10 +++ src/sqlancer/common/oracle/NoRECOracle.java | 67 ++++++++++++---- .../common/oracle/TLPWhereOracle.java | 76 ++++++++++++++++--- .../common/oracle/TestOracleUtils.java | 23 ++++++ 5 files changed, 162 insertions(+), 28 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 33cf5fbe3..d65649049 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -502,10 +502,18 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } - if (logger.reduceFileWriter != null) { + String bugInformation = reproducer.getBugInformation(); + if (bugInformation != null) { + // log through newGlobalState's logger: it already holds the reduce file + // writer, and opening it through another StateLogger truncates the file + newGlobalState.getLogger().logReducer(bugInformation); + } + + StateLogger reduceLogger = newGlobalState.getLogger(); + if (reduceLogger.reduceFileWriter != null) { try { - logger.reduceFileWriter.close(); - logger.reduceFileWriter = null; + reduceLogger.reduceFileWriter.close(); + reduceLogger.reduceFileWriter = null; } catch (IOException e) { throw new AssertionError(e); } diff --git a/src/sqlancer/Reproducer.java b/src/sqlancer/Reproducer.java index ef64bd0fe..460cc810d 100644 --- a/src/sqlancer/Reproducer.java +++ b/src/sqlancer/Reproducer.java @@ -2,4 +2,14 @@ public interface Reproducer> { boolean bugStillTriggers(G globalState); + + /** + * Describes how to trigger the bug on the database set up by the reduced statements (e.g., the oracle queries to + * run and the failure to expect), so that the reduced test case is complete without the reproducer object. + * + * @return a human-readable description, or null if the reproducer does not provide one + */ + default String getBugInformation() { + return null; + } } diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 0f043a07f..35f0c26af 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -33,10 +33,19 @@ public class NoRECOracle, J extends Join, private static class NoRECReproducer> implements Reproducer { private final Function optimizedQuery; private final Function unoptimizedQuery; - - NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery) { + private final String optimizedQueryString; + private final String unoptimizedQueryString; + // null if the original bug is a count mismatch; otherwise, the message of the unexpected + // DBMS error that the original queries triggered + private final String expectedErrorMessage; + + NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, + String optimizedQueryString, String unoptimizedQueryString, String expectedErrorMessage) { this.optimizedQuery = optimizedQuery; this.unoptimizedQuery = unoptimizedQuery; + this.optimizedQueryString = optimizedQueryString; + this.unoptimizedQueryString = unoptimizedQueryString; + this.expectedErrorMessage = expectedErrorMessage; } @Override @@ -46,9 +55,16 @@ public boolean bugStillTriggers(G globalState) { try { optimizedCount = optimizedQuery.apply(globalState); unoptimizedCount = unoptimizedQuery.apply(globalState); - } catch (RuntimeException | AssertionError e) { - // the queries could not be executed on the reduced database (e.g., a statement they - // depend on was removed), which is not the count mismatch that is being reduced + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs return false; } if (optimizedCount == -1 || unoptimizedCount == -1) { @@ -56,6 +72,22 @@ public boolean bugStillTriggers(G globalState) { } return optimizedCount != unoptimizedCount; } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the row counts of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); + sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + return sb.toString(); + } } public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { @@ -94,21 +126,28 @@ public void check() throws SQLException { state.getLogger().writeCurrent(unoptimizedQueryString); } - int optimizedCount = shouldUseAggregate ? extractCounts(optimizedQueryString, errors, state) - : countRows(optimizedQueryString, errors, state); - int unoptimizedCount = extractCounts(unoptimizedQueryString, errors, state); + Function optimizedQuery = state -> shouldUseAggregate + ? extractCounts(optimizedQueryString, errors, state) : countRows(optimizedQueryString, errors, state); + Function unoptimizedQuery = state -> extractCounts(unoptimizedQueryString, errors, state); + + int optimizedCount; + int unoptimizedCount; + try { + optimizedCount = optimizedQuery.apply(state); + unoptimizedCount = unoptimizedQuery.apply(state); + } catch (AssertionError unexpectedError) { + reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, + unoptimizedQueryString, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } if (optimizedCount == -1 || unoptimizedCount == -1) { throw new IgnoreMeException(); } if (unoptimizedCount != optimizedCount) { - Function optimizedQuery = state -> shouldUseAggregate - ? extractCounts(optimizedQueryString, errors, state) - : countRows(optimizedQueryString, errors, state); - - Function unoptimizedQuery = state -> extractCounts(unoptimizedQueryString, errors, state); - reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery); + reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, + unoptimizedQueryString, null); String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 60b8d0c29..968064158 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -35,14 +35,18 @@ private class TLPWhereReproducer implements Reproducer { final String thirdQueryString; final String originalQueryString; final boolean orderBy; + // null if the original bug is a result set mismatch; otherwise, the message of the + // unexpected DBMS error that the original queries triggered + final String expectedErrorMessage; TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, - String originalQueryString, boolean orderBy) { + String originalQueryString, boolean orderBy, String expectedErrorMessage) { this.firstQueryString = firstQueryString; this.secondQueryString = secondQueryString; this.thirdQueryString = thirdQueryString; this.originalQueryString = originalQueryString; this.orderBy = orderBy; + this.expectedErrorMessage = expectedErrorMessage; } @Override @@ -53,11 +57,23 @@ public boolean bugStillTriggers(G globalState) { try { firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + if (firstQueryString == null) { + // the original bug was a DBMS error on the original query alone, which no + // longer occurs + return false; + } secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, globalState, errors); - } catch (SQLException | RuntimeException | AssertionError e) { - // the queries could not be executed on the reduced database (e.g., a statement they - // depend on was removed), which is not the result set mismatch that is being reduced + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs return false; } try { @@ -68,6 +84,32 @@ public boolean bugStillTriggers(G globalState) { } return false; } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (firstQueryString != null) { + if (orderBy) { + sb.append("-- ").append(firstQueryString).append(';').append(System.lineSeparator()); + sb.append("-- ").append(secondQueryString).append(';').append(System.lineSeparator()); + sb.append("-- ").append(thirdQueryString).append(';').append(System.lineSeparator()); + } else { + sb.append("-- ").append(firstQueryString).append(" UNION ALL ").append(secondQueryString) + .append(" UNION ALL ").append(thirdQueryString).append(';') + .append(System.lineSeparator()); + } + } + return sb.toString(); + } } public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { @@ -96,8 +138,14 @@ public void check() throws SQLException { String originalQueryString = select.asString(); generatedQueryString = originalQueryString; - List firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - state); + List firstResultSet; + try { + firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + } catch (AssertionError unexpectedError) { + reproducer = new TLPWhereReproducer(null, null, null, originalQueryString, false, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } boolean orderBy = Randomly.getBooleanWithSmallProbability(); if (orderBy) { @@ -113,13 +161,19 @@ public void check() throws SQLException { select.setWhereClause(predicates.isNullPredicate); String thirdQueryString = select.asString(); - reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - orderBy); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); + List secondResultSet; + try { + secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + } catch (AssertionError unexpectedError) { + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, + originalQueryString, orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, + orderBy, null); ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); } diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java index bab2e26c9..5233cfa80 100644 --- a/src/sqlancer/common/oracle/TestOracleUtils.java +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -34,6 +34,29 @@ public static final class PredicateVariants, C extends A return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables())); } + /** + * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the + * AssertionError that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it + * to check that a reduced test case still triggers the same error, rather than an unrelated one introduced by the + * reduction itself. + * + * @param error + * the AssertionError wrapping the DBMS error + * + * @return the message of the innermost cause that has one, or the error's own message + */ + public static String getUnexpectedErrorMessage(AssertionError error) { + String message = error.getMessage(); + Throwable current = error.getCause(); + while (current != null) { + if (current.getMessage() != null) { + message = current.getMessage(); + } + current = current.getCause(); + } + return message; + } + public static , T extends AbstractTable, C extends AbstractTableColumn> PredicateVariants initializeTernaryPredicateVariants( PartitionGenerator gen, E predicate) { if (gen == null) { From a02b149a65598b8af893717ceacee764f9d6c395 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 14 Jul 2026 14:47:01 +0800 Subject: [PATCH 1250/1284] Clarify logs when --use-reducer is applied (the log outside of the reduce directory gives the final reduced test case, while the log inside the reduce directory gives the step-by-step reduction) --- src/sqlancer/ASTBasedReducer.java | 6 +- src/sqlancer/Main.java | 76 +++++++++++++------ src/sqlancer/StateToReproduce.java | 4 + src/sqlancer/StatementReducer.java | 3 +- .../common/oracle/TLPWhereOracle.java | 3 +- .../common/oracle/TestOracleUtils.java | 7 +- 6 files changed, 66 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java index 876a2da12..f9468af76 100644 --- a/src/sqlancer/ASTBasedReducer.java +++ b/src/sqlancer/ASTBasedReducer.java @@ -103,8 +103,7 @@ public void reduce(G state, Reproducer reproducer, G newGlobalState) throws E }); if (!initFlag) { - newGlobalState.getLogger() - .logReducer("warning: failed parsing the statement at transformer : " + t); + System.out.println("Error when parsing the statement at transformer :" + t); continue; } t.apply(); @@ -114,7 +113,8 @@ public void reduce(G state, Reproducer reproducer, G newGlobalState) throws E } while (observeChange); newGlobalState.getState().setStatements(new ArrayList<>(reducedStatements)); - newGlobalState.getLogger().logReduced(newGlobalState.getState()); + newGlobalState.getLogger().logReduced(newGlobalState.getState(), + "AST-based reduction finished; the following statements remain"); } public boolean bugStillTriggers() throws Exception { diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index d65649049..47ba2aedf 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -80,6 +80,9 @@ public static final class StateLogger { private FileWriter queryPlanFileWriter; private FileWriter reduceFileWriter; private Path reproduceFilePath; + private List> reduceSetupStatements; + private String reduceBugInformation; + private int nrReductionAttempts; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; @@ -262,33 +265,32 @@ public void writeQueryPlan(String queryPlan) { } } - public void logReducer(String reducerLog) { - FileWriter reduceFileWriter = getReduceFileWriter(); - - StringBuilder sb = new StringBuilder(); - sb.append("[reducer log] "); - sb.append(reducerLog); - try { - reduceFileWriter.write(sb.toString()); - } catch (IOException e) { - throw new AssertionError(e); - } finally { - try { - reduceFileWriter.flush(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } + public void setReductionContext(List> setupStatements, String bugInformation) { + this.reduceSetupStatements = setupStatements; + this.reduceBugInformation = bugInformation; } public void logReduced(StateToReproduce state) { + nrReductionAttempts++; + logReduced(state, "Reduction attempt " + nrReductionAttempts + + ": the bug was still triggered with the following statements"); + } + + public void logReduced(StateToReproduce state, String description) { FileWriter reduceFileWriter = getReduceFileWriter(); StringBuilder sb = new StringBuilder(); - for (Query s : state.getStatements()) { - sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); + sb.append("-- ").append(description).append(System.lineSeparator()); + if (reduceSetupStatements != null && !reduceSetupStatements.isEmpty()) { + appendStatements(sb, reduceSetupStatements); + // e.g. DROP DATABASE IF EXISTS db; CREATE DATABASE db; USE db; + // these statements are executed at the start of every test case and are never reduced + } + appendStatements(sb, state.getStatements()); + if (reduceBugInformation != null) { + sb.append(reduceBugInformation); } + sb.append(System.lineSeparator()); try { reduceFileWriter.write(sb.toString()); @@ -305,6 +307,12 @@ public void logReduced(StateToReproduce state) { } + private void appendStatements(StringBuilder sb, List> statements) { + for (Query s : statements) { + sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); + } + } + public void logException(Throwable reduce, StateToReproduce state) { Loggable stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); @@ -460,6 +468,9 @@ public void run() throws Exception { if (options.logEachSelect()) { logger.writeCurrent(state.getState()); } + // statements logged so far stem from the database setup (e.g., DROP DATABASE IF + // EXISTS, CREATE DATABASE, USE), performed by createDatabase + int nrSetupStatements = stateToRepro.getStatements().size(); Reproducer reproducer = null; if (options.enableQPG()) { provider.generateAndTestDatabaseWithQueryPlanGuidance(state); @@ -484,6 +495,17 @@ public void run() throws Exception { logger.getReduceFileWriter().write("current oracle does not support experimental reducer."); throw new IgnoreMeException(); } + + // reduce only the generation statements: the database setup (logged by + // createDatabase) is re-executed by the reducers for every candidate, and the + // oracle queries (logged by the oracle's local state) by the reproducer + List> allStatements = new ArrayList<>(stateToRepro.getStatements()); + List> setupStatements = new ArrayList<>(allStatements.subList(0, nrSetupStatements)); + List> oracleQueryStatements = stateToRepro.getLocalState() == null ? new ArrayList<>() + : new ArrayList<>(stateToRepro.getLocalState().getStatements()); + stateToRepro.setStatements(new ArrayList<>(allStatements.subList(nrSetupStatements, + allStatements.size() - oracleQueryStatements.size()))); + G newGlobalState = createGlobalState(); newGlobalState.setState(stateToRepro); newGlobalState.setRandomly(r); @@ -493,6 +515,7 @@ public void run() throws Exception { QueryManager newManager = new QueryManager<>(newGlobalState); newGlobalState.setStateLogger(new StateLogger(databaseName, provider, options)); newGlobalState.setManager(newManager); + newGlobalState.getLogger().setReductionContext(setupStatements, reproducer.getBugInformation()); Reducer reducer = new StatementReducer<>(provider); reducer.reduce(state, reproducer, newGlobalState); @@ -502,11 +525,18 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } + // reassemble the statements so that the main log looks like one produced + // without the reducer, with the generation statements replaced by the reduced + // ones and the oracle queries at the end + List> finalStatements = new ArrayList<>(setupStatements); + finalStatements.addAll(stateToRepro.getStatements()); + finalStatements.addAll(oracleQueryStatements); + stateToRepro.setStatements(finalStatements); String bugInformation = reproducer.getBugInformation(); if (bugInformation != null) { - // log through newGlobalState's logger: it already holds the reduce file - // writer, and opening it through another StateLogger truncates the file - newGlobalState.getLogger().logReducer(bugInformation); + for (String line : bugInformation.split(System.lineSeparator())) { + stateToRepro.logStatement(line); + } } StateLogger reduceLogger = newGlobalState.getLogger(); diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index e44d0ccf6..17bb367fd 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -128,6 +128,10 @@ public void log(String s) { statements.add(databaseProvider.getLoggableFactory().getQueryForStateToReproduce(s)); } + public List> getStatements() { + return Collections.unmodifiableList(statements); + } + @Override public void close() { if (!success) { diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java index e066aca84..6545fb2af 100644 --- a/src/sqlancer/StatementReducer.java +++ b/src/sqlancer/StatementReducer.java @@ -77,7 +77,8 @@ && hasNotReachedLimit(currentReduceTime, maxReduceTime)) { // System.out.println("Reduced query:"); // printQueries(knownToReproduceBugStatements); newGlobalState.getState().setStatements(new ArrayList<>(knownToReproduceBugStatements)); - newGlobalState.getLogger().logReduced(newGlobalState.getState()); + newGlobalState.getLogger().logReduced(newGlobalState.getState(), + "Statement reduction finished; the following statements remain"); } diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 968064158..8fc87ac3d 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -104,8 +104,7 @@ public String getBugInformation() { sb.append("-- ").append(thirdQueryString).append(';').append(System.lineSeparator()); } else { sb.append("-- ").append(firstQueryString).append(" UNION ALL ").append(secondQueryString) - .append(" UNION ALL ").append(thirdQueryString).append(';') - .append(System.lineSeparator()); + .append(" UNION ALL ").append(thirdQueryString).append(';').append(System.lineSeparator()); } } return sb.toString(); diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java index 5233cfa80..9bef86762 100644 --- a/src/sqlancer/common/oracle/TestOracleUtils.java +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -35,10 +35,9 @@ public static final class PredicateVariants, C extends A } /** - * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the - * AssertionError that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it - * to check that a reduced test case still triggers the same error, rather than an unrelated one introduced by the - * reduction itself. + * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the AssertionError + * that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it to check that a + * reduced test case still triggers the same error, rather than an unrelated one introduced by the reduction itself. * * @param error * the AssertionError wrapping the DBMS error From abbdeed6cbe4bad3c9c7e43b92c880ab47cedf5a Mon Sep 17 00:00:00 2001 From: splf Date: Mon, 27 Jul 2026 23:03:27 +0500 Subject: [PATCH 1251/1284] Fix MySQL index hint syntax in the DQP hint generator MySQL's grammar for index-level optimizer hints is hint_name(tbl_name index_name [, index_name] ...): a space separates the table name from the first index name, not a comma. MySQLHintGenerator emitted NO_INDEX(t0, PRIMARY), which MySQL rejects with warning 1064 and silently ignores, leaving the query plan unchanged. This affects the 16 of 32 hint kinds that go through indexesHint(). --- src/sqlancer/mysql/gen/MySQLHintGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLHintGenerator.java b/src/sqlancer/mysql/gen/MySQLHintGenerator.java index 141aea279..dc6138b23 100644 --- a/src/sqlancer/mysql/gen/MySQLHintGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLHintGenerator.java @@ -170,7 +170,7 @@ private void indexesHint(String string) { MySQLTable table = Randomly.fromList(tables); List allIndexes = table.getIndexes(); sb.append(table.getName()); - sb.append(", "); + sb.append(" "); if (allIndexes.isEmpty()) { sb.append("PRIMARY"); } else { From eb5e013a227eae7834c5b1f3865c451e57bb3f51 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 9 Jul 2026 13:21:12 +0800 Subject: [PATCH 1252/1284] Add EET (Equivalent Expression Transformation) oracle for MySQL This commit implements EET from Jiang & Su, "Detecting Logic Bugs in Database Engines via Equivalent Expression Transformation" (OSDI'24). Common, DBMS-independent core: - EETTransformation: the 7 transformation rules of the paper's Table 2. - EETNodeFactory: primitive node constructors the rules are built from. - EETGenerator: generator interface an EET-capable DBMS implements. - EETOracle: orchestration and result comparison (paper Figure 4), with a Reproducer for confirmation/reduction. MySQL-specific implementation: - MySQLEETTransformer recursively rewrites the MySQL AST, rebuilding each node from its transformed children and threading a boolean/scalar context flag so the determined-boolean rules are only applied where sound. - MySQLEETNodeFactory builds the MySQL nodes; MySQLExpressionGenerator implements EETGenerator; MySQLUnaryPrefixOperation exposes getOp(); MySQLOracleFactory registers the EET oracle. - MySQLTableGenerator skips ZEROFILL for EET (as for TLP_WHERE/PQS/DQP) to avoid display-metadata-only false positives while preserving search space. Runs via: java -jar target/sqlancer-*.jar mysql --oracle EET --- src/sqlancer/common/gen/EETGenerator.java | 44 +++++++ .../common/oracle/EETNodeFactory.java | 39 ++++++ src/sqlancer/common/oracle/EETOracle.java | 121 +++++++++++++++++ .../common/oracle/EETTransformation.java | 83 ++++++++++++ src/sqlancer/mysql/MySQLOracleFactory.java | 10 ++ .../mysql/ast/MySQLUnaryPrefixOperation.java | 4 + .../mysql/gen/MySQLEETNodeFactory.java | 70 ++++++++++ .../mysql/gen/MySQLEETTransformer.java | 124 ++++++++++++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 13 +- .../mysql/gen/MySQLTableGenerator.java | 2 +- 10 files changed, 508 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/common/gen/EETGenerator.java create mode 100644 src/sqlancer/common/oracle/EETNodeFactory.java create mode 100644 src/sqlancer/common/oracle/EETOracle.java create mode 100644 src/sqlancer/common/oracle/EETTransformation.java create mode 100644 src/sqlancer/mysql/gen/MySQLEETNodeFactory.java create mode 100644 src/sqlancer/mysql/gen/MySQLEETTransformer.java diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java new file mode 100644 index 000000000..f468ff6d2 --- /dev/null +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -0,0 +1,44 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like + * the other oracle generators), an EET generator can transform an expression into a semantically equivalent one + * according to the EET transformation rules. + */ +public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + EETGenerator setTablesAndColumns(AbstractTables tables); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + E generateBooleanExpression(); + + /** + * Transforms an expression into a semantically equivalent one (the core of EET). Typically this recursively + * traverses the expression's AST and replaces sub-expressions with equivalent ones. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (e.g. a WHERE predicate); this controls + * which transformation rules are applicable + * + * @return a semantically equivalent expression + */ + E transformExpression(E expr, boolean booleanContext); +} diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java new file mode 100644 index 000000000..2cc933ced --- /dev/null +++ b/src/sqlancer/common/oracle/EETNodeFactory.java @@ -0,0 +1,39 @@ +package sqlancer.common.oracle; + +/** + * Factory for constructing the AST nodes needed by the {@link EETTransformation equivalent expression transformation} + * rules. Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface + * lets the (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. + * + * @param + * the DBMS-specific expression type + */ +public interface EETNodeFactory { + + /** Builds {@code left AND right}. */ + E and(E left, E right); + + /** Builds {@code left OR right}. */ + E or(E left, E right); + + /** Builds {@code NOT expr}. */ + E not(E expr); + + /** Builds {@code expr IS NULL}. */ + E isNull(E expr); + + /** Builds {@code expr IS NOT NULL}. */ + E isNotNull(E expr); + + /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ + E caseWhen(E condition, E thenExpr, E elseExpr); + + /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ + E generateBooleanExpression(); + + /** + * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not + * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + */ + boolean isCaseWhenApplicable(E expr); +} diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java new file mode 100644 index 000000000..0f192f9ab --- /dev/null +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -0,0 +1,121 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.EETGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * EET (Equivalent Expression Transformation) oracle, based on "Detecting Logic Bugs in Database Engines via Equivalent + * Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * The oracle generates a random query and then transforms its expressions (the WHERE predicate and the fetch columns) + * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation preserves + * semantics, the original and the transformed query must return the same result set; any discrepancy indicates a logic + * bug in the DBMS. + */ +public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private EETGenerator gen; + private final ExpectedErrors errors; + + private Reproducer reproducer; + private String generatedQueryString; + + private final class EETReproducer implements Reproducer { + private final String originalQueryString; + private final String transformedQueryString; + private final List resultSet; + + EETReproducer(String originalQueryString, String transformedQueryString, List resultSet) { + this.originalQueryString = originalQueryString; + this.transformedQueryString = transformedQueryString; + this.resultSet = resultSet; + } + + @Override + public boolean bugStillTriggers(G globalState) { + try { + List transformedResultSet = ComparatorHelper + .getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, transformedResultSet, originalQueryString, + List.of(transformedQueryString), globalState); + } catch (AssertionError triggeredError) { + return true; + } catch (SQLException ignored) { + } + return false; + } + } + + public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + reproducer = null; + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + Z select = gen.generateSelect(); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + List fetchColumns = gen.generateFetchColumns(true); + select.setFetchColumns(fetchColumns); + E whereClause = gen.generateBooleanExpression(); + select.setWhereClause(whereClause); + + String originalQueryString = select.asString(); + generatedQueryString = originalQueryString; + List originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, + // while the WHERE clause is evaluated in a boolean context. + List transformedFetchColumns = fetchColumns.stream().map(c -> gen.transformExpression(c, false)) + .collect(Collectors.toList()); + select.setFetchColumns(transformedFetchColumns); + select.setWhereClause(gen.transformExpression(whereClause, true)); + + String transformedQueryString = select.asString(); + List transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, + errors, state); + + ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, + List.of(transformedQueryString), state); + + reproducer = new EETReproducer(originalQueryString, transformedQueryString, originalResultSet); + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/common/oracle/EETTransformation.java b/src/sqlancer/common/oracle/EETTransformation.java new file mode 100644 index 000000000..ab9b7fefe --- /dev/null +++ b/src/sqlancer/common/oracle/EETTransformation.java @@ -0,0 +1,83 @@ +package sqlancer.common.oracle; + +import sqlancer.Randomly; + +/** + * Implements the semantic-preserving expression transformation rules of EET (Equivalent Expression Transformation, Jiang + * & Su, OSDI'24), Table 2. Given an expression, {@link #applyRandomRule} returns a semantically equivalent + * expression built from the primitives provided by an {@link EETNodeFactory}. The rules are DBMS-independent; only the + * node construction (via the factory) is DBMS-specific. + * + *

+ * The rules rely on two always-determined boolean expressions built from an arbitrary boolean {@code p}: + *

    + *
  • {@code true_expr(p) = p OR (NOT p) OR (p IS NULL)}, which always evaluates to TRUE, and
    • + *
  • {@code false_expr(p) = p AND (NOT p) AND (p IS NOT NULL)}, which always evaluates to FALSE.
    • + *
+ * + * @param + * the DBMS-specific expression type + */ +public class EETTransformation { + + private final EETNodeFactory factory; + + public EETTransformation(EETNodeFactory factory) { + this.factory = factory; + } + + // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE + private E trueExpr() { + E p = factory.generateBooleanExpression(); + return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); + } + + // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE + private E falseExpr() { + E p = factory.generateBooleanExpression(); + return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); + } + + /** + * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen, applicable + * transformation rule. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (e.g. a WHERE predicate or an operand of a + * logical operator). Only in a boolean context may the determined-boolean rules (No. 1 and 2), which + * reduce the expression to a boolean value, be applied; in a scalar context they would change the + * expression's value/type and are therefore excluded. + * + * @return a semantically equivalent expression + */ + public E applyRandomRule(E expr, boolean booleanContext) { + int rule; + if (booleanContext) { + // Rules No. 1-6 are all value-preserving in a boolean context. + rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); + } else { + if (!factory.isCaseWhenApplicable(expr)) { + return expr; // rule No. 7: transform the expression to itself + } + // In a scalar context only the CASE WHEN rules preserve the exact value and type. + rule = Randomly.fromOptions(3, 4, 5, 6); + } + switch (rule) { + case 1: // bool_expr => false_expr OR bool_expr + return factory.or(falseExpr(), expr); + case 2: // bool_expr => true_expr AND bool_expr + return factory.and(trueExpr(), expr); + case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END + return factory.caseWhen(falseExpr(), expr, expr); + case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END + return factory.caseWhen(trueExpr(), expr, expr); + case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END + case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END + return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); + default: + throw new AssertionError(rule); + } + } +} diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index 83e08677a..ed5ddc489 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -5,6 +5,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.EETOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; @@ -82,5 +83,14 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLDQEOracle(globalState); } + }, + EET { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + return new EETOracle<>(globalState, gen, expectedErrors); + } }; } diff --git a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java index c87048a4b..45a1fe888 100644 --- a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java +++ b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java @@ -62,6 +62,10 @@ public MySQLUnaryPrefixOperation(MySQLExpression expr, MySQLUnaryPrefixOperator super(expr, op); } + public MySQLUnaryPrefixOperator getOp() { + return op; + } + @Override public MySQLConstant getExpectedValue() { MySQLConstant subExprVal = expr.getExpectedValue(); diff --git a/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java b/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java new file mode 100644 index 000000000..33f3ab587 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java @@ -0,0 +1,70 @@ +package sqlancer.mysql.gen; + +import java.util.List; + +import sqlancer.common.oracle.EETNodeFactory; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; + +/** + * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformation EET transformation} + * rules. + */ +public class MySQLEETNodeFactory implements EETNodeFactory { + + private final MySQLExpressionGenerator gen; + + public MySQLEETNodeFactory(MySQLExpressionGenerator gen) { + this.gen = gen; + } + + @Override + public MySQLExpression and(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); + } + + @Override + public MySQLExpression or(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); + } + + @Override + public MySQLExpression not(MySQLExpression expr) { + return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); + } + + @Override + public MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); + } + + @Override + public MySQLExpression isNotNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); + } + + @Override + public MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { + return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); + } + + @Override + public MySQLExpression generateBooleanExpression() { + return gen.generateBooleanExpression(); + } + + @Override + public boolean isCaseWhenApplicable(MySQLExpression expr) { + // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET + // paper); aggregates are excluded to avoid placing them in invalid contexts. + return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); + } +} diff --git a/src/sqlancer/mysql/gen/MySQLEETTransformer.java b/src/sqlancer/mysql/gen/MySQLEETTransformer.java new file mode 100644 index 000000000..000e29bb6 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLEETTransformer.java @@ -0,0 +1,124 @@ +package sqlancer.mysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.oracle.EETTransformation; +import sqlancer.mysql.ast.MySQLBetweenOperation; +import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLCastOperation; +import sqlancer.mysql.ast.MySQLComputableFunction; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; + +/** + * Recursively applies the {@link EETTransformation EET} transformation rules throughout a MySQL expression's AST. At + * each node the transformer first recurses into (and rebuilds the node from) its transformed children, then, with some + * probability, wraps the resulting sub-expression with a randomly chosen transformation rule. + * + *

+ * A boolean/scalar context flag is threaded through the recursion so that the determined-boolean rules (which reduce an + * expression to a boolean value) are only ever applied where the expression is used purely for its truth value. + */ +public class MySQLEETTransformer { + + private static final boolean BOOLEAN = true; + private static final boolean SCALAR = false; + + private final EETTransformation transformation; + + public MySQLEETTransformer(MySQLExpressionGenerator gen) { + this.transformation = new EETTransformation<>(new MySQLEETNodeFactory(gen)); + } + + /** + * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the + * root, guaranteeing that the returned expression differs from the input. + */ + public MySQLExpression transform(MySQLExpression expr, boolean booleanContext) { + return transformNode(expr, booleanContext, true); + } + + private MySQLExpression transformNode(MySQLExpression expr, boolean booleanContext, boolean forceApply) { + MySQLExpression descended = descend(expr, booleanContext); + if (forceApply || Randomly.getBoolean()) { + return transformation.applyRandomRule(descended, booleanContext); + } + return descended; + } + + /** + * Rebuilds {@code expr} with its children transformed. Leaf nodes (columns, constants, ...) and node types that are + * not rebuilt here are returned unchanged; any applicable transformation is still applied to them by the calling + * {@link #transformNode}. + */ + private MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { + if (expr instanceof MySQLBinaryLogicalOperation) { + // AND/OR/XOR: both operands are evaluated in a boolean context. + MySQLBinaryLogicalOperation op = (MySQLBinaryLogicalOperation) expr; + return new MySQLBinaryLogicalOperation(transformNode(op.getLeft(), BOOLEAN, false), + transformNode(op.getRight(), BOOLEAN, false), op.getOp()); + } else if (expr instanceof MySQLBinaryComparisonOperation) { + MySQLBinaryComparisonOperation op = (MySQLBinaryComparisonOperation) expr; + return new MySQLBinaryComparisonOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLBinaryOperation) { + MySQLBinaryOperation op = (MySQLBinaryOperation) expr; + return new MySQLBinaryOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + MySQLUnaryPrefixOperation op = (MySQLUnaryPrefixOperation) expr; + boolean childContext = op.getOp() == MySQLUnaryPrefixOperator.NOT ? BOOLEAN : SCALAR; + return new MySQLUnaryPrefixOperation(transformNode(op.getExpression(), childContext, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPostfixOperation) { + // The operand is transformed value-preservingly (scalar), which is safe for IS NULL/IS TRUE/IS FALSE. + MySQLUnaryPostfixOperation op = (MySQLUnaryPostfixOperation) expr; + return new MySQLUnaryPostfixOperation(transformNode(op.getExpression(), SCALAR, false), op.getOperator(), + op.isNegated()); + } else if (expr instanceof MySQLCastOperation) { + MySQLCastOperation op = (MySQLCastOperation) expr; + return new MySQLCastOperation(transformNode(op.getExpr(), SCALAR, false), op.getType()); + } else if (expr instanceof MySQLBetweenOperation) { + MySQLBetweenOperation op = (MySQLBetweenOperation) expr; + return new MySQLBetweenOperation(transformNode(op.getExpr(), SCALAR, false), + transformNode(op.getLeft(), SCALAR, false), transformNode(op.getRight(), SCALAR, false)); + } else if (expr instanceof MySQLInOperation) { + MySQLInOperation op = (MySQLInOperation) expr; + List listElements = op.getListElements().stream() + .map(e -> transformNode(e, SCALAR, false)).collect(Collectors.toList()); + return new MySQLInOperation(transformNode(op.getExpr(), SCALAR, false), listElements, op.isTrue()); + } else if (expr instanceof MySQLComputableFunction) { + MySQLComputableFunction op = (MySQLComputableFunction) expr; + MySQLExpression[] args = op.getArguments(); + MySQLExpression[] newArgs = new MySQLExpression[args.length]; + for (int i = 0; i < args.length; i++) { + newArgs[i] = transformNode(args[i], SCALAR, false); + } + return new MySQLComputableFunction(op.getFunction(), newArgs); + } else if (expr instanceof MySQLCaseOperator) { + return descendCase((MySQLCaseOperator) expr); + } + return expr; + } + + private MySQLExpression descendCase(MySQLCaseOperator caseOp) { + MySQLExpression switchCondition = caseOp.getSwitchCondition(); + // Without a switch operand the WHEN conditions are boolean; with one they are compared against the operand. + boolean conditionContext = switchCondition == null ? BOOLEAN : SCALAR; + MySQLExpression newSwitch = switchCondition == null ? null : transformNode(switchCondition, SCALAR, false); + List conditions = caseOp.getConditions().stream() + .map(e -> transformNode(e, conditionContext, false)).collect(Collectors.toList()); + List expressions = caseOp.getExpressions().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); + MySQLExpression elseExpr = caseOp.getElseExpr() == null ? null + : transformNode(caseOp.getElseExpr(), SCALAR, false); + return new MySQLCaseOperator(newSwitch, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index d8ce5dd37..29b8d5ac5 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; @@ -48,11 +49,13 @@ public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, - CERTGenerator { + CERTGenerator, + EETGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; private List tables; + private MySQLEETTransformer eetTransformer; public MySQLExpressionGenerator(MySQLGlobalState state) { this.state = state; @@ -236,6 +239,14 @@ public MySQLSelect generateSelect() { return new MySQLSelect(); } + @Override + public MySQLExpression transformExpression(MySQLExpression expr, boolean booleanContext) { + if (eetTransformer == null) { + eetTransformer = new MySQLEETTransformer(this); + } + return eetTransformer.transform(expr, booleanContext); + } + @Override public List getRandomJoinClauses() { return List.of(); diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 054a66cb6..c17ccf0d5 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -362,7 +362,7 @@ private void appendType(MySQLDataType randomType) { } if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS - || o == MySQLOracleFactory.DQP)) { + || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET)) { sb.append(" ZEROFILL"); } } From 7c9f80f81440213b7a00997e8c0605533fc06b90 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 10 Jul 2026 15:31:17 +0800 Subject: [PATCH 1253/1284] Clarify that the E in the EET generics must extend the Expression class --- src/sqlancer/common/oracle/EETNodeFactory.java | 4 +++- src/sqlancer/common/oracle/EETTransformation.java | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java index 2cc933ced..c446a6054 100644 --- a/src/sqlancer/common/oracle/EETNodeFactory.java +++ b/src/sqlancer/common/oracle/EETNodeFactory.java @@ -1,5 +1,7 @@ package sqlancer.common.oracle; +import sqlancer.common.ast.newast.Expression; + /** * Factory for constructing the AST nodes needed by the {@link EETTransformation equivalent expression transformation} * rules. Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface @@ -8,7 +10,7 @@ * @param * the DBMS-specific expression type */ -public interface EETNodeFactory { +public interface EETNodeFactory> { /** Builds {@code left AND right}. */ E and(E left, E right); diff --git a/src/sqlancer/common/oracle/EETTransformation.java b/src/sqlancer/common/oracle/EETTransformation.java index ab9b7fefe..ecf2eea74 100644 --- a/src/sqlancer/common/oracle/EETTransformation.java +++ b/src/sqlancer/common/oracle/EETTransformation.java @@ -1,5 +1,6 @@ package sqlancer.common.oracle; +import sqlancer.common.ast.newast.Expression; import sqlancer.Randomly; /** @@ -18,7 +19,7 @@ * @param * the DBMS-specific expression type */ -public class EETTransformation { +public class EETTransformation> { private final EETNodeFactory factory; @@ -39,7 +40,7 @@ private E falseExpr() { } /** - * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen, applicable + * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen applicable * transformation rule. * * @param expr From 95dcc8ef94c5793595ad1a2e5481fbd7411835c2 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 11 Jul 2026 20:56:54 +0800 Subject: [PATCH 1254/1284] Refactor EET oracle structure for consistency and extensibility Merge `EETTransformation` into a new abstract base class `EETTransformer` (template method pattern: rules + tree-walking orchestration in the base, `descend` abstract for DBMS-specific AST reconstruction). Move `MySQLEETNodeFactory` and `MySQLEETTransformer` from `mysql/gen` to `mysql/oracle` to mirror the placement of their common counterparts. `MySQLEETTransformer` now extends `EETTransformer`. --- .../common/oracle/EETNodeFactory.java | 6 +- .../common/oracle/EETTransformation.java | 84 ------------- .../common/oracle/EETTransformer.java | 112 ++++++++++++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 1 + .../{gen => oracle}/MySQLEETNodeFactory.java | 6 +- .../{gen => oracle}/MySQLEETTransformer.java | 45 ++----- 6 files changed, 128 insertions(+), 126 deletions(-) delete mode 100644 src/sqlancer/common/oracle/EETTransformation.java create mode 100644 src/sqlancer/common/oracle/EETTransformer.java rename src/sqlancer/mysql/{gen => oracle}/MySQLEETNodeFactory.java (95%) rename src/sqlancer/mysql/{gen => oracle}/MySQLEETTransformer.java (72%) diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java index c446a6054..9f97ea937 100644 --- a/src/sqlancer/common/oracle/EETNodeFactory.java +++ b/src/sqlancer/common/oracle/EETNodeFactory.java @@ -3,9 +3,9 @@ import sqlancer.common.ast.newast.Expression; /** - * Factory for constructing the AST nodes needed by the {@link EETTransformation equivalent expression transformation} - * rules. Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface - * lets the (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. + * Factory for constructing the AST nodes needed by the {@link EETTransformer EET transformer's} transformation rules. + * Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface lets the + * (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. * * @param * the DBMS-specific expression type diff --git a/src/sqlancer/common/oracle/EETTransformation.java b/src/sqlancer/common/oracle/EETTransformation.java deleted file mode 100644 index ecf2eea74..000000000 --- a/src/sqlancer/common/oracle/EETTransformation.java +++ /dev/null @@ -1,84 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.common.ast.newast.Expression; -import sqlancer.Randomly; - -/** - * Implements the semantic-preserving expression transformation rules of EET (Equivalent Expression Transformation, Jiang - * & Su, OSDI'24), Table 2. Given an expression, {@link #applyRandomRule} returns a semantically equivalent - * expression built from the primitives provided by an {@link EETNodeFactory}. The rules are DBMS-independent; only the - * node construction (via the factory) is DBMS-specific. - * - *

- * The rules rely on two always-determined boolean expressions built from an arbitrary boolean {@code p}: - *

    - *
  • {@code true_expr(p) = p OR (NOT p) OR (p IS NULL)}, which always evaluates to TRUE, and
    • - *
  • {@code false_expr(p) = p AND (NOT p) AND (p IS NOT NULL)}, which always evaluates to FALSE.
    • - *
- * - * @param - * the DBMS-specific expression type - */ -public class EETTransformation> { - - private final EETNodeFactory factory; - - public EETTransformation(EETNodeFactory factory) { - this.factory = factory; - } - - // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE - private E trueExpr() { - E p = factory.generateBooleanExpression(); - return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); - } - - // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE - private E falseExpr() { - E p = factory.generateBooleanExpression(); - return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); - } - - /** - * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen applicable - * transformation rule. - * - * @param expr - * the expression to transform - * @param booleanContext - * whether {@code expr} is evaluated purely for its truth value (e.g. a WHERE predicate or an operand of a - * logical operator). Only in a boolean context may the determined-boolean rules (No. 1 and 2), which - * reduce the expression to a boolean value, be applied; in a scalar context they would change the - * expression's value/type and are therefore excluded. - * - * @return a semantically equivalent expression - */ - public E applyRandomRule(E expr, boolean booleanContext) { - int rule; - if (booleanContext) { - // Rules No. 1-6 are all value-preserving in a boolean context. - rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); - } else { - if (!factory.isCaseWhenApplicable(expr)) { - return expr; // rule No. 7: transform the expression to itself - } - // In a scalar context only the CASE WHEN rules preserve the exact value and type. - rule = Randomly.fromOptions(3, 4, 5, 6); - } - switch (rule) { - case 1: // bool_expr => false_expr OR bool_expr - return factory.or(falseExpr(), expr); - case 2: // bool_expr => true_expr AND bool_expr - return factory.and(trueExpr(), expr); - case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END - return factory.caseWhen(falseExpr(), expr, expr); - case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END - return factory.caseWhen(trueExpr(), expr, expr); - case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END - case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END - return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); - default: - throw new AssertionError(rule); - } - } -} diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java new file mode 100644 index 000000000..d66935e18 --- /dev/null +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -0,0 +1,112 @@ +package sqlancer.common.oracle; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; + +/** + * Abstract base class for EET (Equivalent Expression Transformation) tree-walkers, based on "Detecting Logic Bugs in + * Database Engines via Equivalent Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework + * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST + * nodes from their transformed children; everything else (the rule logic, context threading, and tree-walking + * orchestration) is provided here. + * + * @param + * the DBMS-specific expression type + */ +public abstract class EETTransformer> { + + private final EETNodeFactory factory; + + protected EETTransformer(EETNodeFactory factory) { + this.factory = factory; + } + + // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE + private E trueExpr() { + E p = factory.generateBooleanExpression(); + return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); + } + + // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE + private E falseExpr() { + E p = factory.generateBooleanExpression(); + return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); + } + + /** + * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent + * expression. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value; only in a boolean context may the + * determined-boolean rules (No. 1 and 2), which reduce the expression to a boolean value, be applied + * + * @return a semantically equivalent expression + */ + protected E applyRandomRule(E expr, boolean booleanContext) { + int rule; + if (booleanContext) { + // Rules No. 1-6 are all value-preserving in a boolean context. + rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); + } else { + if (!factory.isCaseWhenApplicable(expr)) { + return expr; // rule No. 7: transform the expression to itself + } + // In a scalar context only the CASE WHEN rules preserve the exact value and type. + rule = Randomly.fromOptions(3, 4, 5, 6); + } + switch (rule) { + case 1: // bool_expr => false_expr OR bool_expr + return factory.or(falseExpr(), expr); + case 2: // bool_expr => true_expr AND bool_expr + return factory.and(trueExpr(), expr); + case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END + return factory.caseWhen(falseExpr(), expr, expr); + case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END + return factory.caseWhen(trueExpr(), expr, expr); + case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END + case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END + return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); + default: + throw new AssertionError(rule); + } + } + + /** + * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the + * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. + */ + public E transform(E expr, boolean booleanContext) { + return transformNode(expr, booleanContext, true); + } + + /** + * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node. + */ + protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { + E descended = descend(expr, booleanContext); + if (forceApply || Randomly.getBoolean()) { + return applyRandomRule(descended, booleanContext); + } + return descended; + } + + /** + * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each child. + * Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable transformation + * will still be applied to them by the calling {@link #transformNode}. + * + * @param expr + * the expression to descend into + * @param booleanContext + * the context in which {@code expr} itself is evaluated (used to determine child contexts) + * + * @return a rebuilt copy of {@code expr} with transformed children, or {@code expr} itself if it is a leaf + */ + protected abstract E descend(E expr, boolean booleanContext); +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 29b8d5ac5..904013a37 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -46,6 +46,7 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, diff --git a/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java b/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java similarity index 95% rename from src/sqlancer/mysql/gen/MySQLEETNodeFactory.java rename to src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java index 33f3ab587..c73b576c5 100644 --- a/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java +++ b/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java @@ -1,4 +1,4 @@ -package sqlancer.mysql.gen; +package sqlancer.mysql.oracle; import java.util.List; @@ -13,10 +13,10 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.gen.MySQLExpressionGenerator; /** - * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformation EET transformation} - * rules. + * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformer EET transformer}. */ public class MySQLEETNodeFactory implements EETNodeFactory { diff --git a/src/sqlancer/mysql/gen/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java similarity index 72% rename from src/sqlancer/mysql/gen/MySQLEETTransformer.java rename to src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 000e29bb6..71af94ed8 100644 --- a/src/sqlancer/mysql/gen/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -1,10 +1,9 @@ -package sqlancer.mysql.gen; +package sqlancer.mysql.oracle; import java.util.List; import java.util.stream.Collectors; -import sqlancer.Randomly; -import sqlancer.common.oracle.EETTransformation; +import sqlancer.common.oracle.EETTransformer; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -17,49 +16,23 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.gen.MySQLExpressionGenerator; /** - * Recursively applies the {@link EETTransformation EET} transformation rules throughout a MySQL expression's AST. At - * each node the transformer first recurses into (and rebuilds the node from) its transformed children, then, with some - * probability, wraps the resulting sub-expression with a randomly chosen transformation rule. - * - *

- * A boolean/scalar context flag is threaded through the recursion so that the determined-boolean rules (which reduce an - * expression to a boolean value) are only ever applied where the expression is used purely for its truth value. + * MySQL implementation of the {@link EETTransformer EET} tree-walker. Implements {@link #descend} to rebuild MySQL AST + * nodes from their transformed children, threading the correct boolean/scalar context into each child. */ -public class MySQLEETTransformer { +public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; - private final EETTransformation transformation; - public MySQLEETTransformer(MySQLExpressionGenerator gen) { - this.transformation = new EETTransformation<>(new MySQLEETNodeFactory(gen)); - } - - /** - * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the - * root, guaranteeing that the returned expression differs from the input. - */ - public MySQLExpression transform(MySQLExpression expr, boolean booleanContext) { - return transformNode(expr, booleanContext, true); - } - - private MySQLExpression transformNode(MySQLExpression expr, boolean booleanContext, boolean forceApply) { - MySQLExpression descended = descend(expr, booleanContext); - if (forceApply || Randomly.getBoolean()) { - return transformation.applyRandomRule(descended, booleanContext); - } - return descended; + super(new MySQLEETNodeFactory(gen)); } - /** - * Rebuilds {@code expr} with its children transformed. Leaf nodes (columns, constants, ...) and node types that are - * not rebuilt here are returned unchanged; any applicable transformation is still applied to them by the calling - * {@link #transformNode}. - */ - private MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { + @Override + protected MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { if (expr instanceof MySQLBinaryLogicalOperation) { // AND/OR/XOR: both operands are evaluated in a boolean context. MySQLBinaryLogicalOperation op = (MySQLBinaryLogicalOperation) expr; From 657ec00a07e21a523924b7433283f780bbb07aee Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 09:49:12 +0800 Subject: [PATCH 1255/1284] Move EET transformer ownership to EETOracle, removing EET-specific state from DBMS-specific expression generators --- src/sqlancer/common/gen/EETGenerator.java | 20 +++++++------------ src/sqlancer/common/oracle/EETOracle.java | 6 ++++-- .../mysql/gen/MySQLExpressionGenerator.java | 9 +++------ 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index f468ff6d2..60b91e78a 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -5,14 +5,15 @@ import sqlancer.common.ast.newast.Expression; import sqlancer.common.ast.newast.Join; import sqlancer.common.ast.newast.Select; +import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; /** * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like - * the other oracle generators), an EET generator can transform an expression into a semantically equivalent one - * according to the EET transformation rules. + * the other oracle generators), an EET generator creates a DBMS-specific {@link EETTransformer} that the oracle uses to + * rewrite expressions into semantically equivalent ones. */ public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { @@ -29,16 +30,9 @@ public interface EETGenerator, J extends Join createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 0f192f9ab..0ae229414 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -32,6 +32,7 @@ public class EETOracle, J extends Join, E private final G state; private EETGenerator gen; + private final EETTransformer transformer; private final ExpectedErrors errors; private Reproducer reproducer; @@ -69,6 +70,7 @@ public EETOracle(G state, EETGenerator gen, ExpectedErrors expect } this.state = state; this.gen = gen; + this.transformer = gen.createTransformer(); this.errors = expectedErrors; } @@ -94,10 +96,10 @@ public void check() throws SQLException { // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, // while the WHERE clause is evaluated in a boolean context. - List transformedFetchColumns = fetchColumns.stream().map(c -> gen.transformExpression(c, false)) + List transformedFetchColumns = fetchColumns.stream().map(c -> transformer.transform(c, false)) .collect(Collectors.toList()); select.setFetchColumns(transformedFetchColumns); - select.setWhereClause(gen.transformExpression(whereClause, true)); + select.setWhereClause(transformer.transform(whereClause, true)); String transformedQueryString = select.asString(); List transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 904013a37..f2d577aee 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -46,6 +46,7 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.common.oracle.EETTransformer; import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator @@ -56,7 +57,6 @@ public class MySQLExpressionGenerator extends UntypedExpressionGenerator tables; - private MySQLEETTransformer eetTransformer; public MySQLExpressionGenerator(MySQLGlobalState state) { this.state = state; @@ -241,11 +241,8 @@ public MySQLSelect generateSelect() { } @Override - public MySQLExpression transformExpression(MySQLExpression expr, boolean booleanContext) { - if (eetTransformer == null) { - eetTransformer = new MySQLEETTransformer(this); - } - return eetTransformer.transform(expr, booleanContext); + public EETTransformer createTransformer() { + return new MySQLEETTransformer(this); } @Override From 65957adf9b8fb4b2702c20475c9c5d2e5c972560 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 10:03:19 +0800 Subject: [PATCH 1256/1284] Rearrange MySQLExpressionGenerator code to group oracle-specific sections together --- .../mysql/gen/MySQLExpressionGenerator.java | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index f2d577aee..5d1ec06f4 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -222,6 +222,22 @@ public List generateOrderBys() { return newOrderBys; } + public MySQLAggregate generateAggregate() { + MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); + + if (func.isVariadic()) { + int nrExprs = Randomly.smallNumber() + 1; + List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) + .collect(Collectors.toList()); + + return new MySQLAggregate(exprs, func); + } else { + return new MySQLAggregate(List.of(generateExpression()), func); + } + } + + // --- Shared oracle infrastructure (TLPWhere / CERT / EET) --- + @Override public MySQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); @@ -240,11 +256,6 @@ public MySQLSelect generateSelect() { return new MySQLSelect(); } - @Override - public EETTransformer createTransformer() { - return new MySQLEETTransformer(this); - } - @Override public List getRandomJoinClauses() { return List.of(); @@ -260,6 +271,8 @@ public List generateFetchColumns(boolean shouldCreateDummy) { return columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); } + // --- CERT oracle --- + @Override public String generateExplainQuery(MySQLSelect select) { return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed @@ -267,20 +280,6 @@ public String generateExplainQuery(MySQLSelect select) { // now be specified } - public MySQLAggregate generateAggregate() { - MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); - - if (func.isVariadic()) { - int nrExprs = Randomly.smallNumber() + 1; - List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) - .collect(Collectors.toList()); - - return new MySQLAggregate(exprs, func); - } else { - return new MySQLAggregate(List.of(generateExpression()), func); - } - } - @Override public boolean mutate(MySQLSelect select) { List> mutators = new ArrayList<>(); @@ -364,4 +363,11 @@ boolean mutateOr(MySQLSelect select) { return true; } } + + // --- EET oracle --- + + @Override + public EETTransformer createTransformer() { + return new MySQLEETTransformer(this); + } } From 03c487476cae5d040de76abff895c1fe5c79ceae Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 10:24:34 +0800 Subject: [PATCH 1257/1284] Remove need for EETNodeFactory by merging it into EETTransformer --- .../common/oracle/EETNodeFactory.java | 41 ----------- .../common/oracle/EETTransformer.java | 57 ++++++++++----- .../mysql/oracle/MySQLEETNodeFactory.java | 70 ------------------- .../mysql/oracle/MySQLEETTransformer.java | 50 ++++++++++++- 4 files changed, 88 insertions(+), 130 deletions(-) delete mode 100644 src/sqlancer/common/oracle/EETNodeFactory.java delete mode 100644 src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java deleted file mode 100644 index 9f97ea937..000000000 --- a/src/sqlancer/common/oracle/EETNodeFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.common.ast.newast.Expression; - -/** - * Factory for constructing the AST nodes needed by the {@link EETTransformer EET transformer's} transformation rules. - * Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface lets the - * (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. - * - * @param - * the DBMS-specific expression type - */ -public interface EETNodeFactory> { - - /** Builds {@code left AND right}. */ - E and(E left, E right); - - /** Builds {@code left OR right}. */ - E or(E left, E right); - - /** Builds {@code NOT expr}. */ - E not(E expr); - - /** Builds {@code expr IS NULL}. */ - E isNull(E expr); - - /** Builds {@code expr IS NOT NULL}. */ - E isNotNull(E expr); - - /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ - E caseWhen(E condition, E thenExpr, E elseExpr); - - /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ - E generateBooleanExpression(); - - /** - * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not - * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). - */ - boolean isCaseWhenApplicable(E expr); -} diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index d66935e18..010cd0a56 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -10,30 +10,24 @@ *

* This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST - * nodes from their transformed children; everything else (the rule logic, context threading, and tree-walking - * orchestration) is provided here. + * nodes from their transformed children, and the abstract factory methods to construct new nodes; everything else (the + * rule logic, context threading, and tree-walking orchestration) is provided here. * * @param * the DBMS-specific expression type */ public abstract class EETTransformer> { - private final EETNodeFactory factory; - - protected EETTransformer(EETNodeFactory factory) { - this.factory = factory; - } - // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE private E trueExpr() { - E p = factory.generateBooleanExpression(); - return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); + E p = generateBooleanExpression(); + return or(or(p, not(p)), isNull(p)); } // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE private E falseExpr() { - E p = factory.generateBooleanExpression(); - return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); + E p = generateBooleanExpression(); + return and(and(p, not(p)), isNotNull(p)); } /** @@ -54,7 +48,7 @@ protected E applyRandomRule(E expr, boolean booleanContext) { // Rules No. 1-6 are all value-preserving in a boolean context. rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); } else { - if (!factory.isCaseWhenApplicable(expr)) { + if (!isCaseWhenApplicable(expr)) { return expr; // rule No. 7: transform the expression to itself } // In a scalar context only the CASE WHEN rules preserve the exact value and type. @@ -62,16 +56,16 @@ protected E applyRandomRule(E expr, boolean booleanContext) { } switch (rule) { case 1: // bool_expr => false_expr OR bool_expr - return factory.or(falseExpr(), expr); + return or(falseExpr(), expr); case 2: // bool_expr => true_expr AND bool_expr - return factory.and(trueExpr(), expr); + return and(trueExpr(), expr); case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END - return factory.caseWhen(falseExpr(), expr, expr); + return caseWhen(falseExpr(), expr, expr); case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END - return factory.caseWhen(trueExpr(), expr, expr); + return caseWhen(trueExpr(), expr, expr); case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END - return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); + return caseWhen(generateBooleanExpression(), expr, expr); default: throw new AssertionError(rule); } @@ -109,4 +103,31 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * @return a rebuilt copy of {@code expr} with transformed children, or {@code expr} itself if it is a leaf */ protected abstract E descend(E expr, boolean booleanContext); + + /** Builds {@code left AND right}. */ + protected abstract E and(E left, E right); + + /** Builds {@code left OR right}. */ + protected abstract E or(E left, E right); + + /** Builds {@code NOT expr}. */ + protected abstract E not(E expr); + + /** Builds {@code expr IS NULL}. */ + protected abstract E isNull(E expr); + + /** Builds {@code expr IS NOT NULL}. */ + protected abstract E isNotNull(E expr); + + /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ + protected abstract E caseWhen(E condition, E thenExpr, E elseExpr); + + /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ + protected abstract E generateBooleanExpression(); + + /** + * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not + * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + */ + protected abstract boolean isCaseWhenApplicable(E expr); } diff --git a/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java b/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java deleted file mode 100644 index c73b576c5..000000000 --- a/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java +++ /dev/null @@ -1,70 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.util.List; - -import sqlancer.common.oracle.EETNodeFactory; -import sqlancer.mysql.ast.MySQLAggregate; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; -import sqlancer.mysql.ast.MySQLCaseOperator; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLTableReference; -import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; -import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -/** - * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformer EET transformer}. - */ -public class MySQLEETNodeFactory implements EETNodeFactory { - - private final MySQLExpressionGenerator gen; - - public MySQLEETNodeFactory(MySQLExpressionGenerator gen) { - this.gen = gen; - } - - @Override - public MySQLExpression and(MySQLExpression left, MySQLExpression right) { - return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); - } - - @Override - public MySQLExpression or(MySQLExpression left, MySQLExpression right) { - return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); - } - - @Override - public MySQLExpression not(MySQLExpression expr) { - return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); - } - - @Override - public MySQLExpression isNull(MySQLExpression expr) { - return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); - } - - @Override - public MySQLExpression isNotNull(MySQLExpression expr) { - return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); - } - - @Override - public MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { - return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); - } - - @Override - public MySQLExpression generateBooleanExpression() { - return gen.generateBooleanExpression(); - } - - @Override - public boolean isCaseWhenApplicable(MySQLExpression expr) { - // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET - // paper); aggregates are excluded to avoid placing them in invalid contexts. - return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); - } -} diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 71af94ed8..9b0cac226 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -4,16 +4,20 @@ import java.util.stream.Collectors; import sqlancer.common.oracle.EETTransformer; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; import sqlancer.mysql.ast.MySQLBinaryOperation; import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLComputableFunction; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; import sqlancer.mysql.gen.MySQLExpressionGenerator; @@ -27,8 +31,10 @@ public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; + private final MySQLExpressionGenerator gen; + public MySQLEETTransformer(MySQLExpressionGenerator gen) { - super(new MySQLEETNodeFactory(gen)); + this.gen = gen; } @Override @@ -94,4 +100,46 @@ private MySQLExpression descendCase(MySQLCaseOperator caseOp) { : transformNode(caseOp.getElseExpr(), SCALAR, false); return new MySQLCaseOperator(newSwitch, conditions, expressions, elseExpr); } + + @Override + protected MySQLExpression and(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); + } + + @Override + protected MySQLExpression or(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); + } + + @Override + protected MySQLExpression not(MySQLExpression expr) { + return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); + } + + @Override + protected MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); + } + + @Override + protected MySQLExpression isNotNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); + } + + @Override + protected MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { + return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); + } + + @Override + protected MySQLExpression generateBooleanExpression() { + return gen.generateBooleanExpression(); + } + + @Override + protected boolean isCaseWhenApplicable(MySQLExpression expr) { + // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET + // paper); aggregates are excluded to avoid placing them in invalid contexts. + return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); + } } From 9285e9168dd3a3d06cc080b6cc75d46aea216529 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 18:17:44 +0800 Subject: [PATCH 1258/1284] Correct EET transformation rule comments --- src/sqlancer/common/oracle/EETTransformer.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index 010cd0a56..12999402c 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -55,17 +55,18 @@ protected E applyRandomRule(E expr, boolean booleanContext) { rule = Randomly.fromOptions(3, 4, 5, 6); } switch (rule) { - case 1: // bool_expr => false_expr OR bool_expr + case 1: // expr => false_expr OR expr return or(falseExpr(), expr); - case 2: // bool_expr => true_expr AND bool_expr + case 2: // expr => true_expr AND expr return and(trueExpr(), expr); - case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END + case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END return caseWhen(falseExpr(), expr, expr); - case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END + case 4: // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END return caseWhen(trueExpr(), expr, expr); - case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END - case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END - return caseWhen(generateBooleanExpression(), expr, expr); + case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END + case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END + return caseWhen(generateBooleanExpression(), expr, expr); + // deep copy of expr is not needed, as the AST nodes are immutable anyway default: throw new AssertionError(rule); } From 52ce86661581681b1eb754ebcdc89103c833fa76 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 13 Jul 2026 12:40:14 +0800 Subject: [PATCH 1259/1284] Implement rules 3 and 4 for EET --- src/sqlancer/common/gen/EETGenerator.java | 2 +- src/sqlancer/common/oracle/EETOracle.java | 2 +- .../common/oracle/EETTransformer.java | 51 ++++++- .../mysql/ast/MySQLCastOperation.java | 7 +- .../mysql/gen/MySQLExpressionGenerator.java | 2 +- .../mysql/oracle/MySQLEETTransformer.java | 128 +++++++++++++++++- 6 files changed, 179 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index 60b91e78a..85aaa7462 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -34,5 +34,5 @@ public interface EETGenerator, J extends Join createTransformer(); + EETTransformer createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 0ae229414..148532e9b 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -32,7 +32,7 @@ public class EETOracle, J extends Join, E private final G state; private EETGenerator gen; - private final EETTransformer transformer; + private final EETTransformer transformer; private final ExpectedErrors errors; private Reproducer reproducer; diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index 12999402c..f27a917b9 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -10,13 +10,16 @@ *

* This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST - * nodes from their transformed children, and the abstract factory methods to construct new nodes; everything else (the - * rule logic, context threading, and tree-walking orchestration) is provided here. + * nodes from their transformed children, the abstract factory methods to construct new nodes, and the type hooks + * ({@link #inferType} and {@link #generateExpressionOfType}) that realize the paper's {@code rand_expr(type(expr))}; + * everything else (the rule logic, context threading, and tree-walking orchestration) is provided here. * * @param - * the DBMS-specific expression type + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain used by {@link #inferType} and {@link #generateExpressionOfType} */ -public abstract class EETTransformer> { +public abstract class EETTransformer, T> { // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE private E trueExpr() { @@ -30,6 +33,22 @@ private E falseExpr() { return and(and(p, not(p)), isNotNull(p)); } + /** + * Implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches that of + * {@code expr}. Although the generated expression is never evaluated (it occupies the dead branch of rules No. 3 + * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could + * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to + * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form + * of rules No. 5 and 6). + */ + private E randExprOfSameType(E expr) { + T type = inferType(expr); + if (type == null) { + return expr; + } + return generateExpressionOfType(type); + } + /** * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent * expression. @@ -60,12 +79,12 @@ protected E applyRandomRule(E expr, boolean booleanContext) { case 2: // expr => true_expr AND expr return and(trueExpr(), expr); case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END - return caseWhen(falseExpr(), expr, expr); + return caseWhen(falseExpr(), randExprOfSameType(expr), expr); case 4: // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END - return caseWhen(trueExpr(), expr, expr); + return caseWhen(trueExpr(), expr, randExprOfSameType(expr)); case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END - return caseWhen(generateBooleanExpression(), expr, expr); + return caseWhen(generateBooleanExpression(), expr, expr); // deep copy of expr is not needed, as the AST nodes are immutable anyway default: throw new AssertionError(rule); @@ -126,6 +145,24 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ protected abstract E generateBooleanExpression(); + /** + * Infers the static type of {@code expr}, or returns {@code null} if it cannot be determined. The type domain + * {@code T} is DBMS-specific and may be coarse: it only needs to be precise enough that replacing an expression + * with another of the same {@code T} leaves the DBMS's CASE WHEN result-type resolution unaffected. Returning + * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead + * branch. Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is + * uncertain. + */ + protected abstract T inferType(E expr); + + /** + * Generates a fresh random expression of static type {@code type}, reusing the variables available to the query + * generator. DBMSs with a typed expression generator can delegate to it directly; DBMSs with an untyped generator + * can instead wrap an arbitrary random expression in a CAST to {@code type} (which requires every value of + * {@code T} to be a valid CAST target). + */ + protected abstract E generateExpressionOfType(T type); + /** * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index 8ae783fa0..8a457d0f1 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -6,11 +6,14 @@ public class MySQLCastOperation implements MySQLExpression { private final CastType type; public enum CastType { - SIGNED, UNSIGNED; + SIGNED, UNSIGNED, + // CHAR and DOUBLE are used only by the EET oracle's type-pinning casts and are never evaluated, so + // MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + CHAR, DOUBLE; public static CastType getRandom() { return SIGNED; - // return Randomly.fromOptions(CastType.values()); + // return Randomly.fromOptions(CastType.SIGNED, CastType.UNSIGNED); } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 5d1ec06f4..9b019a6d4 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -367,7 +367,7 @@ boolean mutateOr(MySQLSelect select) { // --- EET oracle --- @Override - public EETTransformer createTransformer() { + public EETTransformer createTransformer() { return new MySQLEETTransformer(this); } } diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 9b0cac226..d16a8e517 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -1,5 +1,6 @@ package sqlancer.mysql.oracle; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -12,7 +13,11 @@ import sqlancer.mysql.ast.MySQLBinaryOperation; import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; +import sqlancer.mysql.ast.MySQLCastOperation.CastType; +import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLComputableFunction; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; import sqlancer.mysql.ast.MySQLTableReference; @@ -25,8 +30,13 @@ /** * MySQL implementation of the {@link EETTransformer EET} tree-walker. Implements {@link #descend} to rebuild MySQL AST * nodes from their transformed children, threading the correct boolean/scalar context into each child. + * + *

+ * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target types + * ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning {@code null} + * when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping it in a CAST. */ -public class MySQLEETTransformer extends EETTransformer { +public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; @@ -136,6 +146,122 @@ protected MySQLExpression generateBooleanExpression() { return gen.generateBooleanExpression(); } + @Override + protected MySQLExpression generateExpressionOfType(CastType type) { + // The MySQL expression generator is untyped, so the type of an arbitrary random expression is pinned by + // wrapping it in a CAST to the requested type. + return new MySQLCastOperation(gen.generateExpression(), type); + } + + @Override + protected CastType inferType(MySQLExpression expr) { + if (expr instanceof MySQLBinaryLogicalOperation || expr instanceof MySQLBinaryComparisonOperation + || expr instanceof MySQLUnaryPostfixOperation || expr instanceof MySQLBetweenOperation + || expr instanceof MySQLInOperation || expr instanceof MySQLExists) { + // Predicates evaluate to the boolean values 0/1, which are signed BIGINT. + return CastType.SIGNED; + } else if (expr instanceof MySQLBinaryOperation) { + // The bit operators &, | and ^ return BIGINT UNSIGNED. + return CastType.UNSIGNED; + } else if (expr instanceof MySQLCastOperation) { + return ((MySQLCastOperation) expr).getType(); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + return inferUnaryPrefixType((MySQLUnaryPrefixOperation) expr); + } else if (expr instanceof MySQLConstant) { + return inferConstantType((MySQLConstant) expr); + } else if (expr instanceof MySQLColumnReference) { + return inferColumnType((MySQLColumnReference) expr); + } else if (expr instanceof MySQLComputableFunction) { + return inferFunctionType((MySQLComputableFunction) expr); + } else if (expr instanceof MySQLCaseOperator) { + return inferCaseType((MySQLCaseOperator) expr); + } + return null; + } + + private CastType inferUnaryPrefixType(MySQLUnaryPrefixOperation op) { + if (op.getOp() == MySQLUnaryPrefixOperator.NOT) { + return CastType.SIGNED; + } + CastType operandType = inferType(op.getExpression()); + if (op.getOp() == MySQLUnaryPrefixOperator.PLUS) { + return operandType; // unary + is the identity + } + // Unary -: strings are coerced to DOUBLE; negating UNSIGNED changes the type (and usually errors). + if (operandType == CastType.CHAR || operandType == CastType.DOUBLE) { + return CastType.DOUBLE; + } + return operandType == CastType.SIGNED ? CastType.SIGNED : null; + } + + private CastType inferConstantType(MySQLConstant constant) { + if (constant instanceof MySQLConstant.MySQLIntConstant) { + return constant.isSigned() ? CastType.SIGNED : CastType.UNSIGNED; + } else if (constant instanceof MySQLConstant.MySQLTextConstant) { + return CastType.CHAR; + } else if (constant instanceof MySQLConstant.MySQLDoubleConstant) { + return CastType.DOUBLE; + } + return null; // the NULL constant has no type of its own + } + + private CastType inferColumnType(MySQLColumnReference ref) { + switch (ref.getColumn().getType()) { + case INT: + return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns + case VARCHAR: + return CastType.CHAR; + case DOUBLE: + return CastType.DOUBLE; + case FLOAT: // FLOAT-to-DOUBLE widening in the CASE result changes the rendered value + case DECIMAL: // the CASE result would need the column's exact precision and scale + default: + return null; + } + } + + private CastType inferFunctionType(MySQLComputableFunction func) { + MySQLExpression[] args = func.getArguments(); + switch (func.getFunction()) { + case BIT_COUNT: + return CastType.SIGNED; + case IF: + // The result type aggregates the types of the two value arguments (the condition does not contribute). + return commonType(args[1], args[2]); + case COALESCE: + case IFNULL: + case LEAST: + case GREATEST: + return commonType(args); + default: + return null; + } + } + + private CastType inferCaseType(MySQLCaseOperator caseOp) { + List branches = new ArrayList<>(caseOp.getExpressions()); + if (caseOp.getElseExpr() != null) { + branches.add(caseOp.getElseExpr()); + } + return commonType(branches.toArray(new MySQLExpression[0])); + } + + /** + * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the + * same inferrable type (a conservative under-approximation of MySQL's aggregation rules). + */ + private CastType commonType(MySQLExpression... exprs) { + CastType common = null; + for (MySQLExpression expr : exprs) { + CastType type = inferType(expr); + if (type == null || common != null && type != common) { + return null; + } + common = type; + } + return common; + } + @Override protected boolean isCaseWhenApplicable(MySQLExpression expr) { // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET From 86e5b719a6b7fffc548e37ad15a6dd54657797c4 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 15 Jul 2026 09:50:26 +0800 Subject: [PATCH 1260/1284] Fix EET reproducer/reduction to port the same fixes to it that were earlier made to TLP WHERE --- src/sqlancer/common/oracle/EETOracle.java | 92 +++++++++++++++++++---- 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 148532e9b..5af0cd1b3 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -40,28 +40,73 @@ public class EETOracle, J extends Join, E private final class EETReproducer implements Reproducer { private final String originalQueryString; + // null if the original bug was a DBMS error on the original query alone private final String transformedQueryString; - private final List resultSet; + // null if the original bug is a result set mismatch; otherwise, the message of the + // unexpected DBMS error that the original or transformed query triggered + private final String expectedErrorMessage; - EETReproducer(String originalQueryString, String transformedQueryString, List resultSet) { + EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; - this.resultSet = resultSet; + this.expectedErrorMessage = expectedErrorMessage; } @Override public boolean bugStillTriggers(G globalState) { + List originalResultSet; + List transformedResultSet; try { - List transformedResultSet = ComparatorHelper - .getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, transformedResultSet, originalQueryString, + // Re-execute both queries against the current (reduced) database instead of comparing + // against a cached result set, which would be stale once statements have been removed. + originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + globalState); + if (transformedQueryString == null) { + // the original bug was a DBMS error on the original query alone, which no + // longer occurs + return false; + } + transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, + globalState); + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs + return false; + } + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), globalState); - } catch (AssertionError triggeredError) { + } catch (AssertionError resultSetMismatch) { return true; - } catch (SQLException ignored) { } return false; } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (transformedQueryString != null) { + sb.append("-- transformed: ").append(transformedQueryString).append(';') + .append(System.lineSeparator()); + } + return sb.toString(); + } } public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { @@ -91,8 +136,16 @@ public void check() throws SQLException { String originalQueryString = select.asString(); generatedQueryString = originalQueryString; - List originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - state); + List originalResultSet; + try { + originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + } catch (AssertionError unexpectedError) { + // an unexpected DBMS error on the original query alone is itself a bug worth reducing; + // transformedQueryString is null because no transformed query is involved + reproducer = new EETReproducer(originalQueryString, null, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, // while the WHERE clause is evaluated in a boolean context. @@ -102,13 +155,24 @@ public void check() throws SQLException { select.setWhereClause(transformer.transform(whereClause, true)); String transformedQueryString = select.asString(); - List transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, - errors, state); + List transformedResultSet; + try { + transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, + state); + } catch (AssertionError unexpectedError) { + // the semantics-preserving transformation made the query trigger a DBMS error that the + // original did not, which is a bug worth reducing + reproducer = new EETReproducer(originalQueryString, transformedQueryString, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + // Set the reproducer before the assertion: assumeResultSetsAreEqual throws when the bug is + // detected, so creating the reproducer afterwards would leave it null and prevent any reduction. + reproducer = new EETReproducer(originalQueryString, transformedQueryString, null); ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), state); - - reproducer = new EETReproducer(originalQueryString, transformedQueryString, originalResultSet); } @Override From 912197dba19e2f6321d9881cd815172c779dec73 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 23 Jul 2026 15:48:24 +0800 Subject: [PATCH 1261/1284] Add FLOAT and DECIMAL to MySQL EET target cast types --- src/sqlancer/common/gen/EETGenerator.java | 6 +++--- src/sqlancer/common/oracle/EETOracle.java | 9 ++++----- .../common/oracle/EETTransformer.java | 16 +++++++-------- .../mysql/ast/MySQLCastOperation.java | 6 +++--- .../mysql/oracle/MySQLEETTransformer.java | 20 +++++++++++++------ 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index 85aaa7462..ef436d975 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -30,9 +30,9 @@ public interface EETGenerator, J extends Join createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 5af0cd1b3..1422d5b6b 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -23,9 +23,9 @@ * *

* The oracle generates a random query and then transforms its expressions (the WHERE predicate and the fetch columns) - * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation preserves - * semantics, the original and the transformed query must return the same result set; any discrepancy indicates a logic - * bug in the DBMS. + * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation + * preserves semantics, the original and the transformed query must return the same result set; any discrepancy + * indicates a logic bug in the DBMS. */ public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> implements TestOracle { @@ -102,8 +102,7 @@ public String getBugInformation() { } sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); if (transformedQueryString != null) { - sb.append("-- transformed: ").append(transformedQueryString).append(';') - .append(System.lineSeparator()); + sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); } return sb.toString(); } diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index f27a917b9..932a6fb8e 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -38,8 +38,8 @@ private E falseExpr() { * {@code expr}. Although the generated expression is never evaluated (it occupies the dead branch of rules No. 3 * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to - * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form - * of rules No. 5 and 6). + * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form of + * rules No. 5 and 6). */ private E randExprOfSameType(E expr) { T type = inferType(expr); @@ -85,7 +85,7 @@ protected E applyRandomRule(E expr, boolean booleanContext) { case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END return caseWhen(generateBooleanExpression(), expr, expr); - // deep copy of expr is not needed, as the AST nodes are immutable anyway + // deep copy of expr is not needed, as the AST nodes are immutable anyway default: throw new AssertionError(rule); } @@ -111,9 +111,9 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { } /** - * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each child. - * Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable transformation - * will still be applied to them by the calling {@link #transformNode}. + * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each + * child. Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable + * transformation will still be applied to them by the calling {@link #transformNode}. * * @param expr * the expression to descend into @@ -149,8 +149,8 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * Infers the static type of {@code expr}, or returns {@code null} if it cannot be determined. The type domain * {@code T} is DBMS-specific and may be coarse: it only needs to be precise enough that replacing an expression * with another of the same {@code T} leaves the DBMS's CASE WHEN result-type resolution unaffected. Returning - * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead - * branch. Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is + * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead branch. + * Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is * uncertain. */ protected abstract T inferType(E expr); diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index 8a457d0f1..b71d0498c 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -7,9 +7,9 @@ public class MySQLCastOperation implements MySQLExpression { public enum CastType { SIGNED, UNSIGNED, - // CHAR and DOUBLE are used only by the EET oracle's type-pinning casts and are never evaluated, so - // MySQLConstant.castAs does not support them; they must not be returned by getRandom(). - CHAR, DOUBLE; + // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never + // evaluated, so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + CHAR, FLOAT, DOUBLE, DECIMAL; public static CastType getRandom() { return SIGNED; diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index d16a8e517..5972996b8 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -80,8 +80,8 @@ protected MySQLExpression descend(MySQLExpression expr, boolean booleanContext) transformNode(op.getLeft(), SCALAR, false), transformNode(op.getRight(), SCALAR, false)); } else if (expr instanceof MySQLInOperation) { MySQLInOperation op = (MySQLInOperation) expr; - List listElements = op.getListElements().stream() - .map(e -> transformNode(e, SCALAR, false)).collect(Collectors.toList()); + List listElements = op.getListElements().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); return new MySQLInOperation(transformNode(op.getExpr(), SCALAR, false), listElements, op.isTrue()); } else if (expr instanceof MySQLComputableFunction) { MySQLComputableFunction op = (MySQLComputableFunction) expr; @@ -211,10 +211,18 @@ private CastType inferColumnType(MySQLColumnReference ref) { return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns case VARCHAR: return CastType.CHAR; + case FLOAT: + // Assumes FLOAT columns are never created with (M, D); otherwise the CAST would need the exact + // precision/scale. + return CastType.FLOAT; case DOUBLE: + // Assumes DOUBLE columns are never created with (M, D); otherwise the CAST would need the exact + // precision/scale. return CastType.DOUBLE; - case FLOAT: // FLOAT-to-DOUBLE widening in the CASE result changes the rendered value - case DECIMAL: // the CASE result would need the column's exact precision and scale + case DECIMAL: + // Assumes DECIMAL columns are never created with (M, D); otherwise the CAST would need the exact + // precision/scale. + return CastType.DECIMAL; default: return null; } @@ -247,8 +255,8 @@ private CastType inferCaseType(MySQLCaseOperator caseOp) { } /** - * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the - * same inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same + * inferrable type (a conservative under-approximation of MySQL's aggregation rules). */ private CastType commonType(MySQLExpression... exprs) { CastType common = null; From 4d6a651d5da612c26b4c180796b174c1fddc2dfc Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 23 Jul 2026 17:38:12 +0800 Subject: [PATCH 1262/1284] Fix treatment of unary minus in MySQL EET implementation --- .../mysql/oracle/MySQLEETTransformer.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 5972996b8..eb107c94b 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -185,13 +185,18 @@ private CastType inferUnaryPrefixType(MySQLUnaryPrefixOperation op) { } CastType operandType = inferType(op.getExpression()); if (op.getOp() == MySQLUnaryPrefixOperator.PLUS) { - return operandType; // unary + is the identity + return operandType; } - // Unary -: strings are coerced to DOUBLE; negating UNSIGNED changes the type (and usually errors). - if (operandType == CastType.CHAR || operandType == CastType.DOUBLE) { - return CastType.DOUBLE; + if (op.getOp() == MySQLUnaryPrefixOperator.MINUS) { + if (operandType == CastType.UNSIGNED) { + return CastType.SIGNED; + } else if (operandType == CastType.FLOAT) { + return CastType.DOUBLE; + } else if (operandType != CastType.CHAR) { + return operandType; + } } - return operandType == CastType.SIGNED ? CastType.SIGNED : null; + return null; } private CastType inferConstantType(MySQLConstant constant) { From e3d9f61ec8e09310aed7e8b686d777116c36776a Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 26 Jul 2026 15:02:27 +0800 Subject: [PATCH 1263/1284] Restrict creation of (M, D) columns to prevent false positives in EET --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 9 +++++++-- .../mysql/oracle/MySQLEETTransformer.java | 16 +++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index c17ccf0d5..40e325041 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -368,8 +368,13 @@ private void appendType(MySQLDataType randomType) { } } - public static void optionallyAddPrecisionAndScale(StringBuilder sb) { - if (Randomly.getBoolean() && !MySQLBugs.bug99183) { + private void optionallyAddPrecisionAndScale(StringBuilder sb) { + // The EET oracle's type inference assumes FLOAT/DOUBLE/DECIMAL columns are created without (M, D) (see + // MySQLEETTransformer#inferColumnType), so precision/scale is omitted while EET is active. This restriction can + // be lifted once (M, D) is tracked through the codebase and reflected in the CAST target types. + boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.EET); + if (Randomly.getBoolean() && !MySQLBugs.bug99183 && !eetActive) { sb.append("("); // The maximum number of digits (M) for DECIMAL is 65 long m = Randomly.getNotCachedInteger(1, 65); diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index eb107c94b..27a62869a 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -32,9 +32,10 @@ * nodes from their transformed children, threading the correct boolean/scalar context into each child. * *

- * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target types - * ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning {@code null} - * when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping it in a CAST. + * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target + * types ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning + * {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping + * it in a CAST. */ public class MySQLEETTransformer extends EETTransformer { @@ -216,17 +217,14 @@ private CastType inferColumnType(MySQLColumnReference ref) { return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns case VARCHAR: return CastType.CHAR; + // FLOAT/DOUBLE/DECIMAL columns are created without (M, D) while EET is active, so the plain + // CAST target below matches the column's type. Reintroducing (M, D) for better coverage would + // require tracking it here and emitting the exact precision/scale in the CAST. case FLOAT: - // Assumes FLOAT columns are never created with (M, D); otherwise the CAST would need the exact - // precision/scale. return CastType.FLOAT; case DOUBLE: - // Assumes DOUBLE columns are never created with (M, D); otherwise the CAST would need the exact - // precision/scale. return CastType.DOUBLE; case DECIMAL: - // Assumes DECIMAL columns are never created with (M, D); otherwise the CAST would need the exact - // precision/scale. return CastType.DECIMAL; default: return null; From 0a09fba83235c0b129e10a11538e3835ee1a7a1c Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 26 Jul 2026 15:15:33 +0800 Subject: [PATCH 1264/1284] Fix checkstyle violations --- src/sqlancer/common/gen/EETGenerator.java | 13 +++ src/sqlancer/common/oracle/EETOracle.java | 15 +++ .../common/oracle/EETTransformer.java | 110 ++++++++++++++++-- .../mysql/gen/MySQLExpressionGenerator.java | 2 +- .../mysql/oracle/MySQLEETTransformer.java | 7 +- 5 files changed, 135 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index ef436d975..a87a635ec 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -14,6 +14,17 @@ * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like * the other oracle generators), an EET generator creates a DBMS-specific {@link EETTransformer} that the oracle uses to * rewrite expressions into semantically equivalent ones. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class */ public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { @@ -33,6 +44,8 @@ public interface EETGenerator, J extends Join createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 1422d5b6b..b5aedf2cb 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -26,6 +26,21 @@ * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation * preserves semantics, the original and the transformed query must return the same result set; any discrepancy * indicates a logic bug in the DBMS. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific schema class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + * @param + * the DBMS-specific global state class */ public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> implements TestOracle { diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index 932a6fb8e..c46e148e1 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -24,7 +24,7 @@ public abstract class EETTransformer, T> { // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE private E trueExpr() { E p = generateBooleanExpression(); - return or(or(p, not(p)), isNull(p)); + return orExpr(orExpr(p, not(p)), isNull(p)); } // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE @@ -40,6 +40,11 @@ private E falseExpr() { * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form of * rules No. 5 and 6). + * + * @param expr + * the expression whose static type the generated expression must match + * + * @return a random expression whose static type matches that of {@code expr} */ private E randExprOfSameType(E expr) { T type = inferType(expr); @@ -75,7 +80,7 @@ protected E applyRandomRule(E expr, boolean booleanContext) { } switch (rule) { case 1: // expr => false_expr OR expr - return or(falseExpr(), expr); + return orExpr(falseExpr(), expr); case 2: // expr => true_expr AND expr return and(trueExpr(), expr); case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END @@ -94,6 +99,13 @@ protected E applyRandomRule(E expr, boolean booleanContext) { /** * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * + * @return a semantically equivalent expression */ public E transform(E expr, boolean booleanContext) { return transformNode(expr, booleanContext, true); @@ -101,6 +113,15 @@ public E transform(E expr, boolean booleanContext) { /** * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * @param forceApply + * whether a rule must be applied at this node rather than only with some probability + * + * @return the transformed expression */ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { E descended = descend(expr, booleanContext); @@ -124,25 +145,79 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { */ protected abstract E descend(E expr, boolean booleanContext); - /** Builds {@code left AND right}. */ + /** + * Builds {@code left AND right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left AND right} expression + */ protected abstract E and(E left, E right); - /** Builds {@code left OR right}. */ - protected abstract E or(E left, E right); + /** + * Builds {@code left OR right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left OR right} expression + */ + protected abstract E orExpr(E left, E right); - /** Builds {@code NOT expr}. */ + /** + * Builds {@code NOT expr}. + * + * @param expr + * the operand + * + * @return the {@code NOT expr} expression + */ protected abstract E not(E expr); - /** Builds {@code expr IS NULL}. */ + /** + * Builds {@code expr IS NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NULL} expression + */ protected abstract E isNull(E expr); - /** Builds {@code expr IS NOT NULL}. */ + /** + * Builds {@code expr IS NOT NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NOT NULL} expression + */ protected abstract E isNotNull(E expr); - /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ + /** + * Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. + * + * @param condition + * the WHEN condition + * @param thenExpr + * the THEN branch + * @param elseExpr + * the ELSE branch + * + * @return the CASE WHEN expression + */ protected abstract E caseWhen(E condition, E thenExpr, E elseExpr); - /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ + /** + * Generates a fresh random boolean expression, reusing the variables available to the query generator. + * + * @return a fresh random boolean expression + */ protected abstract E generateBooleanExpression(); /** @@ -152,6 +227,11 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead branch. * Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is * uncertain. + * + * @param expr + * the expression whose static type is to be inferred + * + * @return the inferred static type of {@code expr}, or {@code null} if it cannot be determined */ protected abstract T inferType(E expr); @@ -160,12 +240,22 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * generator. DBMSs with a typed expression generator can delegate to it directly; DBMSs with an untyped generator * can instead wrap an arbitrary random expression in a CAST to {@code type} (which requires every value of * {@code T} to be a valid CAST target). + * + * @param type + * the static type the generated expression must have + * + * @return a fresh random expression of static type {@code type} */ protected abstract E generateExpressionOfType(T type); /** * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + * + * @param expr + * the expression to test + * + * @return {@code true} if {@code expr} may be wrapped in a CASE WHEN expression */ protected abstract boolean isCaseWhenApplicable(E expr); } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9b019a6d4..da304ac67 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -12,6 +12,7 @@ import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTables; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; @@ -46,7 +47,6 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; -import sqlancer.common.oracle.EETTransformer; import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 27a62869a..2e4394eec 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -118,7 +118,7 @@ protected MySQLExpression and(MySQLExpression left, MySQLExpression right) { } @Override - protected MySQLExpression or(MySQLExpression left, MySQLExpression right) { + protected MySQLExpression orExpr(MySQLExpression left, MySQLExpression right) { return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); } @@ -260,6 +260,11 @@ private CastType inferCaseType(MySQLCaseOperator caseOp) { /** * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same * inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * + * @param exprs + * the result-type-determining subexpressions + * + * @return the common inferred type of {@code exprs}, or {@code null} if they do not share one */ private CastType commonType(MySQLExpression... exprs) { CastType common = null; From c71d198fff6cb271e1f946addbe5c43597580636 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 13:09:02 +0800 Subject: [PATCH 1265/1284] Refactor EETTransformer to encode the transformation rules as an enum --- .../common/oracle/EETTransformer.java | 159 ++++++++++++++---- 1 file changed, 129 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index c46e148e1..b472b8aff 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -1,5 +1,8 @@ package sqlancer.common.oracle; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Expression; @@ -35,11 +38,10 @@ private E falseExpr() { /** * Implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches that of - * {@code expr}. Although the generated expression is never evaluated (it occupies the dead branch of rules No. 3 + * {@code expr}. Although the generated expression is never evaluated (it occupies the redundant branch of rules 3 * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to - * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form of - * rules No. 5 and 6). + * {@code expr} itself, which trivially has the correct type (degenerating to rules 5 and 6). * * @param expr * the expression whose static type the generated expression must match @@ -54,46 +56,143 @@ private E randExprOfSameType(E expr) { return generateExpressionOfType(type); } + /** + * The first six transformation rules of the EET paper (Table 2). Each rule knows how to apply itself + * ({@link #apply}) and in which contexts it preserves the expression's value ({@link #isApplicable}). Rule No. 7 + * (transform the expression to itself) is not modelled here: it is the fallback applied by {@link #applyRandomRule} + * when no other rule is applicable. + */ + private enum Rule { + // expr => false_expr OR expr + RULE_1 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.orExpr(t.falseExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => true_expr AND expr + RULE_2 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.and(t.trueExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END + RULE_3 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.falseExpr(), t.randExprOfSameType(expr), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END + RULE_4 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.trueExpr(), expr, t.randExprOfSameType(expr)); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END + RULE_5 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END + RULE_6 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }; + + /** + * Applies this rule to {@code expr}, producing a semantically equivalent expression. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain + * @param t + * the transformer providing the DBMS-specific node factories + * @param expr + * the expression to transform + * + * @return a semantically equivalent expression + */ + abstract , T> E apply(EETTransformer t, E expr); + + /** + * Whether this rule preserves {@code expr}'s value in the given context. + * + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (rules 1 and 2 are only applicable if + * this is the case) + * @param caseWhenApplicable + * whether {@code expr} may be wrapped in a CASE WHEN expression + * + * @return {@code true} if this rule preserves {@code expr}'s value in the given context + */ + abstract boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable); + } + /** * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent - * expression. + * expression. When no rule is applicable, {@code expr} is returned unchanged (rule No. 7 of the EET paper). * * @param expr * the expression to transform * @param booleanContext - * whether {@code expr} is evaluated purely for its truth value; only in a boolean context may the - * determined-boolean rules (No. 1 and 2), which reduce the expression to a boolean value, be applied + * whether {@code expr} is evaluated purely for its truth value * * @return a semantically equivalent expression */ protected E applyRandomRule(E expr, boolean booleanContext) { - int rule; - if (booleanContext) { - // Rules No. 1-6 are all value-preserving in a boolean context. - rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); - } else { - if (!isCaseWhenApplicable(expr)) { - return expr; // rule No. 7: transform the expression to itself + boolean caseWhenApplicable = isCaseWhenApplicable(expr); + List applicableRules = new ArrayList<>(); + for (Rule rule : Rule.values()) { + if (rule.isApplicable(booleanContext, caseWhenApplicable)) { + applicableRules.add(rule); } - // In a scalar context only the CASE WHEN rules preserve the exact value and type. - rule = Randomly.fromOptions(3, 4, 5, 6); } - switch (rule) { - case 1: // expr => false_expr OR expr - return orExpr(falseExpr(), expr); - case 2: // expr => true_expr AND expr - return and(trueExpr(), expr); - case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END - return caseWhen(falseExpr(), randExprOfSameType(expr), expr); - case 4: // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END - return caseWhen(trueExpr(), expr, randExprOfSameType(expr)); - case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END - case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END - return caseWhen(generateBooleanExpression(), expr, expr); - // deep copy of expr is not needed, as the AST nodes are immutable anyway - default: - throw new AssertionError(rule); + if (applicableRules.isEmpty()) { + return expr; // rule 7 fallback: transform expression to itself } + return Randomly.fromList(applicableRules).apply(this, expr); } /** From 29291c7a7c73eafd2621240bec3da4d8db4be6dd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 14:35:44 +0800 Subject: [PATCH 1266/1284] Factor out reproducer logic for reuse among multiple different test oracles --- .../oracle/AbstractComparisonReproducer.java | 142 ++++++++++++++++++ src/sqlancer/common/oracle/EETOracle.java | 72 ++++----- src/sqlancer/common/oracle/NoRECOracle.java | 62 ++++---- .../common/oracle/TLPWhereOracle.java | 85 ++++++----- 4 files changed, 239 insertions(+), 122 deletions(-) create mode 100644 src/sqlancer/common/oracle/AbstractComparisonReproducer.java diff --git a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java new file mode 100644 index 000000000..d20a9636a --- /dev/null +++ b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java @@ -0,0 +1,142 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; + +/** + * Shared skeleton for the {@link Reproducer}s of oracles that detect a bug by comparing two evaluations of a + * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}. All of these + * reduce the bug the same way: re-evaluate both sides against the reduced database and report whether they still + * disagree (or, when the original bug was an unexpected DBMS error, whether that same error still fires). + * + *

+ * This class owns that control flow (including distinguishing a still-reproducing error from an unrelated one + * introduced by the reduction) and the {@link #getBugInformation()} header. Subclasses supply the parts specific to + * their oracle: how each side is evaluated, how the two are compared, and how the failing queries are rendered in the + * reduced test case. + * + * @param + * the DBMS-specific global state class + * @param + * the type each side evaluates to (e.g. a result set as a list of strings, a row count, a post-image) + */ +public abstract class AbstractComparisonReproducer, R> implements Reproducer { + + /** + * The message of the unexpected DBMS error the original bug was, or {@code null} if the original bug was a + * comparison mismatch rather than an error. + */ + protected final String expectedErrorMessage; + + protected AbstractComparisonReproducer(String expectedErrorMessage) { + this.expectedErrorMessage = expectedErrorMessage; + } + + /** + * Whether the recorded bug has a transformed (second) side. It does not when the bug was a DBMS error triggered by + * the original query alone, in which case there is no second side to evaluate or compare. + * + * @return {@code true} if {@link #evaluateTransformed} should be called + */ + protected abstract boolean hasTransformedSide(); + + /** + * Evaluates the original side against the (reduced) database. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @return the original side's value + * + * @throws SQLException + * if a DBMS interaction fails + */ + protected abstract R evaluateOriginal(G globalState) throws SQLException; + + /** + * Evaluates the transformed side against the (reduced) database. Only called when {@link #hasTransformedSide()} is + * {@code true}. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @return the transformed side's value + * + * @throws SQLException + * if a DBMS interaction fails + */ + protected abstract R evaluateTransformed(G globalState) throws SQLException; + + /** + * Whether the two evaluated sides disagree in the way that constitutes the bug. + * + * @param original + * the original side's value + * @param transformed + * the transformed side's value + * @param globalState + * the state the sides were evaluated against + * + * @return {@code true} if the sides differ (i.e. the bug still triggers) + */ + protected abstract boolean sidesDiffer(R original, R transformed, G globalState); + + @Override + public final boolean bugStillTriggers(G globalState) { + R original; + R transformed; + try { + original = evaluateOriginal(globalState); + if (!hasTransformedSide()) { + // the original bug was a DBMS error on the original query alone, which no longer occurs + return false; + } + transformed = evaluateTransformed(globalState); + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs + return false; + } + return sidesDiffer(original, transformed, globalState); + } + + @Override + public final String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage != null) { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage).append(System.lineSeparator()); + } else { + sb.append(mismatchHeaderLine()).append(System.lineSeparator()); + } + appendQueryLines(sb); + return sb.toString(); + } + + /** + * The header line (without trailing line separator) describing the mismatch, used when the original bug was a + * comparison mismatch rather than an error. For example, "-- On the database set up by the statements above, the + * result sets of the following queries mismatch:". + * + * @return the mismatch header line + */ + protected abstract String mismatchHeaderLine(); + + /** + * Appends the failing queries (or statements) to {@code sb}, one commented line each, so the reduced test case is + * self-contained. Called for both the mismatch and the error case, after the header. + * + * @param sb + * the builder to append to + */ + protected abstract void appendQueryLines(StringBuilder sb); +} diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index b5aedf2cb..c3ad4bae9 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -53,50 +53,38 @@ public class EETOracle, J extends Join, E private Reproducer reproducer; private String generatedQueryString; - private final class EETReproducer implements Reproducer { + private final class EETReproducer extends AbstractComparisonReproducer> { private final String originalQueryString; // null if the original bug was a DBMS error on the original query alone private final String transformedQueryString; - // null if the original bug is a result set mismatch; otherwise, the message of the - // unexpected DBMS error that the original or transformed query triggered - private final String expectedErrorMessage; EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { + super(expectedErrorMessage); this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; - this.expectedErrorMessage = expectedErrorMessage; } @Override - public boolean bugStillTriggers(G globalState) { - List originalResultSet; - List transformedResultSet; - try { - // Re-execute both queries against the current (reduced) database instead of comparing - // against a cached result set, which would be stale once statements have been removed. - originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - globalState); - if (transformedQueryString == null) { - // the original bug was a DBMS error on the original query alone, which no - // longer occurs - return false; - } - transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, - globalState); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (SQLException | RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs - return false; - } + protected boolean hasTransformedSide() { + return transformedQueryString != null; + } + + @Override + protected List evaluateOriginal(G globalState) throws SQLException { + // Re-execute against the current (reduced) database instead of comparing against a cached result set, + // which would be stale once statements have been removed. + return ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + } + + @Override + protected List evaluateTransformed(G globalState) throws SQLException { + return ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); + } + + @Override + protected boolean sidesDiffer(List original, List transformed, G globalState) { try { - ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, + ComparatorHelper.assumeResultSetsAreEqual(original, transformed, originalQueryString, List.of(transformedQueryString), globalState); } catch (AssertionError resultSetMismatch) { return true; @@ -105,21 +93,17 @@ public boolean bugStillTriggers(G globalState) { } @Override - public String getBugInformation() { - StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage == null) { - sb.append("-- On the database set up by the statements above, the result sets of the following" - + " queries mismatch:").append(System.lineSeparator()); - } else { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage) - .append(System.lineSeparator()); - } + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); if (transformedQueryString != null) { sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); } - return sb.toString(); } } diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 35f0c26af..11e52b91d 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -30,63 +30,55 @@ public class NoRECOracle, J extends Join, private Reproducer reproducer; private String lastQueryString; - private static class NoRECReproducer> implements Reproducer { + private static class NoRECReproducer> + extends AbstractComparisonReproducer { private final Function optimizedQuery; private final Function unoptimizedQuery; private final String optimizedQueryString; private final String unoptimizedQueryString; - // null if the original bug is a count mismatch; otherwise, the message of the unexpected - // DBMS error that the original queries triggered - private final String expectedErrorMessage; NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, String optimizedQueryString, String unoptimizedQueryString, String expectedErrorMessage) { + super(expectedErrorMessage); this.optimizedQuery = optimizedQuery; this.unoptimizedQuery = unoptimizedQuery; this.optimizedQueryString = optimizedQueryString; this.unoptimizedQueryString = unoptimizedQueryString; - this.expectedErrorMessage = expectedErrorMessage; } @Override - public boolean bugStillTriggers(G globalState) { - int optimizedCount; - int unoptimizedCount; - try { - optimizedCount = optimizedQuery.apply(globalState); - unoptimizedCount = unoptimizedQuery.apply(globalState); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs - return false; - } + protected boolean hasTransformedSide() { + return true; + } + + @Override + protected Integer evaluateOriginal(G globalState) { + return optimizedQuery.apply(globalState); + } + + @Override + protected Integer evaluateTransformed(G globalState) { + return unoptimizedQuery.apply(globalState); + } + + @Override + protected boolean sidesDiffer(Integer optimizedCount, Integer unoptimizedCount, G globalState) { if (optimizedCount == -1 || unoptimizedCount == -1) { return false; } - return optimizedCount != unoptimizedCount; + return optimizedCount.intValue() != unoptimizedCount.intValue(); } @Override - public String getBugInformation() { - StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage == null) { - sb.append("-- On the database set up by the statements above, the row counts of the following" - + " queries mismatch:").append(System.lineSeparator()); - } else { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage) - .append(System.lineSeparator()); - } + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the row counts of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); - return sb.toString(); } } diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 8fc87ac3d..bf759ebfc 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -29,56 +29,59 @@ public class TLPWhereOracle, J extends Join reproducer; private String generatedQueryString; - private class TLPWhereReproducer implements Reproducer { + // A side's result set, together with the human-readable combined-query strings that + // getCombinedResultSet fills in for the transformed side (unused, and null, for the original side) + private static final class TLPResultSet { + final List resultSet; + final List combinedString; + + TLPResultSet(List resultSet, List combinedString) { + this.resultSet = resultSet; + this.combinedString = combinedString; + } + } + + private class TLPWhereReproducer extends AbstractComparisonReproducer { final String firstQueryString; final String secondQueryString; final String thirdQueryString; final String originalQueryString; final boolean orderBy; - // null if the original bug is a result set mismatch; otherwise, the message of the - // unexpected DBMS error that the original queries triggered - final String expectedErrorMessage; TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, String originalQueryString, boolean orderBy, String expectedErrorMessage) { + super(expectedErrorMessage); this.firstQueryString = firstQueryString; this.secondQueryString = secondQueryString; this.thirdQueryString = thirdQueryString; this.originalQueryString = originalQueryString; this.orderBy = orderBy; - this.expectedErrorMessage = expectedErrorMessage; } @Override - public boolean bugStillTriggers(G globalState) { - List firstResultSet; + protected boolean hasTransformedSide() { + return firstQueryString != null; + } + + @Override + protected TLPResultSet evaluateOriginal(G globalState) throws SQLException { + return new TLPResultSet( + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState), null); + } + + @Override + protected TLPResultSet evaluateTransformed(G globalState) throws SQLException { List combinedString = new ArrayList<>(); - List secondResultSet; - try { - firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - globalState); - if (firstQueryString == null) { - // the original bug was a DBMS error on the original query alone, which no - // longer occurs - return false; - } - secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, globalState, errors); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (SQLException | RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs - return false; - } + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, globalState, errors); + return new TLPResultSet(secondResultSet, combinedString); + } + + @Override + protected boolean sidesDiffer(TLPResultSet original, TLPResultSet transformed, G globalState) { try { - ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, - combinedString, globalState); + ComparatorHelper.assumeResultSetsAreEqual(original.resultSet, transformed.resultSet, + originalQueryString, transformed.combinedString, globalState); } catch (AssertionError resultSetMismatch) { return true; } @@ -86,16 +89,13 @@ public boolean bugStillTriggers(G globalState) { } @Override - public String getBugInformation() { - StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage == null) { - sb.append("-- On the database set up by the statements above, the result sets of the following" - + " queries mismatch:").append(System.lineSeparator()); - } else { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage) - .append(System.lineSeparator()); - } + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { sb.append("-- ").append(originalQueryString).append(';').append(System.lineSeparator()); if (firstQueryString != null) { if (orderBy) { @@ -107,7 +107,6 @@ public String getBugInformation() { .append(" UNION ALL ").append(thirdQueryString).append(';').append(System.lineSeparator()); } } - return sb.toString(); } } From 2a74be3f623f67398c96e31a4e4f75869f18e973 Mon Sep 17 00:00:00 2001 From: splf Date: Thu, 30 Jul 2026 16:08:07 +0500 Subject: [PATCH 1267/1284] Do not schedule disabled PostgreSQL tablespaces --- src/sqlancer/postgres/PostgresProvider.java | 2 +- .../postgres/TestPostgresProvider.java | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 test/sqlancer/postgres/TestPostgresProvider.java diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index ec7978216..814013df6 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -194,7 +194,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { nrPerformed = r.getInteger(0, 2); break; case CREATE_TABLESPACE: - nrPerformed = r.getInteger(0, 2); + nrPerformed = globalState.getDbmsSpecificOptions().isTestTablespaces() ? r.getInteger(0, 2) : 0; break; case UPDATE: nrPerformed = r.getInteger(0, 10); diff --git a/test/sqlancer/postgres/TestPostgresProvider.java b/test/sqlancer/postgres/TestPostgresProvider.java new file mode 100644 index 000000000..81e795568 --- /dev/null +++ b/test/sqlancer/postgres/TestPostgresProvider.java @@ -0,0 +1,21 @@ +package sqlancer.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class TestPostgresProvider { + + @Test + void createTablespaceIsNotScheduledWhenDisabled() { + PostgresGlobalState state = new PostgresGlobalState(); + state.setDbmsSpecificOptions(new PostgresOptions() { + @Override + public boolean isTestTablespaces() { + return false; + } + }); + + assertEquals(0, PostgresProvider.mapActions(state, PostgresProvider.Action.CREATE_TABLESPACE)); + } +} From e954c10c75c79ac11f4bbf52c2edd00a6d442fdd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 20:22:31 +0800 Subject: [PATCH 1268/1284] Factor out unexpected-error reproduction into dedicated UnexpectedErrorReproducer --- .../oracle/AbstractComparisonReproducer.java | 73 +++++------------ src/sqlancer/common/oracle/EETOracle.java | 44 +++++++---- src/sqlancer/common/oracle/NoRECOracle.java | 36 ++++++--- .../common/oracle/TLPWhereOracle.java | 60 +++++++++----- .../oracle/UnexpectedErrorReproducer.java | 79 +++++++++++++++++++ 5 files changed, 190 insertions(+), 102 deletions(-) create mode 100644 src/sqlancer/common/oracle/UnexpectedErrorReproducer.java diff --git a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java index d20a9636a..582a19c1d 100644 --- a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java +++ b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java @@ -7,43 +7,28 @@ /** * Shared skeleton for the {@link Reproducer}s of oracles that detect a bug by comparing two evaluations of a - * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}. All of these - * reduce the bug the same way: re-evaluate both sides against the reduced database and report whether they still - * disagree (or, when the original bug was an unexpected DBMS error, whether that same error still fires). + * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}). Reduction re-runs + * both sides against the reduced database and reports whether they still disagree. * *

- * This class owns that control flow (including distinguishing a still-reproducing error from an unrelated one - * introduced by the reduction) and the {@link #getBugInformation()} header. Subclasses supply the parts specific to - * their oracle: how each side is evaluated, how the two are compared, and how the failing queries are rendered in the - * reduced test case. + * The separate case where the original bug was an unexpected DBMS error rather than a mismatch is handled by + * {@link UnexpectedErrorReproducer}, so a subclass here deals only with comparing two sides and never with error + * handling. + * + *

+ * This class owns the compare-and-report control flow and the {@link #getBugInformation()} header. Subclasses supply + * how each side is evaluated, how the two are compared, and how the failing queries are rendered in the reduced test + * case. * * @param * the DBMS-specific global state class * @param - * the type each side evaluates to (e.g. a result set as a list of strings, a row count, a post-image) + * the type each side evaluates to (e.g. a result set as a list of strings, a row count) */ public abstract class AbstractComparisonReproducer, R> implements Reproducer { /** - * The message of the unexpected DBMS error the original bug was, or {@code null} if the original bug was a - * comparison mismatch rather than an error. - */ - protected final String expectedErrorMessage; - - protected AbstractComparisonReproducer(String expectedErrorMessage) { - this.expectedErrorMessage = expectedErrorMessage; - } - - /** - * Whether the recorded bug has a transformed (second) side. It does not when the bug was a DBMS error triggered by - * the original query alone, in which case there is no second side to evaluate or compare. - * - * @return {@code true} if {@link #evaluateTransformed} should be called - */ - protected abstract boolean hasTransformedSide(); - - /** - * Evaluates the original side against the (reduced) database. + * Evaluates the original side against the reduced database. * * @param globalState * the state whose connection points at the reduced database @@ -56,8 +41,7 @@ protected AbstractComparisonReproducer(String expectedErrorMessage) { protected abstract R evaluateOriginal(G globalState) throws SQLException; /** - * Evaluates the transformed side against the (reduced) database. Only called when {@link #hasTransformedSide()} is - * {@code true}. + * Evaluates the transformed side against the reduced database. * * @param globalState * the state whose connection points at the reduced database @@ -89,21 +73,9 @@ public final boolean bugStillTriggers(G globalState) { R transformed; try { original = evaluateOriginal(globalState); - if (!hasTransformedSide()) { - // the original bug was a DBMS error on the original query alone, which no longer occurs - return false; - } transformed = evaluateTransformed(globalState); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (SQLException | RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs + } catch (AssertionError | SQLException | RuntimeException e) { + // any failure re-running the two sides means this reduced database no longer shows the mismatch return false; } return sidesDiffer(original, transformed, globalState); @@ -112,28 +84,21 @@ public final boolean bugStillTriggers(G globalState) { @Override public final String getBugInformation() { StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage != null) { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage).append(System.lineSeparator()); - } else { - sb.append(mismatchHeaderLine()).append(System.lineSeparator()); - } + sb.append(mismatchHeaderLine()).append(System.lineSeparator()); appendQueryLines(sb); return sb.toString(); } /** - * The header line (without trailing line separator) describing the mismatch, used when the original bug was a - * comparison mismatch rather than an error. For example, "-- On the database set up by the statements above, the - * result sets of the following queries mismatch:". + * The header line (without trailing line separator) describing the mismatch. For example, "-- On the database set + * up by the statements above, the result sets of the following queries mismatch:". * * @return the mismatch header line */ protected abstract String mismatchHeaderLine(); /** - * Appends the failing queries (or statements) to {@code sb}, one commented line each, so the reduced test case is - * self-contained. Called for both the mismatch and the error case, after the header. + * Appends the failing queries to {@code sb}, one commented line each, so the reduced test case is self-contained. * * @param sb * the builder to append to diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index c3ad4bae9..0cd4ae99e 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -55,20 +55,13 @@ public class EETOracle, J extends Join, E private final class EETReproducer extends AbstractComparisonReproducer> { private final String originalQueryString; - // null if the original bug was a DBMS error on the original query alone private final String transformedQueryString; - EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { - super(expectedErrorMessage); + EETReproducer(String originalQueryString, String transformedQueryString) { this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; } - @Override - protected boolean hasTransformedSide() { - return transformedQueryString != null; - } - @Override protected List evaluateOriginal(G globalState) throws SQLException { // Re-execute against the current (reduced) database instead of comparing against a cached result set, @@ -100,11 +93,32 @@ protected String mismatchHeaderLine() { @Override protected void appendQueryLines(StringBuilder sb) { - sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + renderQueryLines(sb, originalQueryString, transformedQueryString); + } + } + + // Renders the failing queries as commented lines, shared by the mismatch and the unexpected-error reproducers. + // transformedQueryString is null when the error struck the original query before any transformation existed. + private static void renderQueryLines(StringBuilder sb, String originalQueryString, String transformedQueryString) { + sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (transformedQueryString != null) { + sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); + } + } + + // Builds the reproducer for an unexpected DBMS error, which re-runs the query (or both queries) and checks the same + // error still fires. transformedQueryString is null when only the original query ran before the error. + private UnexpectedErrorReproducer errorReproducer(String originalQueryString, String transformedQueryString, + String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); if (transformedQueryString != null) { - sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); + ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); } - } + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, originalQueryString, transformedQueryString); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); } public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { @@ -139,8 +153,8 @@ public void check() throws SQLException { originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); } catch (AssertionError unexpectedError) { // an unexpected DBMS error on the original query alone is itself a bug worth reducing; - // transformedQueryString is null because no transformed query is involved - reproducer = new EETReproducer(originalQueryString, null, + // there is no transformed query yet, so only the original is replayed + reproducer = errorReproducer(originalQueryString, null, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } @@ -160,14 +174,14 @@ public void check() throws SQLException { } catch (AssertionError unexpectedError) { // the semantics-preserving transformation made the query trigger a DBMS error that the // original did not, which is a bug worth reducing - reproducer = new EETReproducer(originalQueryString, transformedQueryString, + reproducer = errorReproducer(originalQueryString, transformedQueryString, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } // Set the reproducer before the assertion: assumeResultSetsAreEqual throws when the bug is // detected, so creating the reproducer afterwards would leave it null and prevent any reduction. - reproducer = new EETReproducer(originalQueryString, transformedQueryString, null); + reproducer = new EETReproducer(originalQueryString, transformedQueryString); ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), state); diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 11e52b91d..ba48e80bb 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -38,19 +38,13 @@ private static class NoRECReproducer> private final String unoptimizedQueryString; NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, - String optimizedQueryString, String unoptimizedQueryString, String expectedErrorMessage) { - super(expectedErrorMessage); + String optimizedQueryString, String unoptimizedQueryString) { this.optimizedQuery = optimizedQuery; this.unoptimizedQuery = unoptimizedQuery; this.optimizedQueryString = optimizedQueryString; this.unoptimizedQueryString = unoptimizedQueryString; } - @Override - protected boolean hasTransformedSide() { - return true; - } - @Override protected Integer evaluateOriginal(G globalState) { return optimizedQuery.apply(globalState); @@ -77,11 +71,29 @@ protected String mismatchHeaderLine() { @Override protected void appendQueryLines(StringBuilder sb) { - sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); - sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + renderQueryLines(sb, optimizedQueryString, unoptimizedQueryString); } } + // Renders the failing queries as commented lines, shared by the mismatch and the unexpected-error reproducers. + private static void renderQueryLines(StringBuilder sb, String optimizedQueryString, String unoptimizedQueryString) { + sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); + sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + } + + // Builds the reproducer for an unexpected DBMS error, which re-runs both queries and checks the same error fires. + private static > UnexpectedErrorReproducer errorReproducer( + Function optimizedQuery, Function unoptimizedQuery, String optimizedQueryString, + String unoptimizedQueryString, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + optimizedQuery.apply(globalState); + unoptimizedQuery.apply(globalState); + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, optimizedQueryString, unoptimizedQueryString); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -128,8 +140,8 @@ public void check() throws SQLException { optimizedCount = optimizedQuery.apply(state); unoptimizedCount = unoptimizedQuery.apply(state); } catch (AssertionError unexpectedError) { - reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, - unoptimizedQueryString, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + reproducer = errorReproducer(optimizedQuery, unoptimizedQuery, optimizedQueryString, unoptimizedQueryString, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } @@ -139,7 +151,7 @@ public void check() throws SQLException { if (unoptimizedCount != optimizedCount) { reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, - unoptimizedQueryString, null); + unoptimizedQueryString); String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index bf759ebfc..5e1d2861b 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -49,8 +49,7 @@ private class TLPWhereReproducer extends AbstractComparisonReproducer errorReproducer(String originalQueryString, String firstQueryString, + String secondQueryString, String thirdQueryString, boolean orderBy, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + if (firstQueryString != null) { + ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, + new ArrayList<>(), !orderBy, globalState, errors); + } + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, originalQueryString, firstQueryString, secondQueryString, thirdQueryString, orderBy); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -140,7 +158,7 @@ public void check() throws SQLException { try { firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); } catch (AssertionError unexpectedError) { - reproducer = new TLPWhereReproducer(null, null, null, originalQueryString, false, + reproducer = errorReproducer(originalQueryString, null, null, null, false, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } @@ -165,13 +183,13 @@ public void check() throws SQLException { secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); } catch (AssertionError unexpectedError) { - reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, - originalQueryString, orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + reproducer = errorReproducer(originalQueryString, firstQueryString, secondQueryString, thirdQueryString, + orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - orderBy, null); + orderBy); ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); } diff --git a/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java b/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java new file mode 100644 index 000000000..e9390943a --- /dev/null +++ b/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java @@ -0,0 +1,79 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; + +/** + * Reproducer for a bug that is an unexpected DBMS error, rather than a violation of oracle logic. When a statement run + * by the oracle raises an error the oracle did not expect, the bug is that error. Reduction re-runs the statement + * execution against the reduced database and reports whether the same error still fires. The oracle supplies how to + * re-run its execution as an {@link Execution} functional interface. + * + *

+ * For an oracle to use this reproducer on its unexpected errors, they must surface as {@link AssertionError}s, even + * though they likely originated as {@link SQLException}s. This is because + * {@link UnexpectedErrorReproducer#bugStillTriggers} treats a {@link SQLException} as a replay failure that means "bug + * does not trigger" (e.g. dropped connection, removed table). + * + * @param + * the DBMS-specific global state class + */ +public final class UnexpectedErrorReproducer> implements Reproducer { + + private final Execution execution; + private final String expectedErrorMessage; + private final String queryLines; + + @FunctionalInterface + public interface Execution { + /** + * Re-runs the oracle's execution against the reduced database. The bug is treated as still present if the run + * continues to raise an {@link AssertionError} with the same message. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @throws SQLException + * if a DBMS interaction fails for a reason other than the recorded bug (e.g. a connection or setup + * failure during replay), which counts as the bug no longer triggering + */ + void execute(G globalState) throws SQLException; + } + + /** + * @param execution + * re-runs the oracle's execution against the reduced database + * @param expectedErrorMessage + * the message of the error the original bug was. The bug is treated as still present if the + * {@link Execution} continues to raise an {@link AssertionError} with the same message. + * @param queryLines + * the failing queries as commented lines (each ending in a line separator), for the reduced test case + */ + public UnexpectedErrorReproducer(Execution execution, String expectedErrorMessage, String queryLines) { + this.execution = execution; + this.expectedErrorMessage = expectedErrorMessage; + this.queryLines = queryLines; + } + + @Override + public boolean bugStillTriggers(G globalState) { + try { + execution.execute(globalState); + } catch (AssertionError unexpectedError) { + // the same error reproduces the bug; a different one is an artifact of the reduction (e.g. a removed table) + return expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + // the error no longer fires + return false; + } + + @Override + public String getBugInformation() { + return "-- On the database set up by the statements above, the following queries trigger an unexpected error" + + " with message: " + expectedErrorMessage + System.lineSeparator() + queryLines; + } +} From 96ab07d6fd1abde8a6f53bfa59b0f6736fbff79b Mon Sep 17 00:00:00 2001 From: splf Date: Fri, 31 Jul 2026 10:07:42 +0500 Subject: [PATCH 1269/1284] Expect the to_char error for RN combined with other formats PostgreSQL rejects a numeric format string that combines RN with any other pattern, for example to_char(1.5, 'RN0'), with the error "RN" is incompatible with other formats. The expression generator can produce such format strings, so this error reached the caller as an unexpected one and aborted the test run. --- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index eeb160a56..180b4e449 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -123,6 +123,7 @@ private static List getToCharFunctionErrors() { errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); errors.add("\"EEEE\" must be the last pattern used"); + errors.add("is incompatible with other formats"); return errors; } From adc0875fe76c4d48939fc4be2a0097ee715e521a Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 28 Jul 2026 16:40:37 +0800 Subject: [PATCH 1270/1284] Add support for DECIMAL(M, D) columns in MySQL EET --- src/sqlancer/mysql/MySQLSchema.java | 12 ++- .../mysql/ast/MySQLCastOperation.java | 80 +++++++++++++++++-- .../mysql/gen/MySQLTableGenerator.java | 21 +++-- .../mysql/oracle/MySQLEETTransformer.java | 27 ++++--- .../mysql/MySQLToStringVisitorTest.java | 6 +- .../mysql/ast/MySQLCaseOperatorTest.java | 4 +- 6 files changed, 120 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index c8a30614f..ab0ba4543 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -57,6 +57,7 @@ public static class MySQLColumn extends AbstractTableColumn getTableColumns(SQLConnection con, String table String columnName = rs.getString("COLUMN_NAME"); String dataType = rs.getString("DATA_TYPE"); int precision = rs.getInt("NUMERIC_PRECISION"); + int scale = rs.getInt("NUMERIC_SCALE"); boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); - MySQLColumn c = new MySQLColumn(columnName, getColumnType(dataType), isPrimaryKey, precision); + MySQLColumn c = new MySQLColumn(columnName, getColumnType(dataType), isPrimaryKey, precision, + scale); columns.add(c); } } diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index b71d0498c..07c45a429 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -1,21 +1,91 @@ package sqlancer.mysql.ast; +import java.util.Objects; + public class MySQLCastOperation implements MySQLExpression { private final MySQLExpression expr; private final CastType type; - public enum CastType { - SIGNED, UNSIGNED, - // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never - // evaluated, so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). - CHAR, FLOAT, DOUBLE, DECIMAL; + /** + * A MySQL {@code CAST} target type. The non-{@code DECIMAL} kinds are interned singletons; {@code DECIMAL} may + * additionally carry an {@code (M, D)} precision/scale (via {@link #decimal}) so that a + * {@code CAST(... AS DECIMAL(M, D))} can reproduce a column's exact type. This is relied on by the EET oracle's + * type-pinning casts (see {@code MySQLEETTransformer}). + */ + public static final class CastType { + + // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never evaluated, + // so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + public static final CastType SIGNED = new CastType(Kind.SIGNED); + public static final CastType UNSIGNED = new CastType(Kind.UNSIGNED); + public static final CastType CHAR = new CastType(Kind.CHAR); + public static final CastType FLOAT = new CastType(Kind.FLOAT); + public static final CastType DOUBLE = new CastType(Kind.DOUBLE); + public static final CastType DECIMAL = new CastType(Kind.DECIMAL); + + private enum Kind { + SIGNED, UNSIGNED, CHAR, FLOAT, DOUBLE, DECIMAL + } + + private final Kind kind; + private final Integer precision; // DECIMAL only, otherwise null + private final Integer scale; // DECIMAL only, otherwise null + + private CastType(Kind kind) { + this(kind, null, null); + } + + private CastType(Kind kind, Integer precision, Integer scale) { + this.kind = kind; + this.precision = precision; + this.scale = scale; + } + + // A DECIMAL(precision, scale) cast target. + public static CastType decimal(int precision, int scale) { + return new CastType(Kind.DECIMAL, precision, scale); + } public static CastType getRandom() { return SIGNED; // return Randomly.fromOptions(CastType.SIGNED, CastType.UNSIGNED); } + public Integer getPrecision() { + return precision; + } + + public Integer getScale() { + return scale; + } + + @Override + public String toString() { + if (precision == null) { + return kind.name(); + } + return kind.name() + "(" + precision + ", " + scale + ")"; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof CastType)) { + return false; + } + CastType other = (CastType) obj; + return kind == other.kind && Objects.equals(precision, other.precision) + && Objects.equals(scale, other.scale); + } + + @Override + public int hashCode() { + return Objects.hash(kind, precision, scale); + } + } public MySQLCastOperation(MySQLExpression expr, CastType type) { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 40e325041..b7c9a3566 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -347,11 +347,11 @@ private void appendType(MySQLDataType randomType) { break; case FLOAT: sb.append("FLOAT"); - optionallyAddPrecisionAndScale(sb); + optionallyAddFloatingPointPrecisionAndScale(sb); break; case DOUBLE: sb.append(Randomly.fromOptions("DOUBLE", "FLOAT")); - optionallyAddPrecisionAndScale(sb); + optionallyAddFloatingPointPrecisionAndScale(sb); break; default: throw new AssertionError(); @@ -368,13 +368,20 @@ private void appendType(MySQLDataType randomType) { } } - private void optionallyAddPrecisionAndScale(StringBuilder sb) { - // The EET oracle's type inference assumes FLOAT/DOUBLE/DECIMAL columns are created without (M, D) (see - // MySQLEETTransformer#inferColumnType), so precision/scale is omitted while EET is active. This restriction can - // be lifted once (M, D) is tracked through the codebase and reflected in the CAST target types. + // FLOAT(M, D)/DOUBLE(M, D) is deprecated and cannot be reproduced as a CAST target, so the EET oracle's type + // inference relies on FLOAT/DOUBLE columns being created without (M, D) (see MySQLEETTransformer#inferColumnType); + // it is therefore omitted while EET is active. DECIMAL(M, D) has no such restriction: the EET oracle tracks its + // (M, D) and reproduces it via CAST(... AS DECIMAL(M, D)), so it keeps using optionallyAddPrecisionAndScale. + private void optionallyAddFloatingPointPrecisionAndScale(StringBuilder sb) { boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.EET); - if (Randomly.getBoolean() && !MySQLBugs.bug99183 && !eetActive) { + if (!eetActive) { + optionallyAddPrecisionAndScale(sb); + } + } + + private void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean() && !MySQLBugs.bug99183) { sb.append("("); // The maximum number of digits (M) for DECIMAL is 65 long m = Randomly.getNotCachedInteger(1, 65); diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 2e4394eec..22c0302df 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -5,6 +5,7 @@ import java.util.stream.Collectors; import sqlancer.common.oracle.EETTransformer; +import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; @@ -33,11 +34,11 @@ * *

* MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target - * types ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning - * {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping - * it in a CAST. + * types ({@link CastType}, which carries {@code (M, D)} for DECIMAL): {@link #inferType} conservatively classifies AST + * nodes into that domain (returning {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type + * of a random expression by wrapping it in a CAST. */ -public class MySQLEETTransformer extends EETTransformer { +public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; @@ -212,20 +213,22 @@ private CastType inferConstantType(MySQLConstant constant) { } private CastType inferColumnType(MySQLColumnReference ref) { - switch (ref.getColumn().getType()) { + MySQLColumn column = ref.getColumn(); + switch (column.getType()) { case INT: return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns case VARCHAR: return CastType.CHAR; - // FLOAT/DOUBLE/DECIMAL columns are created without (M, D) while EET is active, so the plain - // CAST target below matches the column's type. Reintroducing (M, D) for better coverage would - // require tracking it here and emitting the exact precision/scale in the CAST. case FLOAT: + // FLOAT/DOUBLE columns are created without (M, D) while EET is active (the (M, D) form is deprecated and + // not a valid CAST target), so the plain CAST target matches the column's type. return CastType.FLOAT; case DOUBLE: return CastType.DOUBLE; case DECIMAL: - return CastType.DECIMAL; + // DECIMAL columns may carry (M, D); CAST(... AS DECIMAL(M, D)) reproduces the column's exact type. The + // schema reports (M, D) even for a plain DECIMAL column (defaulting to (10, 0)). + return CastType.decimal(column.getPrecision(), column.getScale()); default: return null; } @@ -259,7 +262,8 @@ private CastType inferCaseType(MySQLCaseOperator caseOp) { /** * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same - * inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * inferrable type (a conservative under-approximation of MySQL's aggregation rules). Two DECIMAL subexpressions + * with differing {@code (M, D)} therefore yield {@code null} rather than a guessed aggregate. * * @param exprs * the result-type-determining subexpressions @@ -270,7 +274,8 @@ private CastType commonType(MySQLExpression... exprs) { CastType common = null; for (MySQLExpression expr : exprs) { CastType type = inferType(expr); - if (type == null || common != null && type != common) { + // equals (not ==) so two DECIMAL types with matching (M, D) but distinct instances compare as equal. + if (type == null || common != null && !type.equals(common)) { return null; } common = type; diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 3f991e3f1..d39c2695c 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -17,7 +17,7 @@ public class MySQLToStringVisitorTest { @Test void visitAggregateToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCount = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT); @@ -35,7 +35,7 @@ void visitAggregateToString() { @Test void visitAggregateWithDistinctToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCountDistinct = new MySQLAggregate(List.of(aRef), @@ -57,7 +57,7 @@ void visitAggregateWithDistinctToString() { @Test void visitCaseWhenToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); diff --git a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java index 674ab027e..757241445 100644 --- a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java +++ b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java @@ -14,7 +14,7 @@ public class MySQLCaseOperatorTest { @Test void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createIntConstant(1)); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); @@ -29,7 +29,7 @@ void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { @Test void getExpectedValue_switchConditionHasNoMatches_ReturnsElse() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createNullConstant()); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); From 9b739c0501f848c0a0907728356bbe43bed926c4 Mon Sep 17 00:00:00 2001 From: splf Date: Mon, 3 Aug 2026 12:49:45 +0500 Subject: [PATCH 1271/1284] Do not return null when PostgreSQL tablespaces are disabled --- .../gen/PostgresTableSpaceGenerator.java | 7 ++++-- .../gen/TestPostgresTableSpaceGenerator.java | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java index 3890d5160..1be99ca61 100644 --- a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -1,5 +1,6 @@ package sqlancer.postgres.gen; +import sqlancer.IgnoreMeException; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -19,10 +20,12 @@ public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { } public static SQLQueryAdapter generate(PostgresGlobalState globalState) { - // Skip tablespace generation if the option is disabled + // PostgresProvider.mapActions does not schedule this action when the option is disabled, but QPG + // selects actions by index without consulting the schedule, so the generator has to report that + // it has nothing to generate. PostgresOptions options = globalState.getDbmsSpecificOptions(); if (!options.isTestTablespaces()) { - return null; + throw new IgnoreMeException(); } return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); } diff --git a/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java b/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java new file mode 100644 index 000000000..c2f626041 --- /dev/null +++ b/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.postgres.gen; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import sqlancer.IgnoreMeException; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; + +class TestPostgresTableSpaceGenerator { + + @Test + void generateIsSkippedWhenTablespacesAreDisabled() { + PostgresGlobalState state = new PostgresGlobalState(); + state.setDbmsSpecificOptions(new PostgresOptions() { + @Override + public boolean isTestTablespaces() { + return false; + } + }); + + assertThrows(IgnoreMeException.class, () -> PostgresTableSpaceGenerator.generate(state)); + } +} From 6ac84559193fa5f004ac6c18a020ade4a86ba68b Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 13:37:48 +0800 Subject: [PATCH 1272/1284] Add support for DELETE statements in EET (with MySQL-specific implementation), including generic DML infrastructure for future extensions to other DML statements --- src/sqlancer/common/gen/EETDMLGenerator.java | 155 +++++++++++++++++ src/sqlancer/common/oracle/EETDMLOracle.java | 158 ++++++++++++++++++ src/sqlancer/mysql/MySQLErrors.java | 17 ++ src/sqlancer/mysql/MySQLGlobalState.java | 9 + src/sqlancer/mysql/MySQLOracleFactory.java | 10 ++ .../mysql/gen/MySQLExpressionGenerator.java | 22 ++- .../mysql/gen/MySQLTableGenerator.java | 14 +- 7 files changed, 378 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/common/gen/EETDMLGenerator.java create mode 100644 src/sqlancer/common/oracle/EETDMLOracle.java diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java new file mode 100644 index 000000000..293574a36 --- /dev/null +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -0,0 +1,155 @@ +package sqlancer.common.gen; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.oracle.EETTransformer; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * Generator interface used by {@link sqlancer.common.oracle.EETDMLOracle}, the DML counterpart of {@link EETGenerator}. + * It supplies methods which generate the transformable expressions, create the DBMS-specific {@link EETTransformer} + * that rewrites them, and produce the SQL of the statements the oracle uses to observe the database state a statement + * produces (an approach drawn from the DQE oracle). + * + *

+ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which + * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run + * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are + * produced. + * + *

+ * Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ +public interface EETDMLGenerator, T extends AbstractTable, C extends AbstractTableColumn> { + + /** Name of the auxiliary column that uniquely identifies each row. */ + String ROW_ID_COLUMN = "rowid"; + + /** + * Restricts this generator to the given tables (a single table, for the DML statement under test) and their + * columns. + * + * @param tables + * the tables (and, implicitly, columns) the generated statement operates on + * + * @return this generator + */ + EETDMLGenerator setTablesAndColumns(AbstractTables tables); + + /** + * Generates a fresh random boolean expression over the current tables' columns, used as the DML statement's WHERE + * predicate. + * + * @return a fresh random boolean expression + */ + E generateBooleanExpression(); + + /** + * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's + * expressions into semantically equivalent ones. + * + * @return a DBMS-specific {@link EETTransformer} + */ + EETTransformer createTransformer(); + + // --- DBMS-specific primitives --- + + /** + * Renders an expression to its DBMS-specific SQL string. + * + * @param expr + * the expression to render + * + * @return the SQL text of {@code expr} + */ + String asString(E expr); + + /** + * SQL that assigns every existing row of {@code table} a distinct, stable identifier in the {@link #ROW_ID_COLUMN} + * column. DBMS-specific because it names the DBMS's UUID-generating function. + * + * @param table + * the table whose rows are stamped + * + * @return the SQL statement + */ + String stampRowIdsStatement(T table); + + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- + + /** + * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}. + * + * @param table + * the table to add the column to + * + * @return the SQL statement + */ + default String addRowIdColumnStatement(T table) { + return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " VARCHAR(36)"; + } + + /** + * SQL that drops the auxiliary {@link #ROW_ID_COLUMN} column from {@code table}. + * + * @param table + * the table to drop the column from + * + * @return the SQL statement + */ + default String dropRowIdColumnStatement(T table) { + return "ALTER TABLE " + table.getName() + " DROP COLUMN " + ROW_ID_COLUMN; + } + + /** + * SQL that selects the {@link #ROW_ID_COLUMN} of every row of {@code table} (the surviving-row snapshot). + * + * @param table + * the table to snapshot + * + * @return the SQL statement; its first result column must be the identifiers + */ + default String selectRowIdsStatement(T table) { + return "SELECT " + ROW_ID_COLUMN + " FROM " + table.getName(); + } + + /** + * SQL that deletes the rows of {@code table} matching {@code predicate}. + * + * @param table + * the table to delete from + * @param predicate + * the WHERE predicate; rendered via {@link #asString} + * + * @return the SQL statement + */ + default String deleteStatement(T table, E predicate) { + return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); + } + + /** + * SQL that starts a transaction, so a statement's effect can be observed and then undone. + * + * @return the SQL statement + */ + default String beginTransactionStatement() { + return "BEGIN"; + } + + /** + * SQL that rolls the current transaction back, undoing the statement's effect. + * + * @return the SQL statement + */ + default String rollbackTransactionStatement() { + return "ROLLBACK"; + } +} diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java new file mode 100644 index 000000000..33f9ff411 --- /dev/null +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -0,0 +1,158 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.gen.EETDMLGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * EET (Equivalent Expression Transformation) oracle for DML statements, based on "Detecting Logic Bugs in Database + * Engines via Equivalent Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * Whereas {@link EETOracle} transforms a SELECT and compares the two result sets, this oracle transforms a DML + * statement and compares the two database states produced. + * + *

+ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which + * uniquely identifies each row, and each statement is executed inside a transaction that is rolled back, so the two + * statements can be compared against the same starting state without permanently modifying the database. For a DELETE, + * the state is captured as the set of surviving row identifiers. Because rolling back a statement requires a + * transactional storage engine, the DBMS-specific setup must ensure only such engines are used while this oracle is + * active. + * + *

+ * Only DELETE is currently supported. Statement reduction is not yet implemented (there is no + * {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database reduction. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific schema class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + * @param + * the DBMS-specific global state class + */ +public class EETDMLOracle, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private EETDMLGenerator gen; + private final EETTransformer transformer; + private final ExpectedErrors errors; + + private String generatedQueryString; + + public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.transformer = gen.createTransformer(); + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + List tables = state.getSchema().getDatabaseTables(); + if (tables.isEmpty()) { + throw new IgnoreMeException(); + } + // DELETE targets a single table, so operate on exactly one; confining the generator to it keeps the predicate + // from referencing another table's columns (which would render invalid single-table DML). + T table = Randomly.fromList(tables); + gen = gen.setTablesAndColumns(new AbstractTables<>(List.of(table))); + + E predicate = gen.generateBooleanExpression(); + // The WHERE predicate is evaluated in a boolean context. + E transformedPredicate = transformer.transform(predicate, true); + + String originalDelete = gen.deleteStatement(table, predicate); + String transformedDelete = gen.deleteStatement(table, transformedPredicate); + generatedQueryString = originalDelete; + + // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: + // the ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak + // the column into the next iteration and cause cascading duplicate-column failures. + new SQLQueryAdapter(gen.addRowIdColumnStatement(table), true).execute(state); + try { + // Stamp identifiers once, in autocommit mode, before both DELETEs run: both then observe the same rows. + new SQLQueryAdapter(gen.stampRowIdsStatement(table)).execute(state); + + Set originalSurvivors = executeDeleteAndSnapshot(table, originalDelete); + Set transformedSurvivors = executeDeleteAndSnapshot(table, transformedDelete); + + if (!originalSurvivors.equals(transformedSurvivors)) { + throw new AssertionError( + mismatchMessage(originalDelete, transformedDelete, originalSurvivors, transformedSurvivors)); + } + } finally { + new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), true).execute(state); + } + } + + /** + * Executes {@code deleteStatement} inside a transaction that is always rolled back, and returns the set of row + * identifiers surviving the DELETE (the resulting database state). A DBMS error expected by the oracle aborts the + * whole check ({@link IgnoreMeException}) rather than being reported, matching {@link EETOracle}'s handling; an + * unexpected error surfaces as a bug ({@link AssertionError}, thrown by the query adapter). + * + * @param table + * the table being deleted from + * @param deleteStatement + * the DELETE statement to execute + * + * @return the set of row identifiers surviving the DELETE + * + * @throws SQLException + * if a DBMS interaction fails + */ + private Set executeDeleteAndSnapshot(T table, String deleteStatement) throws SQLException { + new SQLQueryAdapter(gen.beginTransactionStatement()).execute(state); + try { + // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. + boolean succeeded = new SQLQueryAdapter(deleteStatement, errors).execute(state); + if (!succeeded) { + // The DELETE hit an error the oracle tolerates; do not compare states (as EETOracle does for SELECT). + throw new IgnoreMeException(); + } + return new HashSet<>( + ComparatorHelper.getResultSetFirstColumnAsString(gen.selectRowIdsStatement(table), errors, state)); + } finally { + new SQLQueryAdapter(gen.rollbackTransactionStatement()).execute(state); + } + } + + private static String mismatchMessage(String originalDelete, String transformedDelete, + Set originalSurvivors, Set transformedSurvivors) { + return new StringBuilder() + .append("-- The original and transformed DELETE statements left the database in different states") + .append(" (different sets of surviving rows):").append(System.lineSeparator()).append("-- original (") + .append(originalSurvivors.size()).append(" rows survive): ").append(originalDelete).append(';') + .append(System.lineSeparator()).append("-- transformed (").append(transformedSurvivors.size()) + .append(" rows survive): ").append(transformedDelete).append(';').append(System.lineSeparator()) + .toString(); + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index cb9ad4f01..44316dc3f 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -65,4 +65,21 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { errors.addAll(getInsertUpdateErrors()); } + public static List getDMLErrors() { + ArrayList errors = new ArrayList<>(); + + // WHERE-clause type coercion (e.g. string -> number) is only a warning in SELECT but a hard error in + // DELETE/UPDATE under strict sql_mode (MySQL 1292). A semantics-preserving transform may benignly change + // whether it fires, so it is tolerated rather than flagged. + errors.add("Truncated incorrect"); + // Foreign key constraint failure when deleting/updating a referenced row. + errors.add("a foreign key constraint fails"); + + return errors; + } + + public static void addDMLErrors(ExpectedErrors errors) { + errors.addAll(getDMLErrors()); + } + } diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 10132b57c..a34861cbd 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -16,4 +16,13 @@ public boolean usesPQS() { return getDbmsSpecificOptions().oracles.stream().anyMatch(o -> o == MySQLOracleFactory.PQS); } + public boolean usesEET() { + return getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.EET || o == MySQLOracleFactory.EET_DML); + } + + public boolean usesEETDML() { + return getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.EET_DML); + } + } diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index ed5ddc489..d3d4b20b1 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -5,6 +5,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.EETDMLOracle; import sqlancer.common.oracle.EETOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; @@ -92,5 +93,14 @@ public TestOracle create(MySQLGlobalState globalState) throws .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); return new EETOracle<>(globalState, gen, expectedErrors); } + }, + EET_DML { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).with(MySQLErrors.getDMLErrors()).build(); + return new EETDMLOracle<>(globalState, gen, expectedErrors); + } }; } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index da304ac67..5a694bd78 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.EETDMLGenerator; import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; @@ -19,6 +20,7 @@ import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLBetweenOperation; @@ -52,7 +54,8 @@ public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, CERTGenerator, - EETGenerator { + EETGenerator, + EETDMLGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; @@ -364,10 +367,25 @@ boolean mutateOr(MySQLSelect select) { } } - // --- EET oracle --- + // --- EET oracle (including DML) --- @Override public EETTransformer createTransformer() { return new MySQLEETTransformer(this); } + + // --- EET DML only --- + + @Override + public String asString(MySQLExpression expr) { + return MySQLVisitor.asString(expr); + } + + @Override + public String stampRowIdsStatement(MySQLTable table) { + // MySQL's UUID() gives each existing row a distinct value in a single statement. Stamping happens once, before + // both rolled-back DELETE runs, so both observe identical identifiers; the standard-SQL statements (add/drop + // column, delete, snapshot, transaction control) use EETDMLGenerator's defaults. + return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); + } } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index b7c9a3566..0746056ad 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -198,7 +198,10 @@ private void appendTableOptions() { // "NDB": java.sql.SQLSyntaxErrorException: Unknown storage engine 'NDB' // "EXAMPLE": java.sql.SQLSyntaxErrorException: Unknown storage engine 'EXAMPLE' // "MERGE": java.sql.SQLException: Table 't0' is read only - String fromOptions = Randomly.fromOptions("InnoDB", "MyISAM", "MEMORY", "HEAP", "CSV", "ARCHIVE"); + // The EET DML oracle rolls back each statement to compare database states, which requires a + // transactional engine, so only InnoDB is used while it is active. + String fromOptions = globalState.usesEETDML() ? "InnoDB" + : Randomly.fromOptions("InnoDB", "MyISAM", "MEMORY", "HEAP", "CSV", "ARCHIVE"); this.engine = MySQLEngine.get(fromOptions); sb.append("ENGINE = "); sb.append(fromOptions); @@ -362,7 +365,8 @@ private void appendType(MySQLDataType randomType) { } if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS - || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET)) { + || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET + || o == MySQLOracleFactory.EET_DML)) { sb.append(" ZEROFILL"); } } @@ -373,9 +377,9 @@ private void appendType(MySQLDataType randomType) { // it is therefore omitted while EET is active. DECIMAL(M, D) has no such restriction: the EET oracle tracks its // (M, D) and reproduces it via CAST(... AS DECIMAL(M, D)), so it keeps using optionallyAddPrecisionAndScale. private void optionallyAddFloatingPointPrecisionAndScale(StringBuilder sb) { - boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() - .anyMatch(o -> o == MySQLOracleFactory.EET); - if (!eetActive) { + // Both EET oracles rely on the same type inference (MySQLEETTransformer), so both omit FLOAT(M, D)/DOUBLE(M, + // D). + if (!globalState.usesEET()) { optionallyAddPrecisionAndScale(sb); } } From 514f14959e295f56981514248bf8a96660596ccd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 15:10:55 +0800 Subject: [PATCH 1273/1284] Add support for LIMIT on DELETE statements in EET --- src/sqlancer/common/gen/EETDMLGenerator.java | 32 ++++++++++++++++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 13 ++++++-- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 293574a36..375877a63 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -1,5 +1,8 @@ package sqlancer.common.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.ast.newast.Expression; import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTable; @@ -122,17 +125,40 @@ default String selectRowIdsStatement(T table) { } /** - * SQL that deletes the rows of {@code table} matching {@code predicate}. + * SQL that deletes the rows of {@code table} matching {@code predicate}, optionally limited to the first + * {@code limit} rows. + * + *

+ * When {@code limit} is non-null, the statement is ordered by {@code orderByColumns} followed by + * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even when + * the ordering columns tie), so the "first {@code limit}" rows are identical for the original and + * transformed statements. Varying the ordering columns exercises more access + * paths than the row id alone would. The caller must pass the same {@code orderByColumns} and {@code limit} + * to both statements; neither is transformed. * * @param table * the table to delete from * @param predicate * the WHERE predicate; rendered via {@link #asString} + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty); only used when {@code limit} is + * non-null + * @param limit + * the maximum number of rows to delete, or {@code null} for no limit * * @return the SQL statement */ - default String deleteStatement(T table, E predicate) { - return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); + default String deleteStatement(T table, E predicate, List orderByColumns, Integer limit) { + String statement = "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); + if (limit != null) { + List orderBy = new ArrayList<>(); + for (C column : orderByColumns) { + orderBy.add(column.getName()); + } + orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above + statement += " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; + } + return statement; } /** diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 33f9ff411..792bc52c0 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -84,8 +84,17 @@ public void check() throws SQLException { // The WHERE predicate is evaluated in a boolean context. E transformedPredicate = transformer.transform(predicate, true); - String originalDelete = gen.deleteStatement(table, predicate); - String transformedDelete = gen.deleteStatement(table, transformedPredicate); + // Optionally cap the DELETE with a LIMIT. The limit and its ordering (a random column subset, made a total + // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped + // row set is deterministic and equal across the runs while still exercising varied orderings. + Integer limit = null; + List orderByColumns = List.of(); + if (Randomly.getBoolean()) { + limit = (int) Randomly.getNotCachedInteger(0, 10); + orderByColumns = Randomly.subset(table.getColumns()); + } + String originalDelete = gen.deleteStatement(table, predicate, orderByColumns, limit); + String transformedDelete = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); generatedQueryString = originalDelete; // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: From 0dea938244b227cc03c900982c7730a78544f002 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 15:33:50 +0800 Subject: [PATCH 1274/1284] Add existing insert/update errors to EET DML oracle expecteed errors --- src/sqlancer/common/gen/EETDMLGenerator.java | 9 ++++----- src/sqlancer/common/oracle/EETDMLOracle.java | 14 ++++++++++---- src/sqlancer/mysql/MySQLErrors.java | 2 +- src/sqlancer/mysql/MySQLOracleFactory.java | 6 +++++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 375877a63..82888e070 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -130,11 +130,10 @@ default String selectRowIdsStatement(T table) { * *

* When {@code limit} is non-null, the statement is ordered by {@code orderByColumns} followed by - * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even when - * the ordering columns tie), so the "first {@code limit}" rows are identical for the original and - * transformed statements. Varying the ordering columns exercises more access - * paths than the row id alone would. The caller must pass the same {@code orderByColumns} and {@code limit} - * to both statements; neither is transformed. + * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even + * when the ordering columns tie), so the "first {@code limit}" rows are identical for the original and transformed + * statements. Varying the ordering columns exercises more access paths than the row id alone would. The caller must + * pass the same {@code orderByColumns} and {@code limit} to both statements; neither is transformed. * * @param table * the table to delete from diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 792bc52c0..ce41039d6 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -99,11 +99,17 @@ public void check() throws SQLException { // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: // the ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak - // the column into the next iteration and cause cascading duplicate-column failures. - new SQLQueryAdapter(gen.addRowIdColumnStatement(table), true).execute(state); + // the column into the next iteration and cause cascading duplicate-column failures. The row-identity setup + // touches rows, so — like the DELETE itself — it can raise tolerated errors (e.g. functional-index maintenance + // truncation); such an error aborts the iteration (IgnoreMeException) rather than being reported as a bug. + if (!new SQLQueryAdapter(gen.addRowIdColumnStatement(table), errors, true).execute(state)) { + throw new IgnoreMeException(); + } try { // Stamp identifiers once, in autocommit mode, before both DELETEs run: both then observe the same rows. - new SQLQueryAdapter(gen.stampRowIdsStatement(table)).execute(state); + if (!new SQLQueryAdapter(gen.stampRowIdsStatement(table), errors).execute(state)) { + throw new IgnoreMeException(); + } Set originalSurvivors = executeDeleteAndSnapshot(table, originalDelete); Set transformedSurvivors = executeDeleteAndSnapshot(table, transformedDelete); @@ -113,7 +119,7 @@ public void check() throws SQLException { mismatchMessage(originalDelete, transformedDelete, originalSurvivors, transformedSurvivors)); } } finally { - new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), true).execute(state); + new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); } } diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 44316dc3f..0b6eb8284 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -66,7 +66,7 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { } public static List getDMLErrors() { - ArrayList errors = new ArrayList<>(); + ArrayList errors = new ArrayList<>(getInsertUpdateErrors()); // WHERE-clause type coercion (e.g. string -> number) is only a warning in SELECT but a hard error in // DELETE/UPDATE under strict sql_mode (MySQL 1292). A semantics-preserving transform may benignly change diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index d3d4b20b1..8b37efb13 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -99,7 +99,11 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) - .withRegex(MySQLErrors.getExpressionRegexErrors()).with(MySQLErrors.getDMLErrors()).build(); + .withRegex(MySQLErrors.getExpressionRegexErrors()) + // The DML statements and the row-identity setup (adding/stamping the auxiliary column) touch rows, + // so they can raise the full range of DML errors — e.g. functional-index maintenance truncation — + // beyond the SELECT-based expression errors. + .with(MySQLErrors.getDMLErrors()).build(); return new EETDMLOracle<>(globalState, gen, expectedErrors); } }; From 85d837a7215911cd11418e8e1caec6ca93c633f7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 8 Aug 2026 10:39:52 +0800 Subject: [PATCH 1275/1284] Increase generality of EETDMLGenerator.addRowIdColumnStatement --- src/sqlancer/common/gen/EETDMLGenerator.java | 15 ++++++++++++--- .../mysql/gen/MySQLExpressionGenerator.java | 6 ++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 82888e070..b56580a9d 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -77,7 +77,7 @@ public interface EETDMLGenerator, T extends AbstractTabl /** * SQL that assigns every existing row of {@code table} a distinct, stable identifier in the {@link #ROW_ID_COLUMN} - * column. DBMS-specific because it names the DBMS's UUID-generating function. + * column. For example, a 36-character UUID string. * * @param table * the table whose rows are stamped @@ -86,10 +86,19 @@ public interface EETDMLGenerator, T extends AbstractTabl */ String stampRowIdsStatement(T table); + /** + * The SQL type of the auxiliary {@link #ROW_ID_COLUMN} column. It must be able to hold the identifiers that + * {@link #stampRowIdsStatement} produces, so it belongs with that statement as the other half of the row-id + * representation. For example, {@code VARCHAR(36)} would fit a 36-character UUID string. + * + * @return the column type + */ + String rowIdColumnType(); + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- /** - * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}. + * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}, typed as {@link #rowIdColumnType}. * * @param table * the table to add the column to @@ -97,7 +106,7 @@ public interface EETDMLGenerator, T extends AbstractTabl * @return the SQL statement */ default String addRowIdColumnStatement(T table) { - return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " VARCHAR(36)"; + return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " " + rowIdColumnType(); } /** diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 5a694bd78..9a62c15ba 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -388,4 +388,10 @@ public String stampRowIdsStatement(MySQLTable table) { // column, delete, snapshot, transaction control) use EETDMLGenerator's defaults. return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); } + + @Override + public String rowIdColumnType() { + // Holds a 36-character UUID string produced by stampRowIdsStatement. + return "VARCHAR(36)"; + } } From 7b9b0549aa945ef583f94fed0b90bef7dad318b7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 19:48:07 +0800 Subject: [PATCH 1276/1284] Implement UPDATE support for EET DML oracle --- src/sqlancer/common/gen/EETDMLGenerator.java | 111 ++++++++--- src/sqlancer/common/oracle/EETDMLOracle.java | 174 +++++++++++++----- .../mysql/gen/MySQLExpressionGenerator.java | 18 +- 3 files changed, 231 insertions(+), 72 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index b56580a9d..9fb21ea35 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import sqlancer.common.ast.newast.Expression; import sqlancer.common.oracle.EETTransformer; @@ -19,7 +20,8 @@ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are - * produced. + * produced. The resulting state is compared as a full post-image (each surviving row's identifier and content column + * values), which covers every DML statement: a DELETE removes rows from it, an UPDATE changes values in it. * *

* Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. @@ -55,6 +57,15 @@ public interface EETDMLGenerator, T extends AbstractTabl */ E generateBooleanExpression(); + /** + * Generates a fresh set of {@code column = value} assignments over the current tables' columns, used as an UPDATE + * statement's SET clause. The columns are a random non-empty subset and each value is a fresh random expression; + * both the columns and their assigned expressions are transformed by the oracle. + * + * @return the assignments, as {@code (column, value expression)} pairs (at least one) + */ + List> generateSetAssignments(); + /** * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's * expressions into semantically equivalent ones. @@ -122,27 +133,32 @@ default String dropRowIdColumnStatement(T table) { } /** - * SQL that selects the {@link #ROW_ID_COLUMN} of every row of {@code table} (the surviving-row snapshot). + * SQL that reads back the full post-image of {@code table}: the {@link #ROW_ID_COLUMN} identifier and every content + * column of every surviving row, ordered by the (unique) identifier so the two statements' snapshots align + * row-for-row. + * + *

+ * This single value-level snapshot is the comparison surface for all DML statements: a DELETE removes rows from it, + * an UPDATE changes column values in it. Row identity alone (which the identifier already captures) would suffice + * for DELETE, but not for UPDATE, where the two runs could touch the same rows yet write different values. * * @param table * the table to snapshot * - * @return the SQL statement; its first result column must be the identifiers + * @return the SQL statement; its first result column is the identifier, followed by {@code table}'s content columns */ - default String selectRowIdsStatement(T table) { - return "SELECT " + ROW_ID_COLUMN + " FROM " + table.getName(); + default String selectPostImageStatement(T table) { + List selected = new ArrayList<>(); + selected.add(ROW_ID_COLUMN); + for (C column : table.getColumns()) { + selected.add(column.getName()); + } + return "SELECT " + String.join(", ", selected) + " FROM " + table.getName() + " ORDER BY " + ROW_ID_COLUMN; } /** * SQL that deletes the rows of {@code table} matching {@code predicate}, optionally limited to the first - * {@code limit} rows. - * - *

- * When {@code limit} is non-null, the statement is ordered by {@code orderByColumns} followed by - * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even - * when the ordering columns tie), so the "first {@code limit}" rows are identical for the original and transformed - * statements. Varying the ordering columns exercises more access paths than the row id alone would. The caller must - * pass the same {@code orderByColumns} and {@code limit} to both statements; neither is transformed. + * {@code limit} rows (see {@link #orderByLimitClause}). * * @param table * the table to delete from @@ -157,16 +173,67 @@ default String selectRowIdsStatement(T table) { * @return the SQL statement */ default String deleteStatement(T table, E predicate, List orderByColumns, Integer limit) { - String statement = "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); - if (limit != null) { - List orderBy = new ArrayList<>(); - for (C column : orderByColumns) { - orderBy.add(column.getName()); - } - orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above - statement += " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; + return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate) + + orderByLimitClause(orderByColumns, limit); + } + + /** + * SQL that updates the rows of {@code table} matching {@code predicate}, setting each column in {@code assignments} + * to its assigned value expression, optionally limited to the first {@code limit} rows (see + * {@link #orderByLimitClause}). + * + * @param table + * the table to update + * @param assignments + * the {@code (column, value expression)} pairs to assign; each value is rendered via {@link #asString} + * @param predicate + * the WHERE predicate; rendered via {@link #asString} + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty); only used when {@code limit} is + * non-null + * @param limit + * the maximum number of rows to update, or {@code null} for no limit + * + * @return the SQL statement + */ + default String updateStatement(T table, List> assignments, E predicate, List orderByColumns, + Integer limit) { + List setClauses = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + setClauses.add(assignment.getKey().getName() + " = " + asString(assignment.getValue())); + } + return "UPDATE " + table.getName() + " SET " + String.join(", ", setClauses) + " WHERE " + asString(predicate) + + orderByLimitClause(orderByColumns, limit); + } + + /** + * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and + * {@link #updateStatement}, or the empty string when {@code limit} is null. + * + *

+ * The rows are ordered by {@code orderByColumns} followed by {@link #ROW_ID_COLUMN} as a tiebreaker. Because the + * identifiers are unique, this is always a total order (even when the ordering columns tie), so the "first + * {@code limit}" rows are identical for the original and transformed statements. Varying the ordering columns + * exercises more access paths than the row id alone would. The caller must pass the same {@code orderByColumns} and + * {@code limit} to both statements; neither is transformed. + * + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty) + * @param limit + * the maximum number of rows, or {@code null} for no limit (yielding an empty clause) + * + * @return the {@code ORDER BY ... LIMIT n} clause, or the empty string when {@code limit} is null + */ + default String orderByLimitClause(List orderByColumns, Integer limit) { + if (limit == null) { + return ""; + } + List orderBy = new ArrayList<>(); + for (C column : orderByColumns) { + orderBy.add(column.getName()); } - return statement; + orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above + return " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; } /** diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index ce41039d6..3443c9756 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -1,11 +1,11 @@ package sqlancer.common.oracle; import java.sql.SQLException; -import java.util.HashSet; +import java.util.AbstractMap; +import java.util.ArrayList; import java.util.List; -import java.util.Set; +import java.util.Map; -import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLGlobalState; @@ -13,6 +13,7 @@ import sqlancer.common.gen.EETDMLGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; @@ -29,14 +30,17 @@ *

* Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row, and each statement is executed inside a transaction that is rolled back, so the two - * statements can be compared against the same starting state without permanently modifying the database. For a DELETE, - * the state is captured as the set of surviving row identifiers. Because rolling back a statement requires a - * transactional storage engine, the DBMS-specific setup must ensure only such engines are used while this oracle is - * active. + * statements can be compared against the same starting state without permanently modifying the database. The state is + * captured as a full post-image: each surviving row's identifier together with its content column values, ordered by + * the identifier. This single value-level surface covers every DML statement — a DELETE removes rows from it, an UPDATE + * changes values in it (row identity alone would suffice for DELETE, but not for UPDATE, which also transforms the + * written values). Because rolling back a statement requires a transactional storage engine, the DBMS-specific setup + * must ensure only such engines are used while this oracle is active. * *

- * Only DELETE is currently supported. Statement reduction is not yet implemented (there is no - * {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database reduction. + * DELETE and UPDATE are currently supported (one is chosen at random per check). Statement reduction is not yet + * implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database + * reduction. * * @param * the DBMS-specific expression class @@ -75,8 +79,9 @@ public void check() throws SQLException { if (tables.isEmpty()) { throw new IgnoreMeException(); } - // DELETE targets a single table, so operate on exactly one; confining the generator to it keeps the predicate - // from referencing another table's columns (which would render invalid single-table DML). + // A DML statement targets a single table, so operate on exactly one; confining the generator to it keeps the + // predicate and value expressions from referencing another table's columns (which would render invalid + // single-table DML). T table = Randomly.fromList(tables); gen = gen.setTablesAndColumns(new AbstractTables<>(List.of(table))); @@ -84,7 +89,7 @@ public void check() throws SQLException { // The WHERE predicate is evaluated in a boolean context. E transformedPredicate = transformer.transform(predicate, true); - // Optionally cap the DELETE with a LIMIT. The limit and its ordering (a random column subset, made a total + // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. Integer limit = null; @@ -93,30 +98,47 @@ public void check() throws SQLException { limit = (int) Randomly.getNotCachedInteger(0, 10); orderByColumns = Randomly.subset(table.getColumns()); } - String originalDelete = gen.deleteStatement(table, predicate, orderByColumns, limit); - String transformedDelete = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); - generatedQueryString = originalDelete; - - // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: - // the ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak - // the column into the next iteration and cause cascading duplicate-column failures. The row-identity setup - // touches rows, so — like the DELETE itself — it can raise tolerated errors (e.g. functional-index maintenance - // truncation); such an error aborts the iteration (IgnoreMeException) rather than being reported as a bug. + + String originalStatement; + String transformedStatement; + if (Randomly.getBoolean()) { + // UPDATE also transforms the written values: each SET value expression is transformed in a scalar context. + List> assignments = gen.generateSetAssignments(); + List> transformedAssignments = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + E transformedValue = transformer.transform(assignment.getValue(), false); + transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); + } + originalStatement = gen.updateStatement(table, assignments, predicate, orderByColumns, limit); + transformedStatement = gen.updateStatement(table, transformedAssignments, transformedPredicate, + orderByColumns, limit); + } else { + originalStatement = gen.deleteStatement(table, predicate, orderByColumns, limit); + transformedStatement = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); + } + generatedQueryString = originalStatement; + + // The post-image select reads the identifier plus every content column of the table, in that order. + int columnCount = table.getColumns().size() + 1; + + // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the + // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the + // column and cause cascading duplicate-column failures if (!new SQLQueryAdapter(gen.addRowIdColumnStatement(table), errors, true).execute(state)) { throw new IgnoreMeException(); } try { - // Stamp identifiers once, in autocommit mode, before both DELETEs run: both then observe the same rows. + // Stamp identifiers once, in autocommit mode, before both runs: both then observe the same rows. if (!new SQLQueryAdapter(gen.stampRowIdsStatement(table), errors).execute(state)) { throw new IgnoreMeException(); } - Set originalSurvivors = executeDeleteAndSnapshot(table, originalDelete); - Set transformedSurvivors = executeDeleteAndSnapshot(table, transformedDelete); + List> originalImage = executeAndSnapshotPostImage(table, originalStatement, columnCount); + List> transformedImage = executeAndSnapshotPostImage(table, transformedStatement, columnCount); - if (!originalSurvivors.equals(transformedSurvivors)) { + if (!originalImage.equals(transformedImage)) { throw new AssertionError( - mismatchMessage(originalDelete, transformedDelete, originalSurvivors, transformedSurvivors)); + mismatchMessage(originalStatement, transformedStatement, originalImage, transformedImage)); } } finally { new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); @@ -124,46 +146,102 @@ public void check() throws SQLException { } /** - * Executes {@code deleteStatement} inside a transaction that is always rolled back, and returns the set of row - * identifiers surviving the DELETE (the resulting database state). A DBMS error expected by the oracle aborts the - * whole check ({@link IgnoreMeException}) rather than being reported, matching {@link EETOracle}'s handling; an - * unexpected error surfaces as a bug ({@link AssertionError}, thrown by the query adapter). + * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: + * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A + * DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug or unexpected error + * surfaces as {@link AssertionError}. * * @param table - * the table being deleted from - * @param deleteStatement - * the DELETE statement to execute + * the table being modified + * @param statement + * the DML statement to execute + * @param columnCount + * the number of columns the post-image select returns (identifier plus content columns) * - * @return the set of row identifiers surviving the DELETE + * @return the post-image, as one string list (identifier followed by content column values) per surviving row * * @throws SQLException - * if a DBMS interaction fails + * if a DBMS interaction other than running {@code statement} fails; an error from {@code statement} + * itself instead surfaces as {@link IgnoreMeException} or {@link AssertionError} */ - private Set executeDeleteAndSnapshot(T table, String deleteStatement) throws SQLException { + private List> executeAndSnapshotPostImage(T table, String statement, int columnCount) + throws SQLException { new SQLQueryAdapter(gen.beginTransactionStatement()).execute(state); try { // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. - boolean succeeded = new SQLQueryAdapter(deleteStatement, errors).execute(state); + boolean succeeded = new SQLQueryAdapter(statement, errors).execute(state); if (!succeeded) { - // The DELETE hit an error the oracle tolerates; do not compare states (as EETOracle does for SELECT). + // The statement hit an error the oracle tolerates; do not compare states (as EETOracle does for + // SELECT). throw new IgnoreMeException(); } - return new HashSet<>( - ComparatorHelper.getResultSetFirstColumnAsString(gen.selectRowIdsStatement(table), errors, state)); + return snapshotPostImage(gen.selectPostImageStatement(table), columnCount); } finally { new SQLQueryAdapter(gen.rollbackTransactionStatement()).execute(state); } } - private static String mismatchMessage(String originalDelete, String transformedDelete, - Set originalSurvivors, Set transformedSurvivors) { + /** + * Reads the post-image produced by {@code selectStatement} into one string list per row (each column via + * {@code getString}). A DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug + * or unexpected error surfaces as {@link AssertionError}. + * + * @param selectStatement + * the post-image select to read; its columns are the identifier followed by the content columns + * @param columnCount + * the number of columns to read from each row + * + * @return the read rows, in the select's order + * + * @throws SQLException + * if cleanup fails (errors thrown elsewhere will always be rethrown as {@link IgnoreMeException} or + * {@link AssertionError}) + */ + private List> snapshotPostImage(String selectStatement, int columnCount) throws SQLException { + List> rows = new ArrayList<>(); + SQLQueryAdapter q = new SQLQueryAdapter(selectStatement, errors, true, + state.getOptions().canonicalizeSqlString()); + SQLancerResultSet result = null; + try { + result = q.executeAndGet(state); + if (result == null) { + throw new IgnoreMeException(); + } + while (result.next()) { + List row = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + row.add(result.getString(i)); + } + rows.add(row); + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw e; + } + Throwable current = e; + while (current != null) { + if (current.getMessage() != null && errors.errorIsExpected(current.getMessage())) { + throw new IgnoreMeException(); + } + current = current.getCause(); + } + throw new AssertionError(selectStatement, e); + } finally { + if (result != null && !result.isClosed()) { + result.close(); + } + } + return rows; + } + + private static String mismatchMessage(String originalStatement, String transformedStatement, + List> originalImage, List> transformedImage) { return new StringBuilder() - .append("-- The original and transformed DELETE statements left the database in different states") - .append(" (different sets of surviving rows):").append(System.lineSeparator()).append("-- original (") - .append(originalSurvivors.size()).append(" rows survive): ").append(originalDelete).append(';') - .append(System.lineSeparator()).append("-- transformed (").append(transformedSurvivors.size()) - .append(" rows survive): ").append(transformedDelete).append(';').append(System.lineSeparator()) - .toString(); + .append("-- The original and transformed statements left the database in different states") + .append(" (different post-images):").append(System.lineSeparator()).append("-- original (") + .append(originalImage.size()).append(" rows): ").append(originalStatement).append(';') + .append(System.lineSeparator()).append("-- transformed (").append(transformedImage.size()) + .append(" rows): ").append(transformedStatement).append(';').append(System.lineSeparator()).toString(); } @Override diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9a62c15ba..9fbaef12a 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -1,7 +1,9 @@ package sqlancer.mysql.gen; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -254,6 +256,18 @@ public MySQLExpression generateBooleanExpression() { return generateExpression(); } + @Override + public List> generateSetAssignments() { + List> assignments = new ArrayList<>(); + for (MySQLColumn column : Randomly.nonEmptySubset(columns)) { + // As with the normal UPDATE workload, the value is an arbitrary expression (not type-matched to the + // column); + // any resulting type/range error is on the oracle's expected-error allow-list. + assignments.add(new AbstractMap.SimpleEntry<>(column, generateExpression())); + } + return assignments; + } + @Override public MySQLSelect generateSelect() { return new MySQLSelect(); @@ -384,8 +398,8 @@ public String asString(MySQLExpression expr) { @Override public String stampRowIdsStatement(MySQLTable table) { // MySQL's UUID() gives each existing row a distinct value in a single statement. Stamping happens once, before - // both rolled-back DELETE runs, so both observe identical identifiers; the standard-SQL statements (add/drop - // column, delete, snapshot, transaction control) use EETDMLGenerator's defaults. + // both rolled-back statement runs, so both observe identical identifiers; the standard-SQL statements (add/drop + // column, delete/update, snapshot, transaction control) use EETDMLGenerator's defaults. return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); } From 1e53207c5770081e746e67a9ee1e5dac6abc5d94 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 20:11:53 +0800 Subject: [PATCH 1277/1284] Add row discrepancy information to test case logs for EET DML oracle --- src/sqlancer/common/oracle/EETDMLOracle.java | 65 +++++++++++++++++--- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 3443c9756..fbb2d2d3b 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -3,8 +3,12 @@ import java.sql.SQLException; import java.util.AbstractMap; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -61,6 +65,7 @@ public class EETDMLOracle, S extends AbstractSchema transformer; private final ExpectedErrors errors; + private static final int MAX_DIFF_ROWS_REPORTED = 10; // max differing post-image rows displayed in report log private String generatedQueryString; public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { @@ -137,8 +142,8 @@ public void check() throws SQLException { List> transformedImage = executeAndSnapshotPostImage(table, transformedStatement, columnCount); if (!originalImage.equals(transformedImage)) { - throw new AssertionError( - mismatchMessage(originalStatement, transformedStatement, originalImage, transformedImage)); + throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, originalImage, + transformedImage)); } } finally { new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); @@ -234,14 +239,56 @@ private List> snapshotPostImage(String selectStatement, int columnC return rows; } - private static String mismatchMessage(String originalStatement, String transformedStatement, + private String mismatchMessage(T table, String originalStatement, String transformedStatement, List> originalImage, List> transformedImage) { - return new StringBuilder() - .append("-- The original and transformed statements left the database in different states") - .append(" (different post-images):").append(System.lineSeparator()).append("-- original (") - .append(originalImage.size()).append(" rows): ").append(originalStatement).append(';') - .append(System.lineSeparator()).append("-- transformed (").append(transformedImage.size()) - .append(" rows): ").append(transformedStatement).append(';').append(System.lineSeparator()).toString(); + List header = new ArrayList<>(); + header.add(EETDMLGenerator.ROW_ID_COLUMN); + for (C column : table.getColumns()) { + header.add(column.getName()); + } + + Map> originalByRowId = indexByRowId(originalImage); + Map> transformedByRowId = indexByRowId(transformedImage); + Set allRowIds = new TreeSet<>(); + allRowIds.addAll(originalByRowId.keySet()); + allRowIds.addAll(transformedByRowId.keySet()); + + String nl = System.lineSeparator(); + StringBuilder message = new StringBuilder() + .append("-- The original and transformed statements left the database in different states.").append(nl) + .append("-- original: ").append(originalStatement).append(';').append(nl).append("-- transformed: ") + .append(transformedStatement).append(';').append(nl).append("-- differing post-image rows (") + .append(String.join(", ", header)).append("):").append(nl); + int shown = 0; + for (String rowId : allRowIds) { + List originalRow = originalByRowId.get(rowId); + List transformedRow = transformedByRowId.get(rowId); + if (Objects.equals(originalRow, transformedRow)) { + continue; + } + if (shown == MAX_DIFF_ROWS_REPORTED) { + message.append("-- ... (further differences omitted)").append(nl); + break; + } + message.append("-- original: ").append(renderRow(originalRow)).append(nl); + message.append("-- transformed: ").append(renderRow(transformedRow)).append(nl); + shown++; + } + return message.toString(); + } + + // Indexes a post-image by its row identifier (the first column of each row) + private static Map> indexByRowId(List> image) { + Map> byRowId = new LinkedHashMap<>(); + for (List row : image) { + byRowId.put(row.get(0), row); + } + return byRowId; + } + + // Renders a post-image row for the finding message, or "(row absent)" when the row is missing on that side + private static String renderRow(List row) { + return row == null ? "(row absent)" : row.toString(); } @Override From 63f50b826a7b217ed12f4fdd8aeb10613ea166df Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 12 Aug 2026 09:28:31 +0800 Subject: [PATCH 1278/1284] Remove hard-coding of EET DML rowId column being at index 0; derive it instead --- src/sqlancer/common/gen/EETDMLGenerator.java | 26 ++++++++++++++++---- src/sqlancer/common/oracle/EETDMLOracle.java | 21 +++++++--------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 9fb21ea35..0c84901f3 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -145,15 +145,31 @@ default String dropRowIdColumnStatement(T table) { * @param table * the table to snapshot * - * @return the SQL statement; its first result column is the identifier, followed by {@code table}'s content columns + * @return the SQL statement; its result columns are those of {@link #postImageColumns}, in that order */ default String selectPostImageStatement(T table) { - List selected = new ArrayList<>(); - selected.add(ROW_ID_COLUMN); + return "SELECT " + String.join(", ", postImageColumns(table)) + " FROM " + table.getName() + " ORDER BY " + + ROW_ID_COLUMN; + } + + /** + * The columns a post-image row consists of, in the order {@link #selectPostImageStatement} returns them: the + * {@link #ROW_ID_COLUMN} identifier followed by {@code table}'s content columns. This is the sole definition of the + * post-image layout, so a consumer can find the identifier's position by looking up {@link #ROW_ID_COLUMN} here + * rather than assuming one. + * + * @param table + * the table being snapshot + * + * @return the post-image column names, in order + */ + default List postImageColumns(T table) { + List columns = new ArrayList<>(); + columns.add(ROW_ID_COLUMN); for (C column : table.getColumns()) { - selected.add(column.getName()); + columns.add(column.getName()); } - return "SELECT " + String.join(", ", selected) + " FROM " + table.getName() + " ORDER BY " + ROW_ID_COLUMN; + return columns; } /** diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index fbb2d2d3b..eb453d9b6 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -123,8 +123,7 @@ public void check() throws SQLException { } generatedQueryString = originalStatement; - // The post-image select reads the identifier plus every content column of the table, in that order. - int columnCount = table.getColumns().size() + 1; + int columnCount = gen.postImageColumns(table).size(); // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the @@ -241,14 +240,12 @@ private List> snapshotPostImage(String selectStatement, int columnC private String mismatchMessage(T table, String originalStatement, String transformedStatement, List> originalImage, List> transformedImage) { - List header = new ArrayList<>(); - header.add(EETDMLGenerator.ROW_ID_COLUMN); - for (C column : table.getColumns()) { - header.add(column.getName()); - } + List header = gen.postImageColumns(table); + // Where the identifier sits within a post-image row, per the layout the generator defines + int rowIdIndex = header.indexOf(EETDMLGenerator.ROW_ID_COLUMN); - Map> originalByRowId = indexByRowId(originalImage); - Map> transformedByRowId = indexByRowId(transformedImage); + Map> originalByRowId = indexByRowId(originalImage, rowIdIndex); + Map> transformedByRowId = indexByRowId(transformedImage, rowIdIndex); Set allRowIds = new TreeSet<>(); allRowIds.addAll(originalByRowId.keySet()); allRowIds.addAll(transformedByRowId.keySet()); @@ -277,11 +274,11 @@ private String mismatchMessage(T table, String originalStatement, String transfo return message.toString(); } - // Indexes a post-image by its row identifier (the first column of each row) - private static Map> indexByRowId(List> image) { + // Indexes a post-image by its row identifier, which each row holds at rowIdIndex + private static Map> indexByRowId(List> image, int rowIdIndex) { Map> byRowId = new LinkedHashMap<>(); for (List row : image) { - byRowId.put(row.get(0), row); + byRowId.put(row.get(rowIdIndex), row); } return byRowId; } From 1ce0341485a5052d9055a120cc3bf98321c3d0ee Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 12 Aug 2026 09:53:21 +0800 Subject: [PATCH 1279/1284] Refactor EET DML statement generation to use dedicated methods for update and delete --- src/sqlancer/common/oracle/EETDMLOracle.java | 86 ++++++++++++++++---- 1 file changed, 69 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index eb453d9b6..4731c04ed 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -104,23 +104,11 @@ public void check() throws SQLException { orderByColumns = Randomly.subset(table.getColumns()); } - String originalStatement; - String transformedStatement; - if (Randomly.getBoolean()) { - // UPDATE also transforms the written values: each SET value expression is transformed in a scalar context. - List> assignments = gen.generateSetAssignments(); - List> transformedAssignments = new ArrayList<>(); - for (Map.Entry assignment : assignments) { - E transformedValue = transformer.transform(assignment.getValue(), false); - transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); - } - originalStatement = gen.updateStatement(table, assignments, predicate, orderByColumns, limit); - transformedStatement = gen.updateStatement(table, transformedAssignments, transformedPredicate, - orderByColumns, limit); - } else { - originalStatement = gen.deleteStatement(table, predicate, orderByColumns, limit); - transformedStatement = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); - } + StatementPair statements = Randomly.getBoolean() + ? generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit) + : generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit); + String originalStatement = statements.original; + String transformedStatement = statements.transformed; generatedQueryString = originalStatement; int columnCount = gen.postImageColumns(table).size(); @@ -149,6 +137,70 @@ public void check() throws SQLException { } } + /** + * A DML statement and its transformed counterpart, which must leave the database in the same state. + */ + private static final class StatementPair { + private final String original; + private final String transformed; + + StatementPair(String original, String transformed) { + this.original = original; + this.transformed = transformed; + } + } + + /** + * Generates an UPDATE and its transformed counterpart. Besides the WHERE predicate, UPDATE also transforms the + * written values: each SET value expression is transformed in a scalar context. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the statement, empty if it is not capped by a limit + * @param limit + * the maximum number of rows to modify, or {@code null} for no limit + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateUpdateStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { + List> assignments = gen.generateSetAssignments(); + List> transformedAssignments = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + E transformedValue = transformer.transform(assignment.getValue(), false); + transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); + } + return new StatementPair(gen.updateStatement(table, assignments, predicate, orderByColumns, limit), + gen.updateStatement(table, transformedAssignments, transformedPredicate, orderByColumns, limit)); + } + + /** + * Generates a DELETE and its transformed counterpart, which differ only in their WHERE predicate. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the statement, empty if it is not capped by a limit + * @param limit + * the maximum number of rows to modify, or {@code null} for no limit + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateDeleteStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { + return new StatementPair(gen.deleteStatement(table, predicate, orderByColumns, limit), + gen.deleteStatement(table, transformedPredicate, orderByColumns, limit)); + } + /** * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A From 4cf31d15f2f841686365f0167ad489222eb4b90f Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 13 Aug 2026 13:40:00 +0800 Subject: [PATCH 1280/1284] Implement INSERT support for EET DML oracle --- src/sqlancer/common/gen/EETDMLGenerator.java | 72 +++++++++++++++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 61 ++++++++++++---- .../mysql/gen/MySQLExpressionGenerator.java | 16 +++++ 3 files changed, 130 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 0c84901f3..c10c59b5a 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -19,9 +19,9 @@ *

* Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run - * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are - * produced. The resulting state is compared as a full post-image (each surviving row's identifier and content column - * values), which covers every DML statement: a DELETE removes rows from it, an UPDATE changes values in it. + * (each in a rolled-back transaction), so both executions observe the same identifiers. The resulting state is compared + * as a full post-image (each surviving row's identifier and content column values), which covers any of the three DML + * statements (DELETE, UPDATE, INSERT). * *

* Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. @@ -66,6 +66,15 @@ public interface EETDMLGenerator, T extends AbstractTabl */ List> generateSetAssignments(); + /** + * Generates a fresh value expression for each content column of the current table, used as an INSERT statement's + * inserted values. The returned expressions are positionally aligned with {@link AbstractTable#getColumns()}, and + * each is transformed by the oracle. + * + * @return one fresh random value expression per content column, in {@link AbstractTable#getColumns()} order + */ + List generateInsertValues(); + /** * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's * expressions into semantically equivalent ones. @@ -106,6 +115,17 @@ public interface EETDMLGenerator, T extends AbstractTabl */ String rowIdColumnType(); + /** + * A SQL expression, evaluated once per source row of an {@code INSERT ... SELECT}, that derives the inserted row's + * {@link #ROW_ID_COLUMN} value from the source row's identifier. It must be deterministic (so both the original and + * transformed statements assign the same identifiers), unique per source row, and distinct from every existing + * identifier (so an inserted row never collides with the source row it was derived from in the post-image). DBMS- + * specific because it names a suitable derivation function (e.g. a hash of the source identifier). + * + * @return the SQL expression deriving an inserted row's identifier from the source row's {@link #ROW_ID_COLUMN} + */ + String insertedRowIdExpression(); + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- /** @@ -139,8 +159,9 @@ default String dropRowIdColumnStatement(T table) { * *

* This single value-level snapshot is the comparison surface for all DML statements: a DELETE removes rows from it, - * an UPDATE changes column values in it. Row identity alone (which the identifier already captures) would suffice - * for DELETE, but not for UPDATE, where the two runs could touch the same rows yet write different values. + * an UPDATE changes column values in it, an INSERT adds rows to it. Row identity alone (which the identifier + * already captures) would suffice for DELETE, but not for UPDATE, where the two runs could touch the same rows yet + * write different values. * * @param table * the table to snapshot @@ -222,6 +243,47 @@ default String updateStatement(T table, List> assignments, E pre + orderByLimitClause(orderByColumns, limit); } + /** + * SQL that inserts a new row into {@code table} for each source row (optionally filtered by {@code predicate}), + * setting each content column to its corresponding value in {@code values}. + * + *

+ * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value + * expressions reference the table's columns (the transformer injects column references into its equivalent + * sub-expressions), which are legal in a {@code SELECT} but not in a {@code VALUES} clause. Each inserted row's + * {@link #ROW_ID_COLUMN} is derived from its source row via {@link #insertedRowIdExpression()}, giving it a + * deterministic identifier that is unique and distinct from every existing one, so the two statements' post-images + * align (and inserted rows never collide with their source rows). + * + * @param table + * the table to insert into + * @param values + * one value expression per content column, positionally aligned with {@link AbstractTable#getColumns()}; + * each is rendered via {@link #asString} + * @param predicate + * the WHERE predicate filtering the source rows, or {@code null} to insert from every source row; + * rendered via {@link #asString} + * + * @return the SQL statement + */ + default String insertStatement(T table, List values, E predicate) { + List columnNames = new ArrayList<>(); + columnNames.add(ROW_ID_COLUMN); + List selectItems = new ArrayList<>(); + selectItems.add(insertedRowIdExpression()); + List columns = table.getColumns(); + for (int i = 0; i < columns.size(); i++) { + columnNames.add(columns.get(i).getName()); + selectItems.add(asString(values.get(i))); + } + String statement = "INSERT INTO " + table.getName() + " (" + String.join(", ", columnNames) + ") SELECT " + + String.join(", ", selectItems) + " FROM " + table.getName(); + if (predicate != null) { + statement += " WHERE " + asString(predicate); + } + return statement; + } + /** * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and * {@link #updateStatement}, or the empty string when {@code limit} is null. diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 4731c04ed..ba9b9d6c7 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -9,6 +9,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import java.util.function.Supplier; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -37,13 +38,15 @@ * statements can be compared against the same starting state without permanently modifying the database. The state is * captured as a full post-image: each surviving row's identifier together with its content column values, ordered by * the identifier. This single value-level surface covers every DML statement — a DELETE removes rows from it, an UPDATE - * changes values in it (row identity alone would suffice for DELETE, but not for UPDATE, which also transforms the - * written values). Because rolling back a statement requires a transactional storage engine, the DBMS-specific setup - * must ensure only such engines are used while this oracle is active. + * changes values in it, an INSERT adds rows to it (row identity alone would suffice for DELETE, but not for UPDATE, + * which also transforms the written values). Because rolling back a statement requires a transactional storage engine, + * the DBMS-specific setup must ensure only such engines are used while this oracle is active. * *

- * DELETE and UPDATE are currently supported (one is chosen at random per check). Statement reduction is not yet - * implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database + * DELETE, UPDATE and INSERT are currently supported (one is chosen at random per check). INSERT uses the + * {@code INSERT ... SELECT} form so its transformed value expressions may reference columns; each inserted row is given + * a deterministic identifier derived from its source row so the two runs' post-images align. Statement reduction is not + * yet implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database * reduction. * * @param @@ -97,16 +100,16 @@ public void check() throws SQLException { // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. - Integer limit = null; - List orderByColumns = List.of(); - if (Randomly.getBoolean()) { - limit = (int) Randomly.getNotCachedInteger(0, 10); - orderByColumns = Randomly.subset(table.getColumns()); - } + boolean withLimit = Randomly.getBoolean(); + Integer limit = withLimit ? (int) Randomly.getNotCachedInteger(0, 10) : null; + List orderByColumns = withLimit ? Randomly.subset(table.getColumns()) : List.of(); - StatementPair statements = Randomly.getBoolean() - ? generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit) - : generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit); + // Generators for the different kinds of statement this oracle supports. One is chosen at random per check + List> statementGenerators = List.of( + () -> generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit), + () -> generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit), + () -> generateInsertStatements(table, predicate, transformedPredicate)); + StatementPair statements = Randomly.fromList(statementGenerators).get(); String originalStatement = statements.original; String transformedStatement = statements.transformed; generatedQueryString = originalStatement; @@ -201,6 +204,36 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform gen.deleteStatement(table, transformedPredicate, orderByColumns, limit)); } + /** + * Generates an {@code INSERT ... SELECT} and its transformed counterpart. Besides the WHERE predicate, which + * filters the source rows and is optional here, INSERT also transforms each inserted value in a scalar context. + * + *

+ * Unlike DELETE and UPDATE, no limit is applied: {@link EETDMLGenerator#insertStatement} renders no ordering or + * limit, so one row is inserted per source row the predicate keeps. Nothing about INSERT rules a limit out — its + * source SELECT could carry the same ordering and limit the other statement kinds use, and the two runs would still + * read the same source rows — it is just not generated. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate) { + List values = gen.generateInsertValues(); + List transformedValues = new ArrayList<>(); + for (E value : values) { + transformedValues.add(transformer.transform(value, false)); + } + boolean withPredicate = Randomly.getBoolean(); + return new StatementPair(gen.insertStatement(table, values, withPredicate ? predicate : null), + gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null)); + } + /** * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9fbaef12a..baea11f65 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -268,6 +268,14 @@ public List> generateSetAssignments() { return assignments; } + @Override + public List generateInsertValues() { + // One value per content column, in schema order (aligned with the INSERT column list). As with the normal + // INSERT workload, each value is an arbitrary expression (not type-matched to the column); any resulting + // type/range/constraint error is on the oracle's expected-error allow-list. + return columns.stream().map(c -> generateExpression()).collect(Collectors.toList()); + } + @Override public MySQLSelect generateSelect() { return new MySQLSelect(); @@ -408,4 +416,12 @@ public String rowIdColumnType() { // Holds a 36-character UUID string produced by stampRowIdsStatement. return "VARCHAR(36)"; } + + @Override + public String insertedRowIdExpression() { + // The source row's identifier with its dashes removed: deterministic (identical across both runs) and unique + // per + // source row. Fits the identifier column's VARCHAR(36). + return String.format("REPLACE(%s, '-', '')", ROW_ID_COLUMN); + } } From 7ead2e762106cdab3c71be9b094b1c42b033afb7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 13 Aug 2026 13:50:17 +0800 Subject: [PATCH 1281/1284] Add support for LIMIT on source rows of INSERT ... SELECT statements in EET --- src/sqlancer/common/gen/EETDMLGenerator.java | 16 ++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 53 ++++++++++++++------ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index c10c59b5a..557220d06 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -245,7 +245,8 @@ default String updateStatement(T table, List> assignments, E pre /** * SQL that inserts a new row into {@code table} for each source row (optionally filtered by {@code predicate}), - * setting each content column to its corresponding value in {@code values}. + * setting each content column to its corresponding value in {@code values}, optionally limited to the first + * {@code limit} source rows (see {@link #orderByLimitClause}). * *

* The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value @@ -263,10 +264,15 @@ default String updateStatement(T table, List> assignments, E pre * @param predicate * the WHERE predicate filtering the source rows, or {@code null} to insert from every source row; * rendered via {@link #asString} + * @param orderByColumns + * the columns to order the source rows by before the row-id tiebreaker (may be empty); only used when + * {@code limit} is non-null + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit * * @return the SQL statement */ - default String insertStatement(T table, List values, E predicate) { + default String insertStatement(T table, List values, E predicate, List orderByColumns, Integer limit) { List columnNames = new ArrayList<>(); columnNames.add(ROW_ID_COLUMN); List selectItems = new ArrayList<>(); @@ -281,12 +287,12 @@ default String insertStatement(T table, List values, E predicate) { if (predicate != null) { statement += " WHERE " + asString(predicate); } - return statement; + return statement + orderByLimitClause(orderByColumns, limit); } /** - * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and - * {@link #updateStatement}, or the empty string when {@code limit} is null. + * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement}, + * {@link #updateStatement} and {@link #insertStatement}, or the empty string when {@code limit} is null. * *

* The rows are ordered by {@code orderByColumns} followed by {@link #ROW_ID_COLUMN} as a tiebreaker. Because the diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index ba9b9d6c7..02bff8165 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -9,7 +9,6 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; -import java.util.function.Supplier; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -100,16 +99,18 @@ public void check() throws SQLException { // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. - boolean withLimit = Randomly.getBoolean(); - Integer limit = withLimit ? (int) Randomly.getNotCachedInteger(0, 10) : null; - List orderByColumns = withLimit ? Randomly.subset(table.getColumns()) : List.of(); + Integer limit = null; + List orderByColumns = List.of(); + if (Randomly.getBoolean()) { + limit = (int) Randomly.getNotCachedInteger(0, 10); + orderByColumns = Randomly.subset(table.getColumns()); + } // Generators for the different kinds of statement this oracle supports. One is chosen at random per check - List> statementGenerators = List.of( - () -> generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit), - () -> generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit), - () -> generateInsertStatements(table, predicate, transformedPredicate)); - StatementPair statements = Randomly.fromList(statementGenerators).get(); + List> statementGenerators = List.of(this::generateDeleteStatements, + this::generateUpdateStatements, this::generateInsertStatements); + StatementPair statements = Randomly.fromList(statementGenerators).generate(table, predicate, + transformedPredicate, orderByColumns, limit); String originalStatement = statements.original; String transformedStatement = statements.transformed; generatedQueryString = originalStatement; @@ -140,6 +141,22 @@ public void check() throws SQLException { } } + /** + * Generates a DML statement of one kind together with its transformed counterpart. The kinds share this signature + * so the oracle can pick one of them at random per check. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ + @FunctionalInterface + private interface DMLStatementGenerator { + StatementPair generate(T table, E predicate, E transformedPredicate, List orderByColumns, Integer limit); + } + /** * A DML statement and its transformed counterpart, which must leave the database in the same state. */ @@ -209,10 +226,7 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform * filters the source rows and is optional here, INSERT also transforms each inserted value in a scalar context. * *

- * Unlike DELETE and UPDATE, no limit is applied: {@link EETDMLGenerator#insertStatement} renders no ordering or - * limit, so one row is inserted per source row the predicate keeps. Nothing about INSERT rules a limit out — its - * source SELECT could carry the same ordering and limit the other statement kinds use, and the two runs would still - * read the same source rows — it is just not generated. + * The ordering and limit cap the source rows the statement reads, so it inserts one row per source row kept. * * @param table * the table being modified @@ -220,18 +234,25 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform * the WHERE predicate of the original statement * @param transformedPredicate * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the source rows, empty if the statement is not capped by a limit + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit * * @return the original statement together with its transformed counterpart */ - private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate) { + private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { List values = gen.generateInsertValues(); List transformedValues = new ArrayList<>(); for (E value : values) { transformedValues.add(transformer.transform(value, false)); } boolean withPredicate = Randomly.getBoolean(); - return new StatementPair(gen.insertStatement(table, values, withPredicate ? predicate : null), - gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null)); + return new StatementPair( + gen.insertStatement(table, values, withPredicate ? predicate : null, orderByColumns, limit), + gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null, + orderByColumns, limit)); } /** From f46934ecb6ec699c0315631e38e71a8e97e8b27f Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 10:11:10 +0800 Subject: [PATCH 1282/1284] Improve robustness of MySQL EET DML by ensuring InnoDB engine is always chosen --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 0746056ad..d27d6e681 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -164,7 +164,15 @@ public static List getRandomTableOptions() { } private void appendTableOptions() { - List tableOptions = TableOptions.getRandomTableOptions(); + List tableOptions = new ArrayList<>(TableOptions.getRandomTableOptions()); + // The EET DML oracle rolls back each statement to compare database states, which requires a transactional + // engine. The ENGINE option already forces InnoDB when the oracle is active (see the ENGINE case below), but it + // is only emitted when randomly chosen; otherwise the table would inherit the server's default engine, which is + // not guaranteed transactional. Force the option to always be present so the engine is never left to the + // server default. + if (globalState.usesEETDML() && !tableOptions.contains(TableOptions.ENGINE)) { + tableOptions.add(TableOptions.ENGINE); + } int i = 0; for (TableOptions o : tableOptions) { if (i++ != 0) { From f1e65caeb45e122bd968fdf7f7a6e783edd122d2 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 14 Aug 2026 10:27:43 +0800 Subject: [PATCH 1283/1284] Clarify EET DML insertStatement documentation --- src/sqlancer/common/gen/EETDMLGenerator.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 557220d06..9dbc9f0d5 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -249,12 +249,12 @@ default String updateStatement(T table, List> assignments, E pre * {@code limit} source rows (see {@link #orderByLimitClause}). * *

- * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value - * expressions reference the table's columns (the transformer injects column references into its equivalent - * sub-expressions), which are legal in a {@code SELECT} but not in a {@code VALUES} clause. Each inserted row's - * {@link #ROW_ID_COLUMN} is derived from its source row via {@link #insertedRowIdExpression()}, giving it a - * deterministic identifier that is unique and distinct from every existing one, so the two statements' post-images - * align (and inserted rows never collide with their source rows). + * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because it reuses the source-row + * model already shared by {@link #deleteStatement} and {@link #updateStatement}, and because it offers two kinds of + * transformable expression in one statement (the inserted values and the WHERE predicate) rather than the values + * alone. Each inserted row's {@link #ROW_ID_COLUMN} is derived from its source row via + * {@link #insertedRowIdExpression()}, giving it a deterministic identifier that is unique and distinct from every + * existing one, so the two statements' post-images align (and inserted rows never collide with their source rows). * * @param table * the table to insert into From 3c707e66c0fef2ba22cd29aa724bab54b4872549 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 31 Jul 2026 11:44:33 +0800 Subject: [PATCH 1284/1284] Add reproducer to EET DML --- src/sqlancer/common/oracle/EETDMLOracle.java | 185 ++++++++++++++++--- 1 file changed, 158 insertions(+), 27 deletions(-) diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 02bff8165..77a8b528a 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -12,6 +12,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.SQLGlobalState; import sqlancer.common.ast.newast.Expression; import sqlancer.common.gen.EETDMLGenerator; @@ -44,9 +45,9 @@ *

* DELETE, UPDATE and INSERT are currently supported (one is chosen at random per check). INSERT uses the * {@code INSERT ... SELECT} form so its transformed value expressions may reference columns; each inserted row is given - * a deterministic identifier derived from its source row so the two runs' post-images align. Statement reduction is not - * yet implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database - * reduction. + * a deterministic identifier derived from its source row so the two runs' post-images align. To support reduction, a + * {@link Reproducer} replays the whole comparison (adding and stamping the row-identifier column, running both + * statements in rolled-back transactions and comparing the post-images) against the reduced database. * * @param * the DBMS-specific expression class @@ -69,6 +70,93 @@ public class EETDMLOracle, S extends AbstractSchema reproducer; + + // The SQL and metadata to run and observe one DML comparison, captured as strings so a reproducer can replay it + // against a reduced database without the generator or live schema objects. + private static final class ComparisonQueries { + private final String originalStatement; + private final String transformedStatement; + private final String addRowIdColumn; + private final String stampRowIds; + private final String beginTransaction; + private final String rollback; + private final String dropRowIdColumn; + private final String selectPostImage; + private final int columnCount; + + ComparisonQueries(String originalStatement, String transformedStatement, String addRowIdColumn, + String stampRowIds, String beginTransaction, String rollback, String dropRowIdColumn, + String selectPostImage, int columnCount) { + this.originalStatement = originalStatement; + this.transformedStatement = transformedStatement; + this.addRowIdColumn = addRowIdColumn; + this.stampRowIds = stampRowIds; + this.beginTransaction = beginTransaction; + this.rollback = rollback; + this.dropRowIdColumn = dropRowIdColumn; + this.selectPostImage = selectPostImage; + this.columnCount = columnCount; + } + } + + // The post-images the original and transformed statements produced, compared for equality to detect the bug. + private static final class PostImages { + private final List> original; + private final List> transformed; + + PostImages(List> original, List> transformed) { + this.original = original; + this.transformed = transformed; + } + } + + // Reproduces a post-image mismatch against the reduced database. Unlike EETOracle's comparison reproducer this does + // not extend AbstractComparisonReproducer: the two sides are not independent, because the row-id stamping (UUID()) + // must run once so both observe the same rows, so both post-images are computed together. + private final class EETDMLReproducer implements Reproducer { + private final ComparisonQueries queries; + + EETDMLReproducer(ComparisonQueries queries) { + this.queries = queries; + } + + @Override + public boolean bugStillTriggers(G globalState) { + PostImages images; + try { + images = computePostImages(globalState, queries); + } catch (AssertionError | SQLException | RuntimeException e) { + // any failure re-running the comparison means this reduced database no longer shows the mismatch + return false; + } + return !images.original.equals(images.transformed); + } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + sb.append("-- On the database set up by the statements above, the following statements leave the database" + + " in different states:").append(System.lineSeparator()); + renderStatementLines(sb, queries); + return sb.toString(); + } + } + + // Builds the reproducer for an unexpected DBMS error, which replays the whole comparison and checks the same error + // still fires. + private UnexpectedErrorReproducer errorReproducer(ComparisonQueries queries, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> computePostImages(globalState, queries); + StringBuilder sb = new StringBuilder(); + renderStatementLines(sb, queries); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + + // Renders the failing statements as commented lines, shared by the mismatch and the unexpected-error reproducers. + private static void renderStatementLines(StringBuilder sb, ComparisonQueries queries) { + sb.append("-- original: ").append(queries.originalStatement).append(';').append(System.lineSeparator()); + sb.append("-- transformed: ").append(queries.transformedStatement).append(';').append(System.lineSeparator()); + } public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { @@ -82,6 +170,7 @@ public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expect @Override public void check() throws SQLException { + reproducer = null; List tables = state.getSchema().getDatabaseTables(); if (tables.isEmpty()) { throw new IgnoreMeException(); @@ -115,29 +204,63 @@ public void check() throws SQLException { String transformedStatement = statements.transformed; generatedQueryString = originalStatement; - int columnCount = gen.postImageColumns(table).size(); + // Capture, as strings, everything needed to run and observe this comparison: the two statements plus the + // auxiliary-column setup, per-run snapshot and teardown. A reproducer replays these against a reduced database, + // where the live generator and schema objects no longer apply. + ComparisonQueries queries = new ComparisonQueries(originalStatement, transformedStatement, + gen.addRowIdColumnStatement(table), gen.stampRowIdsStatement(table), gen.beginTransactionStatement(), + gen.rollbackTransactionStatement(), gen.dropRowIdColumnStatement(table), + gen.selectPostImageStatement(table), gen.postImageColumns(table).size()); + PostImages images; + try { + images = computePostImages(state, queries); + } catch (AssertionError unexpectedError) { + reproducer = errorReproducer(queries, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + reproducer = new EETDMLReproducer(queries); + if (!images.original.equals(images.transformed)) { + throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, images.original, + images.transformed)); + } + } + + /** + * Runs the whole comparison against {@code globalState}: adds and stamps the row-identifier column once (so both + * runs observe the same rows), snapshots the post-image each statement produces (each inside a rolled-back + * transaction), and drops the column. Both {@link #check()} and the reproducers call this, the former against the + * live database and the latter against a reduced one. A DBMS error the oracle tolerates aborts with + * {@link IgnoreMeException}; an oracle logic bug or unexpected error surfaces as {@link AssertionError}. + * + * @param globalState + * the state whose connection the comparison runs against + * @param queries + * the statements and auxiliary SQL to run + * + * @return the post-images the original and transformed statements produced + * + * @throws SQLException + * if a DBMS interaction fails + */ + private PostImages computePostImages(G globalState, ComparisonQueries queries) throws SQLException { // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the // column and cause cascading duplicate-column failures - if (!new SQLQueryAdapter(gen.addRowIdColumnStatement(table), errors, true).execute(state)) { + if (!new SQLQueryAdapter(queries.addRowIdColumn, errors, true).execute(globalState)) { throw new IgnoreMeException(); } try { // Stamp identifiers once, in autocommit mode, before both runs: both then observe the same rows. - if (!new SQLQueryAdapter(gen.stampRowIdsStatement(table), errors).execute(state)) { + if (!new SQLQueryAdapter(queries.stampRowIds, errors).execute(globalState)) { throw new IgnoreMeException(); } - - List> originalImage = executeAndSnapshotPostImage(table, originalStatement, columnCount); - List> transformedImage = executeAndSnapshotPostImage(table, transformedStatement, columnCount); - - if (!originalImage.equals(transformedImage)) { - throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, originalImage, - transformedImage)); - } + List> original = snapshotSide(globalState, queries.originalStatement, queries); + List> transformed = snapshotSide(globalState, queries.transformedStatement, queries); + return new PostImages(original, transformed); } finally { - new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); + new SQLQueryAdapter(queries.dropRowIdColumn, errors, true).execute(globalState); } } @@ -261,12 +384,12 @@ private StatementPair generateInsertStatements(T table, E predicate, E transform * DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug or unexpected error * surfaces as {@link AssertionError}. * - * @param table - * the table being modified + * @param globalState + * the state whose connection the statement runs against * @param statement * the DML statement to execute - * @param columnCount - * the number of columns the post-image select returns (identifier plus content columns) + * @param queries + * supplies the transaction control and post-image select SQL and the post-image's column count * * @return the post-image, as one string list (identifier followed by content column values) per surviving row * @@ -274,20 +397,20 @@ private StatementPair generateInsertStatements(T table, E predicate, E transform * if a DBMS interaction other than running {@code statement} fails; an error from {@code statement} * itself instead surfaces as {@link IgnoreMeException} or {@link AssertionError} */ - private List> executeAndSnapshotPostImage(T table, String statement, int columnCount) + private List> snapshotSide(G globalState, String statement, ComparisonQueries queries) throws SQLException { - new SQLQueryAdapter(gen.beginTransactionStatement()).execute(state); + new SQLQueryAdapter(queries.beginTransaction).execute(globalState); try { // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. - boolean succeeded = new SQLQueryAdapter(statement, errors).execute(state); + boolean succeeded = new SQLQueryAdapter(statement, errors).execute(globalState); if (!succeeded) { // The statement hit an error the oracle tolerates; do not compare states (as EETOracle does for // SELECT). throw new IgnoreMeException(); } - return snapshotPostImage(gen.selectPostImageStatement(table), columnCount); + return snapshotPostImage(globalState, queries.selectPostImage, queries.columnCount); } finally { - new SQLQueryAdapter(gen.rollbackTransactionStatement()).execute(state); + new SQLQueryAdapter(queries.rollback).execute(globalState); } } @@ -296,6 +419,8 @@ private List> executeAndSnapshotPostImage(T table, String statement * {@code getString}). A DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug * or unexpected error surfaces as {@link AssertionError}. * + * @param globalState + * the state whose connection the select runs against * @param selectStatement * the post-image select to read; its columns are the identifier followed by the content columns * @param columnCount @@ -307,13 +432,14 @@ private List> executeAndSnapshotPostImage(T table, String statement * if cleanup fails (errors thrown elsewhere will always be rethrown as {@link IgnoreMeException} or * {@link AssertionError}) */ - private List> snapshotPostImage(String selectStatement, int columnCount) throws SQLException { + private List> snapshotPostImage(G globalState, String selectStatement, int columnCount) + throws SQLException { List> rows = new ArrayList<>(); SQLQueryAdapter q = new SQLQueryAdapter(selectStatement, errors, true, - state.getOptions().canonicalizeSqlString()); + globalState.getOptions().canonicalizeSqlString()); SQLancerResultSet result = null; try { - result = q.executeAndGet(state); + result = q.executeAndGet(globalState); if (result == null) { throw new IgnoreMeException(); } @@ -398,4 +524,9 @@ private static String renderRow(List row) { public String getLastQueryString() { return generatedQueryString; } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } }